test_write_cache.cpp 6.27 KB
Newer Older
Huihuo Zheng's avatar
Huihuo Zheng committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#include "hdf5.h"
#include "mpi.h"
#include "stdlib.h"
#include "stdio.h"
#include <sys/time.h>
#include <string.h>
#include "timing.h"
#include "H5Dio_cache.h"
#include <stdlib.h>
#include <pthread.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <unistd.h>
#include "mpi.h"
#include <string.h>
#include <unistd.h>
#include <stdio.h>
#include <sys/statvfs.h>
#include <stdlib.h>
#include "stat.h"
#include "debug.h"
Huihuo Zheng's avatar
Huihuo Zheng committed
23
#include <unistd.h>
Huihuo Zheng's avatar
Huihuo Zheng committed
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
int msleep(long miliseconds)
{
  struct timespec req, rem;

  if(miliseconds > 999)
    {   
      req.tv_sec = (int)(miliseconds / 1000);                            /* Must be Non-Negative */
      req.tv_nsec = (miliseconds - ((long)req.tv_sec * 1000)) * 1000000; /* Must be in range of 0 to 999999999 */
    }   
  else
    {   
      req.tv_sec = 0;                         /* Must be Non-Negative */
      req.tv_nsec = miliseconds * 1000000;    /* Must be in range of 0 to 999999999 */
    }   
  return nanosleep(&req , &rem);
}

int main(int argc, char **argv) {
  char ssd_cache [255] = "no";
  if (getenv("SSD_CACHE")) {
    strcpy(ssd_cache, getenv("SSD_CACHE"));
  }
  bool cache = false; 
  if (strcmp(ssd_cache, "yes")==0) {
    cache=true;
  }

  // Assuming that the dataset is a two dimensional array of 8x5 dimension;
Huihuo Zheng's avatar
Huihuo Zheng committed
52
53
  size_t d1 = 2048; 
  size_t d2 = 2048; 
Huihuo Zheng's avatar
Huihuo Zheng committed
54
  int niter = 10; 
Huihuo Zheng's avatar
test    
Huihuo Zheng committed
55
  char scratch[255] = "./";
Huihuo Zheng's avatar
Huihuo Zheng committed
56
  double sleep=0.0;
Huihuo Zheng's avatar
Huihuo Zheng committed
57
  bool collective=false;
Huihuo Zheng's avatar
Huihuo Zheng committed
58
59
60
61
62
63
64
65
66
67
68
69
70
71
  for(int i=1; i<argc; i++) {
    if (strcmp(argv[i], "--dim")==0) {
      d1 = int(atoi(argv[i+1])); 
      d2 = int(atoi(argv[i+2])); 
      i+=2; 
    } else if (strcmp(argv[i], "--niter")==0) {
      niter = int(atoi(argv[i+1])); 
      i+=1; 
    } else if (strcmp(argv[i], "--scratch")==0) {
      strcpy(scratch, argv[i+1]);
      i+=1;
    } else if (strcmp(argv[i],"--sleep")==0) {
      sleep = atof(argv[i+1]); 
      i+=1; 
Huihuo Zheng's avatar
Huihuo Zheng committed
72
73
    } else if (strcmp(argv[i], "--collective")==0) {
      collective = true;
Huihuo Zheng's avatar
Huihuo Zheng committed
74
75
76
77
78
79
80
81
82
83
84
    }
  }
  hsize_t ldims[2] = {d1, d2};

  hsize_t oned = d1*d2;
  MPI_Comm comm = MPI_COMM_WORLD;
  MPI_Info info = MPI_INFO_NULL;
  int rank, nproc, provided; 
  MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
  MPI_Comm_size(comm, &nproc);
  MPI_Comm_rank(comm, &rank);
Huihuo Zheng's avatar
SSD    
Huihuo Zheng committed
85
  if (rank==0) cout << "MPI_Init_thread provided: " << provided << endl;
Huihuo Zheng's avatar
Huihuo Zheng committed
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
  Timing tt(rank==io_node());
  tt.start_clock("total");
  //printf("     MPI: I am rank %d of %d \n", rank, nproc);
  // find local array dimension and offset; 
  hsize_t gdims[2] = {d1*nproc, d2};
  if (rank==0) {
    printf("=============================================\n");
    printf(" Buf dim: %llu x %llu\n",  ldims[0], ldims[1]);
    printf("Buf size: %f MB\n", float(d1*d2)/1024/1024*sizeof(int));
    printf(" Scratch: %s\n", scratch); 
    printf("   nproc: %d\n", nproc);
    printf("=============================================\n");
    if (cache) printf("** using SSD as a cache **\n"); 
  }
  hsize_t offset[2] = {0, 0};
  // setup file access property list for mpio
  hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
  H5Pset_fapl_mpio(plist_id, comm, info);

  char f[255];
  strcpy(f, scratch);
Huihuo Zheng's avatar
Huihuo Zheng committed
107
  strcat(f, "./parallel_file.h5");
Huihuo Zheng's avatar
Huihuo Zheng committed
108
109
110
111
112
113
114
  tt.start_clock("H5Fcreate");   
  hid_t file_id = H5Fcreate(f, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
  tt.stop_clock("H5Fcreate"); 
  // create memory space
  hid_t memspace = H5Screate_simple(2, ldims, NULL);

  // define local data
Huihuo Zheng's avatar
Huihuo Zheng committed
115
  int* data = new int[ldims[0]*ldims[1]*2];
Huihuo Zheng's avatar
Huihuo Zheng committed
116
117
  // set up dataset access property list 
  hid_t dxf_id = H5Pcreate(H5P_DATASET_XFER);
Huihuo Zheng's avatar
Huihuo Zheng committed
118
119
120
121
  if (collective)
    H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_COLLECTIVE);
  else
    H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_INDEPENDENT);
Huihuo Zheng's avatar
Huihuo Zheng committed
122
123
124
125
126
127
128
129
130
131
132
  
  hsize_t ggdims[2] = {gdims[0]*niter, gdims[1]};
  hid_t filespace = H5Screate_simple(2, ggdims, NULL);

  hid_t dt = H5Tcopy(H5T_NATIVE_INT);
  tt.start_clock("H5Dcreate");
  hid_t dset_id = H5Dcreate(file_id, "dset", dt, filespace, H5P_DEFAULT,
			    H5P_DEFAULT, H5P_DEFAULT);
  tt.stop_clock("H5Dcreate"); 
  hsize_t size = get_buf_size(memspace, dt);
  if (rank==0) 
Huihuo Zheng's avatar
Huihuo Zheng committed
133
    printf(" Total mspace size: %5.5f MB | sizeof (element) %lu\n", float(size)/1024/1024, H5Tget_size(H5T_NATIVE_INT));
Huihuo Zheng's avatar
Huihuo Zheng committed
134
135
  size = get_buf_size(filespace, dt);
  if (rank==0) 
Huihuo Zheng's avatar
Huihuo Zheng committed
136
    printf(" Total fspace size: %5.5f MB \n", float(size)/1024/1024);
Huihuo Zheng's avatar
Huihuo Zheng committed
137
138
139
140
141
142
  
  hsize_t count[2] = {1, 1};


  for (int i=0; i<niter; i++) {
    tt.start_clock("Init_array");     
143
    for(int j=0; j<ldims[0]*ldims[1]; j++)
Huihuo Zheng's avatar
Huihuo Zheng committed
144
      data[j] = i;
145
146
    for(int j=ldims[0]*ldims[1]; j<2*ldims[0]*ldims[1]; j++)
      data[j] = i+niter;
Huihuo Zheng's avatar
Huihuo Zheng committed
147
148
149
    tt.stop_clock("Init_array");
    hsize_t loc_buf_dim[2] = {d1, d2};
    hsize_t data_dim[2] = {2*d1, d2};
Huihuo Zheng's avatar
Huihuo Zheng committed
150
    // select hyperslab
Huihuo Zheng's avatar
Huihuo Zheng committed
151
    tt.start_clock("Select");
Huihuo Zheng's avatar
kp    
Huihuo Zheng committed
152
    hid_t filespace = H5Screate_simple(2, ggdims, NULL);
Huihuo Zheng's avatar
Huihuo Zheng committed
153
154
155
156
    hsize_t loc_buf_select[2] = {d1/2, d2};
    hid_t memspace = H5Screate_simple(2, data_dim, NULL);
    offset[0]= i*gdims[0] + rank*ldims[0];
    H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, loc_buf_dim, count);
Huihuo Zheng's avatar
Huihuo Zheng committed
157
    offset[0]=0;
Huihuo Zheng's avatar
Huihuo Zheng committed
158
159
160
    H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset, NULL, loc_buf_select, count);
    offset[0]=d1;
    H5Sselect_hyperslab(memspace, H5S_SELECT_OR, offset, NULL, loc_buf_select, count);
Huihuo Zheng's avatar
Huihuo Zheng committed
161
    tt.stop_clock("Select");
Huihuo Zheng's avatar
Huihuo Zheng committed
162
    if (rank==0 and i==0) printf(" Selected buffer size (Bytes): %llu (memspace) - %llu (filespace) \n", get_buf_size(memspace, H5T_NATIVE_INT), get_buf_size(filespace, H5T_NATIVE_INT));
Huihuo Zheng's avatar
Huihuo Zheng committed
163
164

    MPI_Barrier(MPI_COMM_WORLD);
Huihuo Zheng's avatar
Huihuo Zheng committed
165
166
167
    tt.start_clock("H5Dwrite");
    hid_t status = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, dxf_id, data); // write memory to file
    tt.stop_clock("H5Dwrite");
Huihuo Zheng's avatar
Huihuo Zheng committed
168
    if (rank==0) printf("  *Iter: %d -   write rate: %f MiB/s\n", i, get_buf_size(memspace, H5T_NATIVE_INT)/tt["H5Dwrite"].t_iter[i]*nproc/1024/1024);
Huihuo Zheng's avatar
Huihuo Zheng committed
169
170
171
172
    tt.start_clock("compute");
    msleep(int(sleep*1000));
    tt.stop_clock("compute");
  }
Huihuo Zheng's avatar
Huihuo Zheng committed
173
  tt.start_clock("H5Dclose");
Huihuo Zheng's avatar
Huihuo Zheng committed
174
  H5Dclose(dset_id);
Huihuo Zheng's avatar
Huihuo Zheng committed
175
176
  tt.stop_clock("H5Dclose");
  tt.start_clock("H5Fclose");
Huihuo Zheng's avatar
Huihuo Zheng committed
177
  H5Fclose(file_id);
Huihuo Zheng's avatar
Huihuo Zheng committed
178
  tt.stop_clock("H5Fclose");
Huihuo Zheng's avatar
Huihuo Zheng committed
179
180
  H5Pclose(dxf_id);
  H5Pclose(plist_id);
Huihuo Zheng's avatar
SSD    
Huihuo Zheng committed
181
  H5Sclose(memspace);
Huihuo Zheng's avatar
Huihuo Zheng committed
182
183
184
185
186
187
  bool master = (rank==0); 
  delete [] data;
  Timer T = tt["H5Dwrite"]; 
  double avg = 0.0; 
  double std = 0.0; 
  stat(&T.t_iter[0], niter, avg, std, 'i');
Huihuo Zheng's avatar
Huihuo Zheng committed
188
  if (rank==0) printf("Overall write rate: %f +/- %f MB/s\n", size*avg/niter/1024/1024, size/niter*std/1024/1024);
Huihuo Zheng's avatar
Huihuo Zheng committed
189
190
191
192
  tt.stop_clock("total");
  MPI_Finalize();
  return 0;
}