/* 
   This is the test program for studying the cache effect in POSIX I/O. 

   Normally, when we write data to the files. The system will keep a copy of the data in the 
   cache. So when we are reading the same data from the files, it actually read from the cache
   instead of from the file system. 
   
   In order to avoid the cache effect, one can write more than one files, so by the time
   reading the first file, the first file is already been pushed out of the cache. 

   The other way to avoid the cache effect is to separate out the read from the write. In the 
   benchmark, we do write first, and then do read in an independent run. The benchmarks script 
   allow us to do this. 

   Huihuo Zheng
   - huihuo.zheng@anl.gov
 */
#include <unistd.h>
#include <sys/stat.h> 
#include <fcntl.h>
#include <iostream>
#include "stdio.h"
#include "string.h"
#include "stdlib.h"
#include "profiling.h"
#include "timing.h"
#include "mpi.h"
using namespace std; 
int main(int argc, char **argv) {
  int dim = 1024576;
  int nbatch = 1024;
  int niter = 10; 
  double dat[dim];
  double vm, rss;
  char path[255] = "./";
  bool write = false; 
  for (int i=1; i<argc; i++) {
    if (strcmp(argv[i], "--dim")==0) {
      dim = int(atof(argv[i+1])); i=i+1;
    } else if (strcmp(argv[i], "--nbatch")==0) {
      nbatch = int(atof(argv[i+1])); i=i+1;
    } else if (strcmp(argv[i], "--niter")==0) {
      niter = int(atof(argv[i+1])); i=i+1;
    } else if (strcmp(argv[i], "--path")==0) {
      strcpy(path, argv[i+1]); i=i+1;
    } else if (strcmp(argv[i], "--write")==0) {
      write = true;
    }
  }
  MPI_Init(&argc, &argv);
  int rank, nproc; 
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  MPI_Comm_size(MPI_COMM_WORLD, &nproc);
  Timing tt(rank==0); 
  if (rank==0) {
    cout << "dim: " << dim << endl;
    cout << "nbatch: " << nbatch << endl;
    cout << "size of file [MiB]: " << dim*nbatch*sizeof(double)/1024/1024 << endl;
    cout << "niter: " << niter << endl; 
    cout << "path: " << path << endl; 
    cout << "nproc: " << nproc << endl; 
  }
  if (write) {
    for(int it =0; it < niter; it++) {
      dat[0] = it; 
      char fname[255];
      strcpy(fname, path);
      strcat(fname, "/cache.dat");
      strcat(fname, to_string(rank).c_str());
      strcat(fname, "-"); 
      strcat(fname, to_string(it).c_str());
      tt.start_clock("write");
      MPI_Barrier(MPI_COMM_WORLD);
      int fd = open(fname, O_RDWR | O_TRUNC | O_CREAT, S_IRUSR | S_IWUSR);
      for(int i=0; i<nbatch; i++) {
	pwrite(fd, dat, sizeof(double)*dim, sizeof(double)*i*dim);
      }
      MPI_Barrier(MPI_COMM_WORLD);
      tt.stop_clock("write");
      fsync(fd);
      if (rank==0) cout << "Write rate: " << sizeof(double)*dim*nbatch/tt["write"].t_iter[it]/1024/1024*nproc << endl; 
      close(fd);
    }
  }
  for(int it =0; it < niter; it++) {
    char fname[255];
    strcpy(fname, path);
    strcat(fname, "/cache.dat");
    strcat(fname, to_string(rank).c_str());
    strcat(fname, "-"); 
    strcat(fname, to_string(it).c_str());
    MPI_Barrier(MPI_COMM_WORLD);
    int fd = open(fname, O_RDONLY);
    tt.start_clock("read");    
    for(int i=0; i<nbatch; i++) {
      pread(fd, dat, sizeof(double)*dim, sizeof(double)*i*dim);
    }
    MPI_Barrier(MPI_COMM_WORLD);
    tt.stop_clock("read");
    if (rank==0) cout << "Read rate: " << sizeof(double)*dim*nbatch/tt["read"].t_iter[it]/1024/1024*nproc << endl; 
    close(fd);
    remove(fname);
  }
  MPI_Finalize();
  return 0;
}