Makefile for BG/Q. Add input parameter for output directory. Fix 32/64 bits datatype.

.gitignore

@@ -6,4 +6,8 @@
 miniHACC-AoS
 miniHACC-SoA
 miniHACC-AoS-MPIIO
-miniHACC-SoA-MPIIO
\ No newline at end of file
+miniHACC-SoA-MPIIO
+examples/S3D-IO/run/*.out
+examples/S3D-IO/run/writeperf.log
+examples/S3D-IO/data/*
+examples/S3D-IO/run/s3d_f90.x
\ No newline at end of file

examples/S3D-IO/build/make.BGQ

@@ -2,8 +2,8 @@
 # SAVEFILEINSEPDIR: do not know
 directives = -DCPQ -DSAVEFILEINSEPDIR -DMPIIO
 
-COMPILE_F90  = mpif90 $(directives) -xf95-cpp-input -c -o $@ 
-COMPILE_F77  = mpif90 -xf77-cpp-input -c -o $@
-COMPILE_C    = mpifcc -c -o $@
+COMPILE_F90  = mpif90 -g $(directives) -xf95-cpp-input -c -o $@ 
+COMPILE_F77  = mpif90 -g -xf77-cpp-input -c -o $@
+COMPILE_C    = mpifcc -g -c -o $@
 
 LINK         = mpif90

examples/S3D-IO/input/BGP/s3d.in.110592_512

+==========================================================================================
+MODE
+==========================================================================================
+0                   - 0 = run DNS code, 1 = post-process DNS results     (mode)
+==========================================================================================
+GRID DIMENSION PARAMETERS
+==========================================================================================
+384                 - global number of grid points in the x-direction    (nx_g)
+384                 - global number of grid points in the y-direction    (ny_g)
+384                  - global number of grid points in the z-direction    (nz_g)
+8                   - number of processors in x-direction                (npx)
+8                   - number of processors in y-direction                (npy)
+8                   - number of processors in z-direction                (npz)
+==========================================================================================
+RUN-TIME PARAMETERS
+==========================================================================================
+0                   - 0 for write output to screen, 1 for write to file  (i_write)
+0                   - 0 for new run, 1 for restart                       (i_restart)
+10                  - ending time step                                   (i_time_end)
+1                   - frequency to save fields in restart files          (i_time_save)
+1.0e+5              - time period to save fields in restart files        (time_save_inc)
+==========================================================================================
+GEOMETRY PARAMETERS
+==========================================================================================
+pressure_wave_test        - title of run, sets initialiation of flow field     (run_title)
+1                   - 0 for no x-direction dependence, 1 for so          (vary_in_x)
+1                   - 0 for no y-direction dependence, 1 for so          (vary_in_y)
+1                   - 0 for no z-direction dependence, 1 for so          (vary_in_z)
+0                   - 0 for non-periodic in x-direction, 1 for periodic  (periodic_x)
+0                   - 0 for non-periodic in y-diretcion, 1 for periodic  (periodic_y)
+0                   - 0 for non-periodic in z-dirtecion, 1 for periodic  (periodic_z)
+0                   - 0 for no turbulence, 1 for isotropic turbulence    (i_turbulence)
+1                   - BC at x=0 boundary; 1 nonreflecting, 0 hard inflow (nrf_x0)
+1                   - BC at x=L boundary; 1 nonreflecting, 0 hard inflow (nrf_xl)
+1                   - BC at y=0 boundary; 1 nonreflecting, 0 hard inflow (nrf_y0)
+1                   - BC at y=L boundary; 1 nonreflecting, 0 hard inflow (nrf_yl)
+1                   - BC at z=0 boundary; 1 nonreflecting, 0 hard inflow (nrf_z0)
+1                   - BC at z=L boundary; 1 nonreflecting, 0 hard inflow (nrf_zl)
+0.2                 - fix factor for pressure drift                      (relax_ct)
+==========================================================================================
+PHYSICAL PARAMETERS
+==========================================================================================
+0.0                 - minimum value of grid in x-direction in cm         (xmin)
+0.0                 - minimum value of grid in y-direction in cm         (ymin)
+0.0                 - minimum value of grid in z-direction in cm         (zmin)
+0.1                 - maximum value of grid in x-direction in cm         (xmax)
+0.1                 - maximum value of grid in y-direction in cm         (ymax)
+0.1                 - minimum value of grid in z-direction in cm         (zmax)
+0.001               - Mach number where re_real/mach_no = re_acoustic    (mach_no)
+100.0               - real convective Reynolds number                    (re_real)
+0.708               - Prandtl number                                     (pr)
+==========================================================================================
+NUMERICS PARAMETERS
+==========================================================================================
+1                   - 0 for no reaction, 1 for reaction                    (i_react)
+8                   - order of spatial derivatives: 6th or 8th only        (iorder)
+10                  - frequency to monitor min/max and active              (i_time_mon)
+-1                  - frequency to check spatial resolution                (i_time_res)
+-1                  - frequency to write tecplot file                      (i_time_tec)
+10                  - order of spatial filter                              (i_filter)
+10                  - frequency to filter solution vector                  (i_time_fil)
+==========================================================================================
+REQUIRED REFERENCE VALUES
+==========================================================================================
+1.4                 - reference ratio of specific heats                    (g_ref)
+347.2               - reference speed of sound (m/s)                       (a_ref)
+300.0               - freestream temperature (K)                           (to)
+1.1766              - reference density (kg/m^3)                           (rho_ref)
+26.14e-3            - reference thermal conductivity (W/m-s)               (lambda_ref)
+==========================================================================================
+flag to enable/disable MPI I/O
+===============================================================================
+0                   - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
+

examples/S3D-IO/input/active.in

@@ -5,5 +5,6 @@ time period to save fields in restart files       (time_save_inc)  1.00000E+05
 frequency to monitor min/max files and active        (i_time_mon)           10
 frequency to check resolution; set < 0 for no check  (i_time_res)           -1
 frequency write current tecplot file                 (i_time_tec)           -1
+frequency to filter solution vector                  (i_time_fil)            0
 initial/constant timestep (sec)                      (tstep_init)  0.00000E+00
 timestep frequency to write controller info         (i_time_cont)            0

examples/S3D-IO/input/s3d.in

 ==========================================================================================
 MODE
 ==========================================================================================
-1                   - 0 = run DNS code, 1 = post-process DNS results     (mode)
+0                   - 0 = run DNS code, 1 = post-process DNS results     (mode)
 ==========================================================================================
 GRID DIMENSION PARAMETERS
 ==========================================================================================
-200                 - global number of grid points in the x-direction    (nx_g)
-200                 - global number of grid points in the y-direction    (ny_g)
-100                  - global number of grid points in the z-direction    (nz_g)
-4                   - number of processors in x-direction                (npx)
-4                   - number of processors in y-direction                (npy)
-2                   - number of processors in z-direction                (npz)
+256                 - global number of grid points in the x-direction    (nx_g)
+256                 - global number of grid points in the y-direction    (ny_g)
+256                 - global number of grid points in the z-direction    (nz_g)
+8                   - number of processors in x-direction                (npx)
+8                   - number of processors in y-direction                (npy)
+8                   - number of processors in z-direction                (npz)
 ==========================================================================================
 RUN-TIME PARAMETERS
 ==========================================================================================
@@ -69,6 +69,9 @@ REQUIRED REFERENCE VALUES
 26.14e-3            - reference thermal conductivity (W/m-s)               (lambda_ref)
 ==========================================================================================
 flag to enable/disable MPI I/O
-===============================================================================
-0                   - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
-
+==========================================================================================
+1                   - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
+==========================================================================================
+Output directory
+==========================================================================================
+/projects/visualization/ftessier/S3D

examples/S3D-IO/input/s3d.in.110592_512

@@ -5,9 +5,9 @@ MODE
 ==========================================================================================
 GRID DIMENSION PARAMETERS
 ==========================================================================================
-384                 - global number of grid points in the x-direction    (nx_g)
-384                 - global number of grid points in the y-direction    (ny_g)
-384                  - global number of grid points in the z-direction    (nz_g)
+256                 - global number of grid points in the x-direction    (nx_g)
+256                 - global number of grid points in the y-direction    (ny_g)
+256                 - global number of grid points in the z-direction    (nz_g)
 8                   - number of processors in x-direction                (npx)
 8                   - number of processors in y-direction                (npy)
 8                   - number of processors in z-direction                (npz)
@@ -69,6 +69,9 @@ REQUIRED REFERENCE VALUES
 26.14e-3            - reference thermal conductivity (W/m-s)               (lambda_ref)
 ==========================================================================================
 flag to enable/disable MPI I/O
-===============================================================================
-0                   - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
-
+==========================================================================================
+1                   - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
+==========================================================================================
+Output directory
+==========================================================================================
+/projects/visualization/ftessier/S3D

examples/S3D-IO/input/s3d.in.110592_64

@@ -5,9 +5,9 @@ MODE
 ==========================================================================================
 GRID DIMENSION PARAMETERS
 ==========================================================================================
-192                 - global number of grid points in the x-direction    (nx_g)
-192                 - global number of grid points in the y-direction    (ny_g)
-192                  - global number of grid points in the z-direction    (nz_g)
+128                 - global number of grid points in the x-direction    (nx_g)
+128                 - global number of grid points in the y-direction    (ny_g)
+128                 - global number of grid points in the z-direction    (nz_g)
 4                   - number of processors in x-direction                (npx)
 4                   - number of processors in y-direction                (npy)
 4                   - number of processors in z-direction                (npz)
@@ -69,6 +69,9 @@ REQUIRED REFERENCE VALUES
 26.14e-3            - reference thermal conductivity (W/m-s)               (lambda_ref)
 ==========================================================================================
 flag to enable/disable MPI I/O
-===============================================================================
-0                   - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
-
+==========================================================================================
+1                   - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
+==========================================================================================
+Output directory
+==========================================================================================
+/projects/visualization/ftessier/S3D

examples/S3D-IO/run/run_bgq.sh

 VARS="PAMID_VERBOSE=1 BG_SHAREDMEMSIZE=64 PAMID_COLLECTIVES=1 BGLOCKLESSMPIO_F_TYPE=0x47504653 PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
-NODES=512
+NODES=32
 PPN=16
 NPROCS=$((NODES*PPN))
-TARGET="/projects/visualization/ftessier/debug"
+TARGET="/projects/visualization/ftessier/S3D"
 
-cd $HOME/TAPIOCA/examples/S3D-IO
+cd $HOME/TAPIOCA/examples/S3D-IO/run/
+rm $TARGET/*
 
 # Run and write out files with 8000pts/node
-cp -f ../input/s3d.in.8000_$PBS_NNODES ../input/s3d.in
-aprun -n $PBS_NNODES ./s3d.x  > $PBS_NNODES.s3d.out.$PBS_JOBID
+cp -f ../input/s3d.in.110592_$NPROCS ../input/s3d.in
+runjob --block $COBALT_PARTNAME --envs $VARS -p $PPN --np $NPROCS : ./s3d_f90.x > $NPROCS.s3d.out
 
-# Run and re-read ouput files written previously; check for errors
-cd  $NEW_WORKDIR/run
-cp -f ../input/s3d.in.8000_$PBS_NNODES.post ../input/s3d.in
-cp -f ../input/intmorph.in.8000_$PBS_NNODES ../input/intmorph.in
-aprun -n $PBS_NNODES ./s3d.x  >> $PBS_NNODES.s3d.out.$PBS_JOBID
+# # Run and re-read ouput files written previously; check for errors
+# cd  $NEW_WORKDIR/run
+# cp -f ../input/s3d.in.8000_$PBS_NNODES.post ../input/s3d.in
+# cp -f ../input/intmorph.in.8000_$PBS_NNODES ../input/intmorph.in
+# aprun -n $PBS_NNODES ./s3d.x  >> $PBS_NNODES.s3d.out.$PBS_JOBID

examples/S3D-IO/source/f90_files/init/init_pressure_wave_test.f90

@@ -122,8 +122,8 @@
   enddo
 
   max = maxval(gauss)
-
-  call MPI_Allreduce(max,max_g,1,MPI_REAL8,MPI_MAX,gcomm,ierr)
+  
+  call MPI_Allreduce(max,max_g,1,MPI_REAL4,MPI_MAX,gcomm,ierr)
 
   gauss = gauss / max_g
 

examples/S3D-IO/source/f90_files/solve/read_input.f90

@@ -25,6 +25,7 @@
   use thermchem_m, only : i_react
 
   use param_m, only : io_method
+  use param_m, only : output_dir
 
 
   implicit none
@@ -162,6 +163,12 @@
   read(1,*)
 
   read(1,*) io_method
+  
+  read(1,*)                 ! 3 comment lines
+  read(1,*)
+  read(1,*)
+
+  read(1,'(A)') output_dir
 
 !-----------------------------------------------------------------------------------------
 
@@ -244,6 +251,7 @@
 ! broadcast io method control
 
   call MPI_Bcast(io_method,   1, MPI_INTEGER, 0, gcomm, ierr)
+  call MPI_Bcast(output_dir,  100, MPI_CHAR, 0, gcomm, ierr)
 !-----------------------------------------------------------------------------------------
 
 ! set time_save equal to time_save_inc initially

examples/S3D-IO/source/f90_files/solve/write_savefile.f90

@@ -9,6 +9,7 @@ subroutine write_savefile(io, precsn)
   use param_m, only : nx, ny, nz, n_spec  !quad
 
   use param_m, only : io_method
+  use param_m, only : output_dir
 #ifdef MPIIO
   use mpi_io_m,  only :  mpi_file_io
   use pnetcdf_m, only : pnetcdf_write
@@ -49,20 +50,20 @@ subroutine write_savefile(io, precsn)
   ! wkliao: set a different file name for MPI-IO, pnetcdf, and HDF5 methods
   ! wkliao: no subdirectories are created if io_method > 0
   if (io_method .EQ. 1) then
-     filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.mpi'
+     filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.mpi'
   elseif (io_method .EQ. 2) then
-     filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.nc'
+     filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.nc'
   elseif (io_method .EQ. 3) then
-     filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.h5'
+     filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.h5'
   elseif (io_method .EQ. 4) then
-     filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.bp'
+     filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.bp'
   elseif (io_method .EQ. 5) then
-     filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.bp'
+     filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.bp'
   elseif (io_method .EQ. 6) then
      filename = trim(run_title)//'.'//time_ext//'.field.bp'
   else
 
-     dirname = '../data/'//trim(run_title)//'.'//time_ext//'/'
+     dirname = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'/'
      if(myid == 0) then
 #ifdef SYSTEMCALLWONTWORK
         ! when system calls do not work, as on infiniband

examples/S3D-IO/source/modules/param_m.f90

@@ -60,6 +60,8 @@
 
   integer io_method
 
+  character*100 output_dir      !output directory
+
   character*8  dat_1, dat_2     !for start and end date (wall clock)
   character*10 tim_1, tim_2     !for start and end time (wall clock)