Commit 48f7d6dc authored by Francois Tessier's avatar Francois Tessier

Makefile for BG/Q. Add input parameter for output directory. Fix 32/64 bits datatype.

parent 8cf6a6ff
......@@ -6,4 +6,8 @@
miniHACC-AoS
miniHACC-SoA
miniHACC-AoS-MPIIO
miniHACC-SoA-MPIIO
\ No newline at end of file
miniHACC-SoA-MPIIO
examples/S3D-IO/run/*.out
examples/S3D-IO/run/writeperf.log
examples/S3D-IO/data/*
examples/S3D-IO/run/s3d_f90.x
\ No newline at end of file
......@@ -2,8 +2,8 @@
# SAVEFILEINSEPDIR: do not know
directives = -DCPQ -DSAVEFILEINSEPDIR -DMPIIO
COMPILE_F90 = mpif90 $(directives) -xf95-cpp-input -c -o $@
COMPILE_F77 = mpif90 -xf77-cpp-input -c -o $@
COMPILE_C = mpifcc -c -o $@
COMPILE_F90 = mpif90 -g $(directives) -xf95-cpp-input -c -o $@
COMPILE_F77 = mpif90 -g -xf77-cpp-input -c -o $@
COMPILE_C = mpifcc -g -c -o $@
LINK = mpif90
==========================================================================================
MODE
==========================================================================================
0 - 0 = run DNS code, 1 = post-process DNS results (mode)
==========================================================================================
GRID DIMENSION PARAMETERS
==========================================================================================
384 - global number of grid points in the x-direction (nx_g)
384 - global number of grid points in the y-direction (ny_g)
384 - global number of grid points in the z-direction (nz_g)
8 - number of processors in x-direction (npx)
8 - number of processors in y-direction (npy)
8 - number of processors in z-direction (npz)
==========================================================================================
RUN-TIME PARAMETERS
==========================================================================================
0 - 0 for write output to screen, 1 for write to file (i_write)
0 - 0 for new run, 1 for restart (i_restart)
10 - ending time step (i_time_end)
1 - frequency to save fields in restart files (i_time_save)
1.0e+5 - time period to save fields in restart files (time_save_inc)
==========================================================================================
GEOMETRY PARAMETERS
==========================================================================================
pressure_wave_test - title of run, sets initialiation of flow field (run_title)
1 - 0 for no x-direction dependence, 1 for so (vary_in_x)
1 - 0 for no y-direction dependence, 1 for so (vary_in_y)
1 - 0 for no z-direction dependence, 1 for so (vary_in_z)
0 - 0 for non-periodic in x-direction, 1 for periodic (periodic_x)
0 - 0 for non-periodic in y-diretcion, 1 for periodic (periodic_y)
0 - 0 for non-periodic in z-dirtecion, 1 for periodic (periodic_z)
0 - 0 for no turbulence, 1 for isotropic turbulence (i_turbulence)
1 - BC at x=0 boundary; 1 nonreflecting, 0 hard inflow (nrf_x0)
1 - BC at x=L boundary; 1 nonreflecting, 0 hard inflow (nrf_xl)
1 - BC at y=0 boundary; 1 nonreflecting, 0 hard inflow (nrf_y0)
1 - BC at y=L boundary; 1 nonreflecting, 0 hard inflow (nrf_yl)
1 - BC at z=0 boundary; 1 nonreflecting, 0 hard inflow (nrf_z0)
1 - BC at z=L boundary; 1 nonreflecting, 0 hard inflow (nrf_zl)
0.2 - fix factor for pressure drift (relax_ct)
==========================================================================================
PHYSICAL PARAMETERS
==========================================================================================
0.0 - minimum value of grid in x-direction in cm (xmin)
0.0 - minimum value of grid in y-direction in cm (ymin)
0.0 - minimum value of grid in z-direction in cm (zmin)
0.1 - maximum value of grid in x-direction in cm (xmax)
0.1 - maximum value of grid in y-direction in cm (ymax)
0.1 - minimum value of grid in z-direction in cm (zmax)
0.001 - Mach number where re_real/mach_no = re_acoustic (mach_no)
100.0 - real convective Reynolds number (re_real)
0.708 - Prandtl number (pr)
==========================================================================================
NUMERICS PARAMETERS
==========================================================================================
1 - 0 for no reaction, 1 for reaction (i_react)
8 - order of spatial derivatives: 6th or 8th only (iorder)
10 - frequency to monitor min/max and active (i_time_mon)
-1 - frequency to check spatial resolution (i_time_res)
-1 - frequency to write tecplot file (i_time_tec)
10 - order of spatial filter (i_filter)
10 - frequency to filter solution vector (i_time_fil)
==========================================================================================
REQUIRED REFERENCE VALUES
==========================================================================================
1.4 - reference ratio of specific heats (g_ref)
347.2 - reference speed of sound (m/s) (a_ref)
300.0 - freestream temperature (K) (to)
1.1766 - reference density (kg/m^3) (rho_ref)
26.14e-3 - reference thermal conductivity (W/m-s) (lambda_ref)
==========================================================================================
flag to enable/disable MPI I/O
===============================================================================
0 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
......@@ -5,5 +5,6 @@ time period to save fields in restart files (time_save_inc) 1.00000E+05
frequency to monitor min/max files and active (i_time_mon) 10
frequency to check resolution; set < 0 for no check (i_time_res) -1
frequency write current tecplot file (i_time_tec) -1
frequency to filter solution vector (i_time_fil) 0
initial/constant timestep (sec) (tstep_init) 0.00000E+00
timestep frequency to write controller info (i_time_cont) 0
==========================================================================================
MODE
==========================================================================================
1 - 0 = run DNS code, 1 = post-process DNS results (mode)
0 - 0 = run DNS code, 1 = post-process DNS results (mode)
==========================================================================================
GRID DIMENSION PARAMETERS
==========================================================================================
200 - global number of grid points in the x-direction (nx_g)
200 - global number of grid points in the y-direction (ny_g)
100 - global number of grid points in the z-direction (nz_g)
4 - number of processors in x-direction (npx)
4 - number of processors in y-direction (npy)
2 - number of processors in z-direction (npz)
256 - global number of grid points in the x-direction (nx_g)
256 - global number of grid points in the y-direction (ny_g)
256 - global number of grid points in the z-direction (nz_g)
8 - number of processors in x-direction (npx)
8 - number of processors in y-direction (npy)
8 - number of processors in z-direction (npz)
==========================================================================================
RUN-TIME PARAMETERS
==========================================================================================
......@@ -69,6 +69,9 @@ REQUIRED REFERENCE VALUES
26.14e-3 - reference thermal conductivity (W/m-s) (lambda_ref)
==========================================================================================
flag to enable/disable MPI I/O
===============================================================================
0 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
==========================================================================================
1 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
==========================================================================================
Output directory
==========================================================================================
/projects/visualization/ftessier/S3D
......@@ -5,9 +5,9 @@ MODE
==========================================================================================
GRID DIMENSION PARAMETERS
==========================================================================================
384 - global number of grid points in the x-direction (nx_g)
384 - global number of grid points in the y-direction (ny_g)
384 - global number of grid points in the z-direction (nz_g)
256 - global number of grid points in the x-direction (nx_g)
256 - global number of grid points in the y-direction (ny_g)
256 - global number of grid points in the z-direction (nz_g)
8 - number of processors in x-direction (npx)
8 - number of processors in y-direction (npy)
8 - number of processors in z-direction (npz)
......@@ -69,6 +69,9 @@ REQUIRED REFERENCE VALUES
26.14e-3 - reference thermal conductivity (W/m-s) (lambda_ref)
==========================================================================================
flag to enable/disable MPI I/O
===============================================================================
0 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
==========================================================================================
1 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
==========================================================================================
Output directory
==========================================================================================
/projects/visualization/ftessier/S3D
......@@ -5,9 +5,9 @@ MODE
==========================================================================================
GRID DIMENSION PARAMETERS
==========================================================================================
192 - global number of grid points in the x-direction (nx_g)
192 - global number of grid points in the y-direction (ny_g)
192 - global number of grid points in the z-direction (nz_g)
128 - global number of grid points in the x-direction (nx_g)
128 - global number of grid points in the y-direction (ny_g)
128 - global number of grid points in the z-direction (nz_g)
4 - number of processors in x-direction (npx)
4 - number of processors in y-direction (npy)
4 - number of processors in z-direction (npz)
......@@ -69,6 +69,9 @@ REQUIRED REFERENCE VALUES
26.14e-3 - reference thermal conductivity (W/m-s) (lambda_ref)
==========================================================================================
flag to enable/disable MPI I/O
===============================================================================
0 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
==========================================================================================
1 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
==========================================================================================
Output directory
==========================================================================================
/projects/visualization/ftessier/S3D
VARS="PAMID_VERBOSE=1 BG_SHAREDMEMSIZE=64 PAMID_COLLECTIVES=1 BGLOCKLESSMPIO_F_TYPE=0x47504653 PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
NODES=512
NODES=32
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug"
TARGET="/projects/visualization/ftessier/S3D"
cd $HOME/TAPIOCA/examples/S3D-IO
cd $HOME/TAPIOCA/examples/S3D-IO/run/
rm $TARGET/*
# Run and write out files with 8000pts/node
cp -f ../input/s3d.in.8000_$PBS_NNODES ../input/s3d.in
aprun -n $PBS_NNODES ./s3d.x > $PBS_NNODES.s3d.out.$PBS_JOBID
cp -f ../input/s3d.in.110592_$NPROCS ../input/s3d.in
runjob --block $COBALT_PARTNAME --envs $VARS -p $PPN --np $NPROCS : ./s3d_f90.x > $NPROCS.s3d.out
# Run and re-read ouput files written previously; check for errors
cd $NEW_WORKDIR/run
cp -f ../input/s3d.in.8000_$PBS_NNODES.post ../input/s3d.in
cp -f ../input/intmorph.in.8000_$PBS_NNODES ../input/intmorph.in
aprun -n $PBS_NNODES ./s3d.x >> $PBS_NNODES.s3d.out.$PBS_JOBID
# # Run and re-read ouput files written previously; check for errors
# cd $NEW_WORKDIR/run
# cp -f ../input/s3d.in.8000_$PBS_NNODES.post ../input/s3d.in
# cp -f ../input/intmorph.in.8000_$PBS_NNODES ../input/intmorph.in
# aprun -n $PBS_NNODES ./s3d.x >> $PBS_NNODES.s3d.out.$PBS_JOBID
......@@ -122,8 +122,8 @@
enddo
max = maxval(gauss)
call MPI_Allreduce(max,max_g,1,MPI_REAL8,MPI_MAX,gcomm,ierr)
call MPI_Allreduce(max,max_g,1,MPI_REAL4,MPI_MAX,gcomm,ierr)
gauss = gauss / max_g
......
......@@ -25,6 +25,7 @@
use thermchem_m, only : i_react
use param_m, only : io_method
use param_m, only : output_dir
implicit none
......@@ -162,6 +163,12 @@
read(1,*)
read(1,*) io_method
read(1,*) ! 3 comment lines
read(1,*)
read(1,*)
read(1,'(A)') output_dir
!-----------------------------------------------------------------------------------------
......@@ -244,6 +251,7 @@
! broadcast io method control
call MPI_Bcast(io_method, 1, MPI_INTEGER, 0, gcomm, ierr)
call MPI_Bcast(output_dir, 100, MPI_CHAR, 0, gcomm, ierr)
!-----------------------------------------------------------------------------------------
! set time_save equal to time_save_inc initially
......
......@@ -9,6 +9,7 @@ subroutine write_savefile(io, precsn)
use param_m, only : nx, ny, nz, n_spec !quad
use param_m, only : io_method
use param_m, only : output_dir
#ifdef MPIIO
use mpi_io_m, only : mpi_file_io
use pnetcdf_m, only : pnetcdf_write
......@@ -49,20 +50,20 @@ subroutine write_savefile(io, precsn)
! wkliao: set a different file name for MPI-IO, pnetcdf, and HDF5 methods
! wkliao: no subdirectories are created if io_method > 0
if (io_method .EQ. 1) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.mpi'
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.mpi'
elseif (io_method .EQ. 2) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.nc'
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.nc'
elseif (io_method .EQ. 3) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.h5'
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.h5'
elseif (io_method .EQ. 4) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.bp'
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.bp'
elseif (io_method .EQ. 5) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.bp'
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.bp'
elseif (io_method .EQ. 6) then
filename = trim(run_title)//'.'//time_ext//'.field.bp'
else
dirname = '../data/'//trim(run_title)//'.'//time_ext//'/'
dirname = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'/'
if(myid == 0) then
#ifdef SYSTEMCALLWONTWORK
! when system calls do not work, as on infiniband
......
......@@ -60,6 +60,8 @@
integer io_method
character*100 output_dir !output directory
character*8 dat_1, dat_2 !for start and end date (wall clock)
character*10 tim_1, tim_2 !for start and end time (wall clock)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment