Merge branch 'yslan' into 'master'

add peb146(E16k) NekRS example from Summit

See merge request !1

146_pebble_yuhsiang_E16k/README.md

+146 non-touching pebbles in a cylinder running with NekRS on Summit-GPUs
+
+
+- To configure the NekRS case, please modify the path in set_path and do
+  ```
+    module load gcc cmake cuda
+    . ./config_path
+  ```
+
+- To run the case, 
+  ```
+  ./submit.sh peb146_v5b 1 6 "00:20"
+  ```
+
+- example logfile: nekRS.859194, run with NekRS from here
+  https://xgitlab.cels.anl.gov/neams-th-coe/nekrs/tree/cardinal
+  
+- case file
+  Note that the mesh is stored in w2.fld

146_pebble_yuhsiang_E16k/config_path

+export NEKRS_HOME=$HOME/.local/nekrs_cardinal
+PATH=${NEKRS_HOME}/bin:${PATH}
+export OCCA_CACHE_DIR=/gpfs/alpine/proj-shared/csc262/ylan/cache_cardinal
+
+which nekrs
+echo $OCCA_CACHE_DIR

146_pebble_yuhsiang_E16k/peb146_v5b.oudf

+// Boundary conditions
+/* wall 1, inflow 2, outflow 3, x-slip 4, y-slip 5, z-slip 6 */
+void insVelocityDirichletConditions3D(bcData *bc)
+{                                                                        
+  bc->uP = 0.0;
+  bc->vP = 0.0;
+  bc->wP = 17.0;
+}
+
+void insVelocityNeumannConditions3D(bcData *bc)
+{
+}
+
+// Stabilized outflow (Dong et al)
+void insPressureDirichletConditions3D(bcData *bc)
+{
+  const dfloat iU0delta = 10.0;
+  const dfloat un = bc->uM*bc->nx + bc->vM*bc->ny + bc->wM*bc->nz;
+  const dfloat s0 = 0.5 * (1.0 - tanh(un*iU0delta)); 
+  bc->pP = -0.5 * (bc->uM*bc->uM + bc->vM*bc->vM + bc->wM*bc->wM) * s0;
+}

146_pebble_yuhsiang_E16k/peb146_v5b.par

+[OCCA]
+backend = CUDA
+deviceNumber = 0 #LOCAL-RANK
+
+[GENERAL]
+polynomialOrder = 7
+startFrom = w2.fld
+stopAt = endTime
+endTime = 200
+
+dealiasing = yes
+
+dt = 2.0e-4
+timeStepper = tombo2
+extrapolation = subCycling
+subCyclingSteps = 2
+
+writeControl = TIMESTEP
+writeInterval = 1000
+
+filtering = hpfrt
+filterWeight = 250
+filterCutoffRatio = 0.9 
+
+[PRESSURE]
+preconditioner = semg_amg
+amgSolver = parAlmond
+galerkincoarseoperator = yes
+residualTol = 1e-04
+residualProj = yes
+
+
+[VELOCITY]
+boundaryTypeMap = inlet, outlet, wall
+density = 1.0
+viscosity = 0.25 #0.000405
+residualTol = 1e-06
+
+#[TEMPERATURE]
+#solver=none
+#boundaryTypeMap = t, O, I, t
+#rhoCp = 10
+#conductivity = 5000
+#residualTol = 1e-08

146_pebble_yuhsiang_E16k/peb146_v5b.udf

+//
+// nekRS User Defined File
+//
+
+#include <math.h>
+#include "udf.hpp"
+
+/* UDF Functions */                                                      
+
+void UDF_LoadKernels(ins_t *ins)
+{
+}
+
+void UDF_Setup(ins_t *ins)
+{
+  // get IC from nek
+  if (!ins->readRestartFile) nek_copyTo(ins, ins->startTime);
+}
+
+void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep)
+{
+  if (ins->isOutputStep) nek_userchk();
+}

146_pebble_yuhsiang_E16k/peb146_v5b.usr

+c-----------------------------------------------------------------------
+      subroutine uservp(i,j,k,eg) ! set variable properties
+      include 'SIZE'
+      include 'TOTAL'
+      include 'NEKUSE'
+
+      integer i,j,k,e,eg
+     
+c     e = gllel(eg)
+
+      udiff  = 0.0
+      utrans = 0.0
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine userf(i,j,k,eg) ! set acceleration term
+c
+c     Note: this is an acceleration term, NOT a force!
+c     Thus, ffx will subsequently be multiplied by rho(x,t).
+c
+      include 'SIZE'
+      include 'TOTAL'
+      include 'NEKUSE'
+
+      integer i,j,k,e,eg
+
+c     e = gllel(eg)
+
+      ffx = 0.0
+      ffy = 0.0
+      ffz = 0.0
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine userq(i,j,k,eg) ! set source term
+      include 'SIZE'
+      include 'TOTAL'
+      include 'NEKUSE'
+
+      integer i,j,k,e,eg
+
+c     e = gllel(eg)
+
+      qvol   = 0.0
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine userbc(i,j,k,f,eg) ! set up boundary conditions
+c     NOTE ::: This routine MAY NOT be called by every process
+      include 'SIZE'
+      include 'TOTAL'
+      include 'NEKUSE'
+
+      integer i,j,k,f,e,eg
+      e=gllel(eg)
+
+      ux   = 0.0
+      uy   = 0.0
+      uz   = 17.0
+
+      flux = 0.0
+      temp = 0.0
+      if (cbc(f,e,1).eq.'W  '.AND.cbc(f,e,2).eq.'t  ') temp=1.0
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine useric(i,j,k,eg) ! set up initial conditions
+      include 'SIZE'
+      include 'TOTAL'
+      include 'NEKUSE'
+      integer i,j,k,e,eg
+
+      ux   = 0.0
+      uy   = 0.0
+      uz   = 17.0
+      temp = 0.0
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine userchk
+      include 'SIZE'
+      include 'TOTAL'
+
+c      common /myoutflow/ d(lx1,ly1,lz1,lelt),m1(lx1*ly1*lz1,lelt)
+c      real m1
+c      rq  = 2.
+c      uin = 0.
+c      call turb_outflow(d,m1,rq,uin)
+
+      n=lx1*ly1*lz1*nelt
+
+      if (istep.eq.0) then
+c        time=0.0
+c
+c        Rfld = 0.9   ! radii of peb from restart file
+c        Rtar = 1.5   ! target radii matching Re/density
+c        scale = 1.5/0.9
+c        do i=1,n
+c          xm1(i,1,1,1)=xm1(i,1,1,1)*scale
+c          ym1(i,1,1,1)=ym1(i,1,1,1)*scale
+c          zm1(i,1,1,1)=zm1(i,1,1,1)*scale
+c        enddo
+c        call fix_geom
+c        call geom_reset(1)
+
+        xxmax = glmax(xm1,n)
+        yymax = glmax(ym1,n)
+        zzmax = glmax(zm1,n)
+        xxmin = glmin(xm1,n)
+        yymin = glmin(ym1,n)
+        zzmin = glmin(zm1,n)
+        if (nio.eq.0) then
+          write(6,18) xxmin,yymin,zzmin
+          write(6,19) xxmax,yymax,zzmax
+   18     format(' xyz min  ',5g13.5)
+   19     format(' xyz max  ',5g13.5)
+        endif
+
+        ifxyo=.true.
+c        call outpost(vx,vy,vz,t,pr,'   ')
+      endif
+
+      if (mod(istep,500).eq.0) then
+         umin=glmin(vx,n)
+         umax=glmax(vx,n)
+         vmin=glmin(vy,n)
+         vmax=glmax(vy,n)
+         wmin=glmin(vz,n)
+         wmax=glmax(vz,n)
+         tmin=glmin(t ,n)
+         tmax=glmax(t ,n)
+         if (nio.eq.0) write(6,1) 
+     $    istep,time,umin,umax,vmin,vmax,wmin,wmax,tmin,tmax
+    1    format(i9,1p9e11.3,' tmax')
+      endif
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine userqtl ! Set thermal divergence
+
+      call userqtl_scig
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine usrdat   ! This routine to modify element vertices
+      include 'SIZE'
+      include 'TOTAL'
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine usrdat3
+      include 'SIZE'
+      include 'TOTAL'
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine usrdat2
+      include 'SIZE'
+      include 'TOTAL'
+      integer e,f
+
+c      ifheat = .true.
+      nface = 2*ldim
+
+      do e=1,nelv
+      do f=1,nface
+        if(cbc(f,e,1).eq.'E  '.or.cbc(f,e,1).eq.'   ')then
+          cbc(f,e,1)='E  '
+          cbc(f,e,2)='E  '
+        elseif(cbc(f,e,1).eq.'v  ')then  ! z_bottom
+          cbc(f,e,2)='t  '
+        elseif(cbc(f,e,1).eq.'O  ')then  ! z_top
+          cbc(f,e,2)='O  '
+        elseif(cbc(f,e,1).eq.'SYM')then  ! side of cylinder
+          cbc(f,e,1)='W  '
+          cbc(f,e,2)='I  '
+        elseif(cbc(f,e,1).eq.'W  ')then  ! pebbles
+          cbc(f,e,2)='t  '
+        else
+          write(*,*)'bc incorrect',e,f,cbc(f,e,1)
+        endif
+      enddo
+      enddo
+
+      do iel=1,nelt
+      do ifc=1,2*ndim
+         if (cbc(ifc,iel,1) .eq. 'v  ') boundaryID(ifc,iel) = 1
+         if (cbc(ifc,iel,1) .eq. 'O  ') boundaryID(ifc,iel) = 2
+         if (cbc(ifc,iel,1) .eq. 'W  ') boundaryID(ifc,iel) = 3
+      enddo
+      enddo
+
+
+      return
+      end
+c-----------------------------------------------------------------------

146_pebble_yuhsiang_E16k/submit.sh

+#!/bin/bash
+
+bin=nekrs
+project_id=nfi114
+time="00:30"
+
+#source $NEKRS_INSTALL_DIR/bin/nekrs.bashrc
+export OCCA_VERBOSE
+
+case=$1
+
+#if [ $# -ne 3 ]; then
+#  echo "usage: $0 <setup file> <number of nodes> <GPUs per node> <hh:mm>"
+#  echo "usage: $0 <setup file> <number of nodes> <GPUs per node>"
+#  exit 0
+#fi
+if [ $# -eq 4 ]; then
+  time=$4
+fi
+
+if [ ! -f $case.par ]; then
+  echo "Cannot find" $case.par
+  exit 1
+fi
+
+mkdir -p $OCCA_CACHE_DIR 2>/dev/null
+
+nodes=$2
+gpu_per_node=$3
+let nn=$nodes*$gpu_per_node
+
+cmd="bsub -nnodes $nodes -W $time -P $project_id -J nekRS jsrun -n$nn -r$gpu_per_node -a1 -c1 -g1 -b packed:1 -d packed $bin --setup $case"
+echo $cmd
+echo $cmd > qq
+$cmd