Commit 8fae897a by ylan

parent d63a569b
 ... ... @@ -9,9 +9,9 @@ Pebble bed, outer geometries: - ann = annulus - box = box |cases| #peb | E | E/peb | p\_ratio1 | p\_ratio2 | |:---|---:|---:|---:|---:|---:| | cyl146 | 146 | 62,132 | 425.56 | 91.95 | |cases| #peb | E | E/peb | p\_ratio1 | p\_ratio2 | note | |:---|---:|---:|---:|---:|---:|---:| | cyl146 | 146 | 62,132 | 425.56 | 72.83 | | Rs=1.5 | | box1053 | 1503 | 376,828 | 250.72 | 34.90 | | cyl1568 | 1568 | 524,386 | 334.43 | 55.32 | | cyl3260 | 3260 | 1,121,214 | 343.93 | 54.85 | ... ... @@ -19,6 +19,7 @@ Pebble bed, outer geometries: | cyl11k | 11145 | 3,575,076 | 320.78 | 53.41 | | cyl44k | 44257 | 13,032,440 | 294.47 | 51.32 | | cyl49k | 49967 | 14,864,766 | 297.49 | 51.28 | | cyl49k | 127602 | 39,240,190 | 307.52 | 51.39 | **p_ratio1** is an upper estimate of the prosperity ratio computed by `(1 - Nsph*4/3*pi*r^3 / vol_geom)` where `vol_geom` is comptued by the `grep xyz` with zlen = (zmax-zmin-8) for 8=extruded length. The more pebbles, the more accurate this estimation is. **p_ratio2** (WIP) is computed directly inside user file. ... ...
 ## 127602 (non-touching) pebbles in an annulus cylinder (Last updated: Jan. 27, 2021) ann127602 N, Version 3.2, E39,240,190 - New features (after Aug, 2020): - fixed edge insert - mesh smoother+optimizer in Nek5000 - tol adjustment in edge collapse - Notes - DEM data from Yiqi (StarCCM++?) - smaller edge collapse tol depends on geom + few local locations - Geometries, pebble locations are stored at `ann127k.dat` - Geometry and BC in re2 file: - linear mesh, no curved sides - Sphere distribution: free-free (selected from some range in Z, free = non-flat) - Sphere radius = 1.0 (Wall) - Cylinder (inner) radius = 24.256 (W01, convert to Wall in production run) - Cylinder (outer) radius = 87.895 (W02, convert to Wall in production run) - Zbot of the cylinder = -27.010 (W03, convert to Inflow in production run) - Ztop of the cylinder = -29.031 (W04, convert to Outflow in production run) ``` xyz min -87.895 -87.895 -27.010 xyz max 87.895 87.895 29.031 mesh metrics: (linear mesh) GLL grid spacing min/max : 6.77E-04 3.52E-01 scaled Jacobian min/max/avg: 1.70E-02 9.99E-01 3.89E-01 aspect ratio min/max/avg: 1.05E+00 9.23E+01 1.58E+01 ``` - Mesh file (`mshmesh_retry2.fld+X`) - lx1=3 GLL points projected onto curved sides - no issues for interpolation onto lx1=8 and mid-multigrid levels at lx1=4 - no issues for lx1=2 corase grid - Production run - Use re2 for boundary conditions, but use restart file for the curved mesh - Time is NOT set to 0 when reading the mesh file, physical time = simulation time - 20.0 - We use dimless scale to run Re10000 with vz=1.0 - NekRS version (old) ``` commit 636fe6281c1db7e45ef0a223e16745bd0fb95a16 Author: Malachi Date: Mon Nov 2 09:08:19 2020 -0600 Time all reduce and copy to device in ellipticResidualProjection (#168) ``` - Runs (check `dat_logfiles`): ``` - run1 step= 1 t= 2.00000100e+01 dt=1.0e-05 C= 0.07 step= 1170 t= 2.00117000e+01 dt=1.0e-05 C= 0.07 U: 2 V: 2 W: 2 P: 15 eTime= 7.32e-01, 8.82609e+02 s - run2 step= 1 t= 2.00100500e+01 dt=5.0e-05 C= 0.34 step= 6590 t= 2.03395000e+01 dt=5.0e-05 C= 0.35 U: 2 V: 2 W: 2 P: 12 eTime= 7.06e-01, 4.36998e+03 s - run3 step= 1 t= 2.03101000e+01 dt=1.0e-04 C= 0.72 step= 3960 t= 2.07060000e+01 dt=1.0e-04 C= 0.84 U: 2 V: 2 W: 2 P: 45 eTime= 2.04e+00, 5.60561e+03 s - run4 step= 1 t= 2.06103000e+01 dt=3.0e-04 C= 2.59 step= 4090 t= 2.18370000e+01 dt=3.0e-04 C= 3.66 U: 3 V: 3 W: 3 P: 120 eTime= 4.87e+00, 1.33436e+04 s - run5 start using Pr-proj=F step= 1 t= 2.18102000e+01 dt=2.0e-04 C= 1.91 step= 5740 t= 2.29580000e+01 dt=2.0e-04 C= 1.92 U: 2 V: 2 W: 3 P: 56 eTime= 2.29e+00, 1.31354e+04 s - run6 step= 1 t= 2.28102000e+01 dt=2.0e-04 C= 1.89 step= 5580 t= 2.39260000e+01 dt=2.0e-04 C= 2.84 U: 3 V: 2 W: 3 P: 54 eTime= 2.45e+00, 1.31585e+04 s - run7 step= 1 t= 2.38101000e+01 dt=1.0e-04 C= 1.21 step= 6430 t= 2.44530000e+01 dt=1.0e-04 C= 0.95 U: 2 V: 2 W: 2 P: 43 eTime= 2.07e+00, 1.30440e+04 s - run8 step= 1 t= 2.44102000e+01 dt=2.0e-04 C= 2.10 step= 5090 t= 2.54280000e+01 dt=2.0e-04 C= 1.89 U: 2 V: 2 W: 3 P: 55 eTime= 2.76e+00, 1.31923e+04 s ```
 // Boundary conditions /* wall 1, inflow 2, outflow 3, x-slip 4, y-slip 5, z-slip 6 */ void insVelocityDirichletConditions3D(bcData *bc) { bc->uP = 0.0; bc->vP = 0.0; bc->wP = 1.0; } void insVelocityNeumannConditions3D(bcData *bc) { } void cdsDirichletConditions3D(bcData *bc) { bc->sP = 0.0; } void cdsNeumannConditions3D(bcData *bc) { bc->sF = 1.0; } // Stabilized outflow (Dong et al) void insPressureDirichletConditions3D(bcData *bc) { const dfloat iU0delta = 10.0; const dfloat un = bc->uM*bc->nx + bc->vM*bc->ny + bc->wM*bc->nz; const dfloat s0 = 0.5 * (1.0 - tanh(un*iU0delta)); bc->pP = -0.5 * (bc->uM*bc->uM + bc->vM*bc->vM + bc->wM*bc->wM) * s0; }
 [OCCA] backend = CUDA deviceNumber = 0 [GENERAL] #verbose = true polynomialOrder = 7 #startFrom = mshmesh_retry2.fld+X startFrom = r8.fld stopAt = numSteps numSteps = 200000 dt = 2.0e-4 timeStepper = tombo2 extrapolation = subCycling subCyclingSteps = 2 writeControl = TIMESTEP writeInterval = 1000 filtering = hpfrt filterWeight = 200 filterModes = 2 [PRESSURE] #preconditioner = semg_amg #amgSolver = parAlmond #galerkinCoarseOperator = yes residualTol = 1e-04 residualProj = false [VELOCITY] #solver = pcg+block boundaryTypeMap = inlet, outlet, wall, wall density = 1.0 viscosity = -10000.0 residualTol = 1e-06 #[TEMPERATURE] #solver = none #boundaryTypeMap = inlet, outlet, insulated, flux #rhoCp = 1.0 #conductivity = -100 #residualTol = 1e-06 #[SCALAR01] #solver = none [BOOMERAMG] #coarsenType = 10 #interpolationType = 6 #smootherType = -1 #iterations = 2 # testing #strongThreshold = 0.25 #nonGalerkinTol = 0.1
 // // nekRS User Defined File // #include #include "udf.hpp" /* UDF Functions */ void UDF_LoadKernels(ins_t *ins) { } void UDF_Setup(ins_t *ins) { // get IC from nek if (!ins->readRestartFile) nek_copyTo(ins->startTime); } void UDF_ExecuteStep(ins_t *ins, dfloat time, int tstep) { if (ins->isOutputStep) { nek_ocopyFrom(time, tstep); nek_userchk(); } }
 c----------------------------------------------------------------------- subroutine uservp(i,j,k,eg) ! set variable properties include 'SIZE' include 'TOTAL' include 'NEKUSE' integer i,j,k,e,eg c e = gllel(eg) udiff = 0.0 utrans = 0.0 return end c----------------------------------------------------------------------- subroutine userf(i,j,k,eg) ! set acceleration term c c Note: this is an acceleration term, NOT a force! c Thus, ffx will subsequently be multiplied by rho(x,t). c include 'SIZE' include 'TOTAL' include 'NEKUSE' integer i,j,k,e,eg c e = gllel(eg) ffx = 0.0 ffy = 0.0 ffz = 0.0 return end c----------------------------------------------------------------------- subroutine userq(i,j,k,eg) ! set source term include 'SIZE' include 'TOTAL' include 'NEKUSE' integer i,j,k,e,eg c e = gllel(eg) qvol = 0.0 return end c----------------------------------------------------------------------- subroutine userbc(i,j,k,f,eg) ! set up boundary conditions c NOTE ::: This routine MAY NOT be called by every process include 'SIZE' include 'TOTAL' include 'NEKUSE' integer i,j,k,f,e,eg e=gllel(eg) ux = 0.0 uy = 0.0 uz = 1.0 if (cbu.eq.'o ') then U0 = 1.0 ! characteristic velocity delta = 0.1 ! small positive constant pa = dongOutflow(i,j,k,e,f,U0,delta) endif flux = 0.0 temp = 0.0 c if (cbc(f,e,1).eq.'W '.AND.cbc(f,e,2).eq.'t ') temp=1.0 if (cbc(f,e,1).eq.'W '.AND.cbc(f,e,2).eq.'f ') flux=1.0 return end c----------------------------------------------------------------------- subroutine useric(i,j,k,eg) ! set up initial conditions include 'SIZE' include 'TOTAL' include 'NEKUSE' integer i,j,k,e,eg ux = 0.0 uy = 0.0 uz = 1.0 temp = 0.0 return end c----------------------------------------------------------------------- subroutine userchk include 'SIZE' include 'TOTAL' integer icalld save icalld data icalld /0/ integer elist(lelt) common /imy_fld/ elist real ca,cb,cc,cd,tmp(lx1,ly1,lz1),tmp2(lx1,ly1,lz1),plmax,plmin n=lx1*ly1*lz1*nelt if (istep.eq.0) then xxmax = glmax(xm1,n) yymax = glmax(ym1,n) zzmax = glmax(zm1,n) xxmin = glmin(xm1,n) yymin = glmin(ym1,n) zzmin = glmin(zm1,n) if (nio.eq.0) then write(6,18) xxmin,yymin,zzmin write(6,19) xxmax,yymax,zzmax 18 format(' xyz min ',5g13.5) 19 format(' xyz max ',5g13.5) endif ifxyo=.true. endif return end c----------------------------------------------------------------------- subroutine usrdat ! This routine to modify element vertices include 'SIZE' include 'TOTAL' return end c----------------------------------------------------------------------- subroutine usrdat3 include 'SIZE' include 'TOTAL' return end c----------------------------------------------------------------------- subroutine usrdat2 include 'SIZE' include 'TOTAL' integer e,f ifheat = .false. nface = 2*ldim do e=1,nelv do f=1,nface if(cbc(f,e,1).eq.'E '.or.cbc(f,e,1).eq.' ')then cbc(f,e,1)='E ' cbc(f,e,2)='E ' elseif(cbc(f,e,1).eq.'W ')then ! pebbles boundaryID(f,e) = 4 cbc(f,e,1)='W ' cbc(f,e,2)='f ' elseif(cbc(f,e,1).eq.'W01')then ! side of cylinder inner boundaryID(f,e) = 3 cbc(f,e,1)='W ' cbc(f,e,2)='I ' elseif(cbc(f,e,1).eq.'W02')then ! side of cylinder outer boundaryID(f,e) = 3 cbc(f,e,1)='W ' cbc(f,e,2)='I ' elseif(cbc(f,e,1).eq.'W03')then ! z_bottom boundaryID(f,e) = 1 cbc(f,e,1)='v ' cbc(f,e,2)='t ' elseif(cbc(f,e,1).eq.'W04')then ! z_top boundaryID(f,e) = 2 cbc(f,e,1)='o ' ! Dong's outflow cbc(f,e,2)='O ' else write(*,*)'bc incorrect',e,f,cbc(f,e,1) endif enddo enddo param(18) = 1 ! use algebraic residual norm return end c----------------------------------------------------------------------- function dongOutflow(ix,iy,iz,iel,iside,u0,delta) include 'SIZE' include 'SOLN' include 'GEOM' real sn(3) ux = vx(ix,iy,iz,iel) uy = vy(ix,iy,iz,iel) uz = vz(ix,iy,iz,iel) call getSnormal(sn,ix,iy,iz,iside,iel) vn = ux*sn(1) + uy*sn(2) + uz*sn(3) S0 = 0.5*(1.0 - tanh(vn/u0/delta)) dongOutflow = -0.5*(ux*ux+uy*uy+uz*uz)*S0 return end c----------------------------------------------------------------------- subroutine comp_Reynolds implicit none include 'SIZE' include 'TOTAL' integer i,n,npeb real Rcyl,Rpeb real z0,z1,wd,fun_z,wt(lx1*ly1*lz1*lelv) real zz,dis,glsum,glsc2 real wght_vol,area_peb,area_cyl1,area_cyl2,vz_bar,Dh1,Dh2 npeb = 1568 Rpeb = 1.0 Rcyl = 13.858 ! min/max of pebble centers are -1.05733229e+01 1.15912238e+01 ! f ------\ ! \ ! \----- ! o ) . ! P 1 | wd | wd = 1.5 z0 = -10.573 - 1.0 - wd*0.5 z1 = 11.159 + 1.0 + wd*0.5 n=lx1*ly1*lz1*nelv do i=1,n zz=zm1(i,1,1,1) if (zz>z1) then dis = -abs(zz-z0) elseif (zz>z0) then dis = min(abs(z1-zz),abs(zz-z0)) else dis = -abs(z0-zz) endif fun_z = tanh(dis*wd) wt(i) = fun_z * bm1(i,1,1,1) enddo wght_vol = glsum(wt,n) area_peb = npeb*4.0*pi*Rpeb*Rpeb area_cyl1 = 2.0*Rcyl*pi*(z1-z0) area_cyl2 = 2.0*Rcyl*pi*(z1-z0+wd) vz_bar=glsc2(wt,vz,n)/wght_vol Dh1 = 4.0*wght_vol/(area_peb+area_cyl1) Dh2 = 4.0*wght_vol/(area_peb+area_cyl2) if(nid.eq.0)then write(*,1)z0,z1,wd,wght_vol write(*,2)area_peb,area_cyl1,area_cyl2 write(*,3)time,vz_bar,Dh1,Dh2 endif 1 format('CompRe, domain',1p4e11.3) 2 format('CompRe, s_area',1p3e11.3) 3 format('CompRe, vz/Dh ',1p4e11.3) return end c-----------------------------------------------------------------------
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
 #/bin/bash : \${PROJ_ID:=""} : \${NEKRS_HOME:="\$HOME/.local/nekrs_repo1105"} : \${OCCA_CACHE_DIR:="\$PWD/.cache/occa"} NVME_HOME="/mnt/bb/\$USER/" XL_HOME="/sw/summit/xl/16.1.1-3/xlC/16.1.1" : \${CPUONLY:=0} export NEKRS_HOME export OCCA_CACHE_DIR export NEKRS_HYPRE_NUM_THREADS=1 export OGS_MPI_SUPPORT=1 export OCCA_CXX="\$XL_HOME/bin/xlc" export OCCA_CXXFLAGS="-O3 -qarch=pwr9 -qhot -DUSE_OCCA_MEM_BYTE_ALIGN=64" export OCCA_LDFLAGS="\$XL_HOME/lib/libibmc++.a" #export OCCA_VERBOSE=1 #export OMPI_LD_PRELOAD_POSTPEND=\$OLCF_SPECTRUM_MPI_ROOT/lib/libmpitrace.so #export PAMI_ENABLE_STRIPING=1 #export PAMI_IBV_ADAPTER_AFFINITY=1 #export PAMI_IBV_DEVICE_NAME="mlx5_0:1,mlx5_3:1" #export PAMI_IBV_DEVICE_NAME_1="mlx5_3:1,mlx5_0:1" if [ -z "\$PROJ_ID" ]; then echo "ERROR: PROJ_ID is empty" exit 1 fi if [ \$# -ne 3 ]; then echo "usage: [PROJ_ID] [CPUONLY=1] \$0 " exit 0 fi module unload darshan-runtime module load gcc bin=\$NEKRS_HOME/bin/nekrs case=\$1 nodes=\$2 gpu_per_node=6 cores_per_socket=21 let nn=\$nodes*\$gpu_per_node let ntasks=nn time=\$3 backend=CUDA if [ \$CPUONLY -eq 1 ]; then backend=CPU let nn=2*\$nodes let ntasks=\$nn*\$cores_per_socket fi if [ ! -f \$bin ]; then echo "Cannot find" \$bin exit 1 fi if [ ! -f \$case.par ]; then echo "Cannot find" \$case.par exit 1 fi if [ ! -f \$case.co2 ]; then echo "Cannot find" \$case.co2 exit 1 fi if [ ! -f \$case.udf ]; then echo "Cannot find" \$case.udf exit 1 fi if [ ! -f \$case.oudf ]; then echo "Cannot find" \$case.oudf exit 1 fi if [ ! -f \$case.re2 ]; then echo "Cannot find" \$case.re2 exit 1 fi mkdir -p \$OCCA_CACHE_DIR 2>/dev/null while true; do read -p "Do you want precompile (recommended)? [N]" yn case \$yn in [Yy]* ) echo \$NEKRS_HOME CUDA_VISIBLE_DEVICES=0 mpirun -pami_noib -np 1 \$NEKRS_HOME/bin/nekrs --setup \$case --build-only \$ntasks --backend \$backend; if [ \$? -ne 0 ]; then exit 1 fi break ;; * ) break ;; esac done if [ \$CPUONLY -eq 1 ]; then jsrun="jsrun -X 1 -n\$nodes -r1 -a1 -c1 -g0 -b packed:1 -d packed cp -a \$OCCA_CACHE_DIR/* \$NVME_HOME; export OCCA_CACHE_DIR=\$NVME_HOME; jsrun -X 1 -n\$nn -a\$cores_per_socket -c\$cores_per_socket -g0 -b packed:1 -d packed \$bin --setup \$case --backend \$backend" else jsrun="jsrun -X 1 -n\$nodes -r1 -a1 -c1 -g0 -b packed:1 -d packed cp -a \$OCCA_CACHE_DIR/* \$NVME_HOME; export OCCA_CACHE_DIR=\$NVME_HOME; jsrun --smpiargs='-gpu' -X 1 -n\$nn -r\$gpu_per_node -a1 -c2 -g1 -b rs -d packed \$bin --setup \$case --backend \$backend --device-id 0" fi cmd="bsub -nnodes \$nodes -alloc_flags NVME -W \$time -P \$PROJ_ID -J nekRS_\$case \"\${jsrun}\"" echo \$cmd echo \$cmd > qq \$cmd
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment