Starting actual Nek coupling

examples/integration_example/Makefile

+###############################################################################
+################### MOOSE Application Standard Makefile #######################
+###############################################################################
+#
+# Optional Environment variables
+# MOOSE_DIR     - Root directory of the MOOSE project
+# FRAMEWORK_DIR - Location of the MOOSE framework
+#
+###############################################################################
+EXAMPLE_DIR        ?= $(shell dirname `pwd`)
+MOON_DIR           ?= $(shell dirname $(EXAMPLE_DIR))
+MOOSE_DIR          ?= $(MOON_DIR)/moose
+FRAMEWORK_DIR      ?= $(MOOSE_DIR)/framework
+###############################################################################
+
+# framework
+include $(FRAMEWORK_DIR)/build.mk
+include $(FRAMEWORK_DIR)/moose.mk
+
+################################## MODULES ####################################
+SOLID_MECHANICS   := yes
+CONTACT           := yes
+HEAT_CONDUCTION   := yes
+WATER_STEAM_EOS   := yes
+MISC              := yes
+PHASE_FIELD       := yes
+XFEM              := yes
+include           $(MOOSE_DIR)/modules/modules.mk
+###############################################################################
+
+# nek
+NEK_SOURCE_DIR     := $(MOON_DIR)/Nek5000
+NEK_CASE_DIR       := $(CURDIR)
+NEK_CASENAME       := $(shell basename $(NEK_CASE_DIR))
+NEK_OBJ            := obj/$(NEK_CASENAME).o
+NEK_LIB            := $(NEK_CASE_DIR)/libnek5000.a
+include            $(NEK_SOURCE_DIR)/nek.mk
+
+# moon
+APPLICATION_DIR    := $(MOON_DIR)
+APPLICATION_NAME   := moon
+DEP_APPS           := $(shell $(FRAMEWORK_DIR)/scripts/find_dep_apps.py $(APPLICATION_NAME))
+include            $(FRAMEWORK_DIR)/app.mk
+
+# case
+APPLICATION_DIR    := $(CURDIR)
+APPLICATION_NAME   := $(shell basename $(APPLICATION_DIR))
+MAIN_DIR           := $(MOON_DIR)/src
+BUILD_EXEC         := yes
+EXTERNAL_FLAGS     := $(NEK_OBJ) $(NEK_LIB) -Wl,-no_compact_unwind
+DEP_APPS           := $(shell $(FRAMEWORK_DIR)/scripts/find_dep_apps.py $(APPLICATION_NAME))
+include            $(FRAMEWORK_DIR)/app.mk
+
+###############################################################################
+# Additional special case targets should be added here

examples/integration_example/SIZE

+C     Dimension file to be included
+C
+C     HCUBE array dimensions
+C
+      parameter (ldim=3)
+      parameter (lx1=5,ly1=lx1,lz1=lx1,lelt=480,lelv=lelt)
+      parameter (lxd=8,lyd=lxd,lzd=lxd)
+      parameter (lelx=1,lely=1,lelz=1)
+ 
+      parameter (lzl=3 + 2*(ldim-3))
+ 
+      parameter (lx2=lx1-2)
+      parameter (ly2=ly1-2)
+      parameter (lz2=lz1-2)
+      parameter (lx3=lx1)
+      parameter (ly3=ly1)
+      parameter (lz3=lz1)
+
+      parameter (lp = 32)
+      parameter (lelg = 500)
+c
+c     parameter (lpelv=lelv,lpelt=lelt,lpert=3)  ! perturbation
+c     parameter (lpx1=lx1,lpy1=ly1,lpz1=lz1)     ! array sizes
+c     parameter (lpx2=lx2,lpy2=ly2,lpz2=lz2)
+c
+      parameter (lpelv=1,lpelt=1,lpert=1)        ! perturbation
+      parameter (lpx1=1,lpy1=1,lpz1=1)           ! array sizes
+      parameter (lpx2=1,lpy2=1,lpz2=1)
+c
+c     parameter (lbelv=lelv,lbelt=lelt)          ! MHD
+c     parameter (lbx1=lx1,lby1=ly1,lbz1=lz1)     ! array sizes
+c     parameter (lbx2=lx2,lby2=ly2,lbz2=lz2)
+c
+      parameter (lbelv=1,lbelt=1)                ! MHD
+      parameter (lbx1=1,lby1=1,lbz1=1)           ! array sizes
+      parameter (lbx2=1,lby2=1,lbz2=1)
+ 
+C     LX1M=LX1 when there are moving meshes; =1 otherwise
+      parameter (lx1m=lx1,ly1m=ly1,lz1m=lz1)
+      parameter (ldimt= 4)                       ! 3 passive scalars + T
+      parameter (ldimt1=ldimt+1)
+      parameter (ldimt3=ldimt+3)
+c
+c     Note:  In the new code, LELGEC should be about sqrt(LELG)
+c
+      PARAMETER (LELGEC = 1)
+      PARAMETER (LXYZ2  = 1)
+      PARAMETER (LXZ21  = 1)
+ 
+      PARAMETER (LMAXV=LX1*LY1*LZ1*LELV)
+      PARAMETER (LMAXT=LX1*LY1*LZ1*LELT)
+      PARAMETER (LMAXP=LX2*LY2*LZ2*LELV)
+      PARAMETER (LXZ=LX1*LZ1)
+      PARAMETER (LORDER=3)
+      PARAMETER (MAXOBJ=4,MAXMBR=LELT*6)
+      PARAMETER (lhis=100)         ! # of pts a proc reads from hpts.in
+                                   ! Note: lhis*np > npoints in hpts.in
+C
+C     Common Block Dimensions
+C
+      PARAMETER (LCTMP0 =2*LX1*LY1*LZ1*LELT)
+      PARAMETER (LCTMP1 =4*LX1*LY1*LZ1*LELT)
+C
+C     The parameter LVEC controls whether an additional 42 field arrays
+C     are required for Steady State Solutions.  If you are not using
+C     Steady State, it is recommended that LVEC=1.
+C
+      PARAMETER (LVEC=1)
+C
+C     Uzawa projection array dimensions
+C
+      parameter (mxprev = 40)
+      parameter (lgmres = 40)
+C
+C     Split projection array dimensions
+C
+      parameter(lmvec = 1)
+      parameter(lsvec = 1)
+      parameter(lstore=lmvec*lsvec)
+c
+c     NONCONFORMING STUFF
+c
+      parameter (maxmor = lelt)
+C
+C     Array dimensions
+C
+      COMMON/DIMN/NELV,NELT,NX1,NY1,NZ1,NX2,NY2,NZ2
+     $,NX3,NY3,NZ3,NDIM,NFIELD,NPERT,NID
+     $,NXD,NYD,NZD
+
+c automatically added by makenek
+      parameter(lxo   = lx1) ! max output grid size (lxo>=lx1)
+
+c automatically added by makenek
+      parameter(lpart = 62000) ! max number of particles
+
+c automatically added by makenek
+      integer ax1,ay1,az1,ax2,ay2,az2
+      parameter (ax1=1,ay1=1,az1=1,ax2=1,ay2=1,az2=1) ! running averages
+
+c automatically added by makenek
+      parameter (lxs=1,lys=lxs,lzs=(lxs-1)*(ldim-2)+1) !New Pressure Preconditioner
+
+c automatically added by makenek
+      parameter (lfdm=0)  ! == 1 for fast diagonalization method
+
+c automatically added by makenek
+      common/IOFLAG/nio  ! for logfile verbosity control
+
+c automatically added by makenek
+      integer toteq
+      parameter(toteq = 1  ) ! Number of conserved variables 
+c IFCMT=FALSE  then toteq=1
+
+c automatically added by makenek
+      integer lelcmt
+      parameter(lelcmt = 1  ) ! # of cmt elements 
+c IFCMT=FALSE  then lelcmt=1

examples/integration_example/build_coupled_nek.sh

+#k!/usr/bin/env bash
+NEK_CASENAME="$(basename $(pwd))"
+NEK_SOURCE_DIR="$(dirname $(dirname $(pwd)))/Nek5000"
+
+# MOOSE issues a warning if these directories do not exist.
+# Creating these dirs will suppress that warning
+mkdir -p src
+mkdir -p include/base
+
+SCH_FILE="$NEK_CASENAME.sch"
+if [ -e $SCH_FILE ]; then
+  echo "Deleting $SCH_FILE..."
+  rm -f $SCH_FILE
+fi
+
+echo "Creating SESSON.NAME..."
+echo "$NEK_CASENAME" > SESSION.NAME && pwd >> SESSION.NAME
+
+echo "Building Nek5000 objects..."
+$NEK_SOURCE_DIR/core/makenek $NEK_CASENAME $NEK_SOURCE_DIR
+
+echo "Building libNek5000.a..."
+make -f makefile.nek lib
+
+echo "Building MOON driver..."
+make

examples/integration_example/coefficient_integration.i

@@ -68,7 +68,7 @@
   # BCFunction just returns 0.0 right now
   [./bc_func]
     type = ConstantFunction
-    
+
   [../]
   [./legendre_function]
     type = LegendrePolynomial
@@ -280,10 +280,9 @@
 []
 
 [Executioner]
-
   type = Transient
   scheme     = 'Explicit-Euler' # Others available: backward Euler, Crank-Nicholson, etc.
-  dt         = 0.001    # Initial timestep size
+  dt         = 0.01     # Initial timestep size
   start_time = 0        # Starting time
   num_steps  = 20       # DIVERGES AFTER 4 TIMESTEPS
   nl_rel_tol = 1e-8     # Nonlinear relative tolerance
@@ -298,3 +297,30 @@
   print_perf_log = true
 []
 
+[MultiApps]
+  [./sub]
+    type = TransientMultiApp
+    app_type = MoonApp
+    sub_cycling = true
+    positions = '0 0 0'
+    input_files = picard_sub_subcycling.i
+  [../]
+[]
+
+[Transfers]
+  [./u_to_sub]
+    type = MultiAppNearestNodeTransfer
+    direction = to_multiapp
+    multi_app = sub
+    source_variable = u
+    variable = aux
+  [../]
+
+[./u_from_sub]
+    type = MultiAppNearestNodeTransfer
+    direction = from_multiapp
+    multi_app = sub
+    source_variable = u
+    variable = aux
+  [../]
+[]

examples/integration_example/integration_example.usr

+c-----------------------------------------------------------------------
+C
+C  USER SPECIFIED ROUTINES:
+C
+C     - boundary conditions
+C     - initial conditions
+C     - variable properties
+C     - local acceleration for fluid (a)
+C     - forcing function for passive scalar (q)
+C     - general purpose routine for checking errors etc.
+C
+c-----------------------------------------------------------------------
+      subroutine uservp (ix,iy,iz,eg)
+      include 'SIZE'
+      include 'TOTAL'
+      include 'NEKUSE'
+      integer e,f,eg
+      udiff =0.
+      utrans=0.
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine userf  (ix,iy,iz,eg)
+      include 'SIZE'
+      include 'TOTAL'
+      include 'NEKUSE'
+      integer e,f,eg
+      ffx=0.0
+      ffy=0.0
+      ffz=0.0     
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine userq  (ix,iy,iz,eg)
+      include 'SIZE'
+      include 'TOTAL'
+      include 'NEKUSE'
+
+      integer e,f,eg
+c     e = gllel(eg)
+
+      qvol   = 0.0
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine userchk
+      include 'SIZE'
+      include 'TOTAL'
+
+      common/test_passing/ flux_moose, temp_nek
+      real sint, sint1, sarea, sarea1, wtmp
+      integer e, f 
+
+      n1=nelt*lx1*ly1*lz1
+      n2=nelt*lx2*ly2*lz2
+
+      pmin=glmin(pr,n2)
+      pmax=glmax(pr,n2)
+
+      wmin=glmin(vz,n1)
+      wmax=glmax(vz,n1)
+
+      tmin=glmin(t,n1)
+      tmax=glmax(t,n1)
+
+      ifflow=.false. 
+
+      sint1=0.0
+      sarea1=0.0
+
+      do e=1,lelt
+        do f=1,6
+          call surface_int(sint,sarea,t,e,f)
+          if (cbc(f,e,1).eq.'W  ') then
+           sint1=sint1+sint  
+           sarea1=sarea1+sarea
+          endif          
+         enddo 
+      enddo
+
+      call  gop(sint1,wtmp,'+  ',1)
+      call  gop(sarea1,wtmp,'+  ',1)
+
+      temp_nek=sint1/sarea1
+      
+      if (nid.eq.0) then
+         write(6,*)"*** Temperature: ",tmin," - ",tmax
+         write(6,*)"*** Av. Temperature: ",temp_nek
+      endif
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine userbc (ix,iy,iz,iside,ieg)
+      include 'SIZE'
+      include 'TOTAL'
+      include 'NEKUSE'
+      
+      common/test_passing/ flux_moose, temp_nek
+
+      integer e,ieg
+      real ucx, ucy, ucz, ucy_e, yy 
+
+      e=gllel(ieg)
+
+      zz  = zm1(ix,iy,iz,e)
+      xx  = xm1(ix,iy,iz,e)
+      yy  = ym1(ix,iy,iz,e)
+      rr  = sqrt(xx**2 + yy**2)
+
+      ux = 0.0
+      uy = 0.0
+      uz = 1.0
+      temp = 0.0
+      flux = 1.0 !flux_moose
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine useric (ix,iy,iz,ieg)
+      include 'SIZE'
+      include 'TOTAL'
+      include 'NEKUSE'
+
+      integer idum, e
+      save    idum
+      data    idum / 0 /
+
+      if (idum.eq.0) idum = 99 + nid
+      eps = .35
+
+      uz=0.0
+      uy=0.0
+      ux=0.0
+      temp=0
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine usrdat
+      include 'SIZE'
+      include 'TOTAL'
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine usrdat2
+      include 'SIZE'
+      include 'TOTAL'
+
+      param(66) = 4.   ! These give the std nek binary i/o and are 
+      param(67) = 4.   ! good default values
+      ifuservp=.false.
+
+      return
+      end
+c-----------------------------------------------------------------------
+      subroutine usrdat3
+      include 'SIZE'
+      include 'TOTAL'
+c
+      return
+      end
+
+C=======================================================================
+      subroutine nek_init_step()
+      include 'SIZE'
+      include 'TSTEP'
+      include 'INPUT'
+      include 'CTIMER'
+      real*4 papi_mflops
+      integer*8 papi_flops
+      integer icall, kstep, i, pstep
+      common /cht_coupler/ pstep
+      save icall
+ 
+      if (icall.eq.0) then
+      call nekgsync()
+      if (instep.eq.0) then
+        if(nid.eq.0) write(6,'(/,A,/,A,/)')
+     &     ' nsteps=0 -> skip time loop',
+     &     ' running solver in post processing mode'
+      else
+        if(nio.eq.0) write(6,'(/,A,/)') 'Starting time loop ...'
+      endif
+      isyc  = 0
+      itime = 0
+      if(ifsync) isyc=1
+      itime = 1
+      call nek_comm_settings(isyc,itime)
+      call nek_comm_startstat()
+      istep  = 0
+      endif
+
+      istep=istep+1
+
+      if (lastep .eq. 1) then
+        pstep=2
+      else
+        call nek_advance
+        pstep=2
+      endif
+
+      return
+      end
+C=======================================================================
+      subroutine nek_step()
+
+      include 'SIZE'
+      include 'TSTEP'
+      include 'INPUT'
+      include 'CTIMER'
+      common /cht_coupler/ pstep
+      integer pstep
+
+      pstep=pstep+1
+      call heat(pstep)
+
+      return
+      end
+C=======================================================================
+      subroutine nek_finalize_step()
+
+      include 'SIZE'
+      include 'TSTEP'
+      include 'INPUT'
+      include 'CTIMER'
+      common /cht_coupler/ pstep
+      integer pstep 
+
+      real*4 papi_mflops
+      integer*8 papi_flops
+      integer icall, kstep, knstep, i
+
+      if (param(103).gt.0)   call q_filter      (param(103))
+      call setup_convect (pstep)  ! Save convective velocity _after_ filter
+
+      call userchk
+      call prepost (.false.,'his')
+      call in_situ_check()
+
+      if (mod(istep,nsteps).eq.0) lastep=1
+      call nek_comm_settings(isyc,0)
+      call comment
+
+      return
+      end
+c-----------------------------------------------------------------------

examples/integration_example/picard_sub_subcycling.i

+# Minimal Nek-wrapped MOOSE application.
+# MOOSE Requires a mesh and a variable even
+# if they are not used.
+
+[Mesh]
+  type = GeneratedMesh
+  dim = 1
+  nx = 1
+[]
+
+[Variables]
+  [./u]
+  [../]
+[]
+
+[AuxVariables]
+  [./aux]
+  [../]
+[]
+
+[Executioner]
+  type = NekExecutioner
+  num_steps = 20
+  [./TimeStepper]
+    type = NekTimeStepper
+    dt = 0.01
+  [../]
+[]
+
+[Problem]
+  kernel_coverage_check = false
+  solve = false
+[]