New version of S3D-IO. More recent and with the ability to disable NetCFD

examples/HACC-IO/run_check.sh

@@ -6,7 +6,7 @@ PPN=16
 NPROCS=$((NODES*PPN))
 TARGET="/projects/visualization/ftessier/debug"
 
-cd $HOME/TAPIOCA/examples
+cd $HOME/TAPIOCA/examples/HACC-IO
 
 export TAPIOCA_DEVNULL=false
 export TAPIOCA_COMMSPLIT=true

examples/S3D-IO/COPYRIGHT

-Copyright 2003-2013 Northwestern University
-
-Portions of this software were developed by the Sandia National Laboratory.
-
-Access and use of this software shall impose the following obligations
-and understandings on the user. The user is granted the right, without
-any fee or cost, to use, copy, modify, alter, enhance and distribute
-this software, and any derivative works thereof, and its supporting
-documentation for any purpose whatsoever, provided that this entire
-notice appears in all copies of the software, derivative works and
-supporting documentation. Further,  Northwestern University requests
-that the user credit Northwestern University in any publications that
-result from the use of this software or in any product that includes
-this software.  The name Northwestern University, however, may not be
-used in any advertising or publicity to endorse or promote any
-products or commercial entity unless specific written permission is
-obtained from Northwestern University. The user also understands that
-Northwestern University is not obligated to provide the user with
-any support, consulting, training or assistance of any kind with regard
-to the use, operation and performance of this software nor to provide
-the user with any updates, revisions, new versions or "bug fixes."
-
-THIS SOFTWARE IS PROVIDED BY NORTHWESTERN UNIVERSITY "AS IS" AND ANY
-EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NORTHWESTERN UNIVERSITY BE
-LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
-DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
-WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
-ARISING OUT OF OR IN CONNECTION WITH THE ACCESS, USE OR PERFORMANCE
-OF THIS SOFTWARE.
-

examples/S3D-IO/Makefile

-#
-#  Copyright (C) 2013, Northwestern University
-#  See COPYRIGHT notice in top-level directory.
-#
-#  $Id: Makefile 3485 2015-12-27 00:06:31Z wkliao $
-#
-#
-# Please change the following variables:
-#    MPIF90        -- MPI Fortran compiler
-#    FCFLAGS       -- Compile flag
-#    PNETCDF_DIR   -- PnetCDF library installation directory
-#
-
-MPIF90      = mpif90
-FCFLAGS     = -Wall -g
-PNETCDF_DIR = $(HOME)/PnetCDF
-PNETCDF_DIR = $(HOME)
-
-
-COMPILE_F90  = $(MPIF90) $(FCFLAGS) $(INC) -c
-LINK         = $(MPIF90) $(FCFLAGS)
-INC          = -I$(PNETCDF_DIR)/include
-LIBS         = -L$(PNETCDF_DIR)/lib -lpnetcdf
-
-SRCS = runtime_m.f90 \
-       param_m.f90 \
-       topology_m.f90 \
-       variables_m.f90 \
-       io_profiling_m.f90 \
-       pnetcdf_m.f90 \
-       init_field.f90 \
-       io.f90 \
-       random_number.f90 \
-       solve_driver.f90 \
-       main.f90
-
-OBJS = $(SRCS:.f90=.o)
-MODS = $(SRCS:.f90=.mod)
-
-TARGET = s3d_io.x
-
-all: $(TARGET)
-
-%.o:%.f90
-	$(COMPILE_F90) $<
-
-$(TARGET): $(OBJS)
-	$(LINK) $(OBJS) -o $(TARGET) $(LIBS)
-
-PACKAGE_NAME = s3d-io-pnetcdf-1.1
-
-PACKING_LIST = $(SRCS) Makefile README COPYRIGHT RELEASE_NOTE
-
-dist:
-	/bin/rm -rf $(PACKAGE_NAME) $(PACKAGE_NAME).tar.gz
-	mkdir $(PACKAGE_NAME)
-	cp $(PACKING_LIST) $(PACKAGE_NAME)
-	tar -cf $(PACKAGE_NAME).tar $(PACKAGE_NAME)
-	gzip $(PACKAGE_NAME).tar
-	/bin/rm -rf $(PACKAGE_NAME)
-
-clean: 
-	/bin/rm -f $(OBJS) $(MODS) $(TARGET)
-
-distclean: clean
-	/bin/rm -rf $(PACKAGE_NAME).tar.gz $(PACKAGE_NAME)

examples/S3D-IO/README

-#
-#  Copyright (C) 2013, Northwestern University
-#  See COPYRIGHT notice in top-level directory.
-#
-#  $Id: README 3457 2015-11-21 23:07:56Z wkliao $
-
-This benchmark programs is the I/O kernel of S3D combustion simulation code.
-http://exactcodesign.org/ There are several I/O methods implemented in S3D.
-This software only contains the method of Parallel NetCDF.
-
-S3D is a continuum scale first principles direct numerical simulation code
-which solves the compressible governing equations of mass continuity, momenta,
-energy and mass fractions of chemical species including chemical reactions.
-Readers are referred to the published paper below.  J. Chen, A. Choudhary, B.
-de Supinski, M. DeVries, E. Hawkes, S. Klasky, W. Liao, K. Ma, J. Crummey, N.
-Podhorszki, R. Sankaran, S. Shende, and C. Yoo. Teras-cale Direct Numerical
-Simulations of Turbulent Combustion Using S3D. In Computational Science and
-Discovery Volume 2, January 2009.
-
-I/O pattern:
-A checkpoint is performed at regular intervals, and its data consist of 8-byte
-three-dimensional arrays. At each checkpoint, four global arrays, representing
-mass, velocity, pressure, and temperature, respectively, are written to a newly
-created file in the canonical order. Mass and velocity are four-dimensional
-arrays while pressure and temperature are three-dimensional arrays. All four
-arrays share the same size for the lowest three spatial dimensions X, Y, and Z,
-which are partitioned among MPI processes in a block-block-block fashion. For
-the mass and velocity arrays, the length of the fourth dimension is 11 and 3,
-respectively. The fourth dimension, the most significant one, is not
-partitioned. As the number of MPI processes increases, the aggregate I/O
-amount proportionally increases as well.
-
-For more detailed description of the data partitioning and I/O patterns,
-please refer to the following paper.
-    W. Liao and A. Choudhary. Dynamically Adapting File Domain Partitioning
-    Methods for Collective I/O Based on Underlying Parallel File System
-    Locking Protocols. In the Proceedings of International Conference for
-    High Performance Computing, Networking, Storage and Analysis, Austin,
-    Texas, November 2008.
-
-To compile:
-Edit Makefile and set/change variables:
-    MPIF90         - MPI Fortran 90 compiler
-    FCFLAGS        - compile flags
-    PNETCDF_DIR    - the path of PnetCDF library
-                     (1.4.0 and higher is required)
-
-    For example:
-        MPIF90      = mpif90
-        FCFLAGS     = -O2
-        PNETCDF_DIR = ${HOME}/PnetCDF
-
-To run:
-   Usage: s3d_io.x nx_g ny_g nz_g npx npy npz dir_path
-   There are 9 command-line arguments:
-       nx_g     - GLOBAL grid size along X dimension
-       ny_g     - GLOBAL grid size along Y dimension
-       nz_g     - GLOBAL grid size along Z dimension
-       npx      - number of MPI processes along X dimension
-       npy      - number of MPI processes along Y dimension
-       npz      - number of MPI processes along Z dimension
-       method   - 0: using PnetCDF blocking APIs, 1: nonblocking APIs
-       restart  - restart from reading a previous written file (True/False)
-       dir_path - the directory name to store the output files
-
-   To change the number of checkpoint dumps (default is set to 5), edit
-   file param_m.f90 and set a different value for i_time_end:
-       i_time_end = 5   ! number of checkpoints (also number of output files)
-
-   The contents of all variables written to files are set to random numbers.
-   This setting can be disabled by comment out the line below in file
-   solve_driver.f90
-       call random_set
-
-Example run command:
-   For a test run with small data size and a short return time, here is an
-   example command for running on 4 MPI processes.
-   mpiexec -n 4 ./s3d_io.x 10 10 10 2 2 1 1 F .
-
-   The command below runs on 4096 MPI processes with the global array
-   of size 800x800x800 and local array of size 50x50x50, output directory
-   /scratch1/scratchdirs/wkliao/FS_1M_96 using nonblocking APIs, and without
-   restart.
-
-   mpiexec -l -n 512 ./s3d_io.x 800 800 800 16 16 16 1 F /scratch1/scratchdirs/wkliao/FS_1M_96
-
-Example output from stdout:
- ++++ I/O is done through PnetCDF ++++
- I/O method          : nonblocking APIs
- Run with restart    : False
- No. MPI processes   :    4096
- Global array size   :     800 x    800 x    800
- output file path    : /scratch1/scratchdirs/wkliao/FS_1M_96
- file striping count :      96
- file striping size  : 1048576 bytes
- -----------------------------------------------
- Time for open       :        0.11 sec
- Time for read       :        0.00 sec
- Time for write      :       18.04 sec
- Time for close      :        0.02 sec
- no. read  calls     :        0    per process
- no. write calls     :       20    per process
- total read  amount  :        0.00 GiB
- total write amount  :      305.18 GiB
- read  bandwidth     :        0.00 MiB/s
- write bandwidth     :    17318.78 MiB/s
- -----------------------------------------------
- total I/O   amount  :      305.18 GiB
- total I/O   time    :       18.17 sec
- I/O   bandwidth     :    17201.53 MiB/s
-
-Questions/Comments:
-email: wkliao@eecs.northwestern.edu
+
+This is an I/O simulator for the DNS code S3D; the physics modules are
+missing but the memory arrangement and I/O routines are taken from the
+production code.
+
+If you use this simulator in your work and didn't receive it directly
+from the S3D group at Sandia National Laboratories, please let us
+know by emailing Jackie Chen at jhchen@sandia.gov. If you do something
+interesting, we ask that you share the results with us. If you're
+planning to publish work that uses theses routines please contact us
+at the outset.
+
+Both makefiles and CMake scripts are provided for building the
+simulator; CMake is the build system used for production S3D runs and
+has the convenient feature of allowing for out of source builds and an
+installation command to arrange the executables and input files
+appropriately to run the simulator, e.g.:
+
+tar -xzvf s3dio.tgz
+mkdir build
+cd build
+ccmake ../S3D-IO
+make 
+make install
+
+During the ccmake step the different IO methods to be built can be
+enabled / disabled, paths to the necessary libraries can be provided,
+and the run tree directory can be specified.
+
+The run tree will look like:
+./s3d_run/run
+./s3d_run/input
+./s3d_run/post
+./s3d_run/data
+
+The executable and a sample job submission script are in the run directory.
+
+The job size and the IO method can be selected by modifying the
+contens of ./s3d_run/input/s3d.in: in the "GRID DIMENSION PARAMETERS"
+section nx_g, ny_g, nz_g is the global grid size, and npx, npy, npz
+are the dimensions of the decomposition topology. It is necessary
+that:
+
+mod(nx_g, npx) == 0
+
+and
+
+nx_g/npx > 10
+
+usually, 30 < nx_g/npx < 45.
+
+The total mpi job size should be npx*npy*npz.
+
+The I/O method is selected on the last line of ./s3d_run/input/s3d.in;
+the availabile methods are: 
+
+Fortran I/O - 1 file per mpi process per output time 
+MPI-IO - 1 file per timestep 
+PnetCDF - 1 netCDF file per timestep written using parallel netCDF 
+HDF5 - 1 HDF5 file per timestep written using parallel HDF5
+
+The latter 3 options, the collective I/O routines, were provided by
+Alok Choudhary (choudhar@ece.northwestern.edu) and Wei-keng Liao
+(wkliao@ece.northwestern.edu) and questions about these routines
+should be sent directly to them.
+
+
+
 

examples/S3D-IO/README.pnetcdf

+Source file for parallel netCDF I/O method is in
+    source/modules/pnetcdf_m.f90
+Two subroutines are :
+    pnetcdf_write() and pnetcdf_read()
+
+
+
+To compile and link a parallel netcdf library, a pnetcdf library is
+available in ~wkliao/PnetCDF
+
+To check the datasets and attributes saved in a netcdf file, use command
+% ncdump -c netcdf_file_name.nc
+
+A serial netcdf library is built in ~wkliao/NetCDF. The command ncdump is
+available in ~wkliao/NetCDF/bin
+
+An example of using ncdump is given below.
+
+jaguar10 ::run(12:59am) #423% ~wkliao/NetCDF/bin/ncdump -c ../data/pressure_wave_test.1.000E-06.field.nc
+
+netcdf pressure_wave_test.0.000E+00.field {
+dimensions:
+        nx_g = 100 ;
+        ny_g = 100 ;
+        nz_g = 50 ;
+        number_of_species = 11 ;
+        number_of_velocity_components = 3 ;
+variables:
+        double yspecies(number_of_species, nz_g, ny_g, nx_g) ;
+                yspecies:specie_name_01 = "Y-H2              " ;
+                yspecies:specie_name_02 = "Y-O2              " ;
+                yspecies:specie_name_03 = "Y-O               " ;
+                yspecies:specie_name_04 = "Y-OH              " ;
+                yspecies:specie_name_05 = "Y-H2O             " ;
+                yspecies:specie_name_06 = "Y-H               " ;
+                yspecies:specie_name_07 = "Y-HO2             " ;
+                yspecies:specie_name_08 = "Y-CO              " ;
+                yspecies:specie_name_09 = "Y-CO2             " ;
+                yspecies:specie_name_10 = "Y-HCO             " ;
+                yspecies:specie_name_11 = "Y-N2              " ;
+        double temp(nz_g, ny_g, nx_g) ;
+        double pressure(nz_g, ny_g, nx_g) ;
+        double u(number_of_velocity_components, nz_g, ny_g, nx_g) ;
+                u:velocity_component_1 = "u" ;
+                u:velocity_component_2 = "v" ;
+                u:velocity_component_3 = "w" ;
+
+// global attributes:
+                :number_of_elements_in_reaction_mechansim = 4 ;
+                :number_of_species_in_reaction_mechansim = 11 ;
+                :number_of_steps_in_reaction_mechansims = 21 ;
+                :number_of_reaction_third-body_reactions = 7 ;
+                :element_name_1 = "C               " ;
+                :element_name_2 = "H               " ;
+                :element_name_3 = "O               " ;
+                :element_name_4 = "N               " ;
+                :molecular_weight:H2 = 0.00201594 ;
+                :molecular_weight:O2 = 0.0319988 ;
+                :molecular_weight:O = 0.0159994 ;
+                :molecular_weight:OH = 0.01700737 ;
+                :molecular_weight:H2O = 0.01801534 ;
+                :molecular_weight:H = 0.00100797 ;
+                :molecular_weight:HO2 = 0.03300677 ;
+                :molecular_weight:CO = 0.02801055 ;
+                :molecular_weight:CO2 = 0.04400995 ;
+                :molecular_weight:HCO = 0.02901852 ;
+                :molecular_weight:N2 = 0.0280134 ;
+                :universal_gas_constant = 8.314 ;
+                :freestream_temperature = 300. ;
+                :reference_ratio_of_specifice_heats = 1.4 ;
+                :reference_speed_of_sound = 347.2 ;
+                :reference_density = 1.1766 ;
+                :reference_conductivity = 0.02614 ;
+                :reference_temperature = 120. ;
+                :reference_pressure = 141836.588544 ;
+                :standard_atmospheric_pressure = 101325. ;
+                :reference_time = 1.29889001100892e-05 ;
+                :reference_specific_heat = 1004.56533333333 ;
+                :reference_length = 0.00450974611822298 ;
+                :reference_viscosity = 1.84230128055384 ;
+                :acoustic_Reynolds_number = 100000. ;
+                :Mach_number = 0.001 ;
+                :convective_Reynolds_number = 100. ;
+                :time = 0. ;
+                :tstep = 0. ;
+                :time_save = 100000. ;
+                :pout = 0. ;
+}
+

examples/S3D-IO/README.wkliao

+There are three I/O methods implemented:
+  1. MPI collective I/O method in source/modules/mpi_io_m.f90
+  2. Parallel netCDF method in source/modules/pnetcdf_m.f90
+  3. HDF5 method in source/modules/hdf5_m.f90
+
+When using these methods, there will be only one file created per checkpoint.
+At each checkpoint, all variables are saved in a single file in its global
+cacnonical order. File extensions are added to the filenames for the three
+I/O methods: mpi, nc, h5 are for MPI-IO, PnetCDF, and HDF5, respectively.
+
+All source changes made by this implementation is shown in file
+Changes.wkliao.
+
+A separate MPI-IO library s built with a few optimizations for I/O
+enhancement particularly for Jaguar. The library is located in
+   /ccs/home/wkliao/ROMIO/lib on Jaguar.
+Please check the make file in build/make.CNL for compiling and linking.
+
+Please send questions to:
+
+Wei-keng Liao
+wkliao@ece.northwestern.edu
+Aug. 9, 2008

examples/S3D-IO/RELEASE_NOTE

-Release note
-
-version 1.1
-  * Add command-line option to choose PnetCDF blocking APIs or nonblocking APIs.
-  * Add command-line option for whether to run from a restart file.
-  * Reorganize performance result output.
-

examples/S3D-IO/build/Makefile

+#  $Id: Makefile,v 1.6.6.1 2006/04/04 02:01:25 rsankar Exp $
+#----------------------------------------------------------------------
+# makefile for program S3D
+#
+# usage:  make MACH=[mach]
+#         where [mach] corresponds to an extension on a "make.[mach]" file.
+#
+# For example, if I was building on an SGI, I would create a file called make.SGI
+# (see below) and then issue the command "make MACH=SGI"
+#
+# Alternatively (and perhaps a better approach) you may set an environment variable
+# "MACH" to the appropriate platform...  eg "setenv MACH SGI"
+#
+#----------------------------------------------------------------------
+
+# the include file provides specific rules for building on your platform.
+# You need to provide a compiler for F90 and F77 files, as well as a link 
+# statement that will link with the appropriate libraries (MPI, math, etc).
+#----------------------------------------------------------------------
+# Revisions - 
+# 01/03/05 - Ramanan Sankaran
+# New version of makefile with dependencies auto-inserted using sfmakedepend
+#----------------------------------------------------------------------
+
+MACH = BGQ
+
+include make.$(MACH)
+
+MYERRMSG = 
+MAKEDEPEND = ./sfmakedepend
+
+#-----------------------------------------------------------
+# Set paths for various source files.
+
+MODULES	= ../source/modules/
+MAIN	= ../source/drivers/
+INIT	= ../source/f90_files/init/
+SOLVE	= ../source/f90_files/solve/
+POST	= ../source/f90_files/post/
+FSEVEN	= ../source/f77_files/
+FSEVEN_HDR	= ../source/include/
+MORPH	= ../source/utils/newmorph/
+OTHER   = ../source/other_files/
+
+#----------------------------------------------------------------------
+# Default compilation rules based on file extension
+
+.f.o:
+	$(COMPILE_F77) -I$(FSEVEN_HDR) $<
+
+%.o:%.f90
+	$(COMPILE_F90) $<
+
+.c.o:
+	$(COMPILE_C) $<
+
+#----------------------------------------------------------------------
+#List of source files
+SRC_MODULES = \
+  $(MODULES)param_m.f90             \
+	$(MODULES)chemkin_m.f90           \
+	$(MODULES)topology_m.f90          \
+	$(MODULES)work_m.f90              \
+  $(MODULES)bc_m.f90                \
+  $(MODULES)derivative_m.f90        \
+  $(MODULES)grid_m.f90              \
+  $(MODULES)reference_m.f90         \
+  $(MODULES)runtime_m.f90           \
+  $(MODULES)thermchem_m.f90         \
+  $(MODULES)variables_m.f90         \
+  $(MODULES)erk_m.f90               \
+  $(MODULES)mpi_io_m.f90            \
+  $(MODULES)pnetcdf_m.f90           \
+  $(MODULES)hdf5_m.f90              \
+  $(MODULES)check_io_m.f90
+
+
+SRC_DRIVERS =  \
+  $(MAIN)main.f90                   \
+  $(MAIN)solve_driver.f90           \
+  $(MAIN)post_driver.f90
+
+SRC_INIT = \
+	$(INIT)init_pressure_wave_test.f90
+
+SRC_SOLVE = \
+	$(SOLVE)calc_cfl_limit.f90       \
+	$(SOLVE)init_field.f90	         \
+	$(SOLVE)io.f90		               \
+	$(SOLVE)terminate_run.f90        
+
+SRC_FSEVEN = \
+	$(FSEVEN)ck2f90_tools.f		       \
+	$(FSEVEN)ckheadersetup.f		     \
+	$(FSEVEN)cklib.f			           \
+	$(FSEVEN)getrates.f		           \
+
+#----------------------------------------------------------------------
+# Add mkdir.c routine for machines on which SYSTEMCALLWONTWORK
+ifeq ($(MACH), $(filter $(MACH), JETPTH JET CNL))
+  SRC_SOLVE += $(SOLVE)mkdir.c
+endif 
+
+#----------------------------------------------------------------------
+# Choose the transport module to use
+#ifndef $(TRANSPORT)
+#  TRANSPORT = MIXAVG
+#endif
+#
+#ifeq ($(TRANSPORT), LEWIS)
+#  SRC_MODULES += $(MODULES)lewis_transport_m.f90
+#else 
+#  ifeq ($(TRANSPORT), MIXAVG)
+#    SRC_MODULES += $(MODULES)mixavg_transport_m.f90
+#  else
+#    #Unknown option for INTMETHOD - error
+#    MYERRMSG += Unknown option $(TRANSPORT) for TRANSPORT.
+#  endif
+#endif
+
+#----------------------------------------------------------------------
+# Choose the integration method
+
+#SRC_ARK = \
+#  $(MODULES)ark_m.f90              \
+#  $(FSEVEN)g_getrates.f            \
+#  $(FSEVEN)g_cklib.f               \
+#  $(FSEVEN)eshfid.f 
+#
+#SRC_ERK = \
+#  $(MODULES)erk_m.f90              \
+#
+#ifndef $(INTMETHOD)
+#  INTMETHOD = ERK
+#endif
+#
+#ifeq ($(INTMETHOD), ARK)
+#  SRC_INT = $(SRC_ARK)
+#else
+#  ifeq ($(INTMETHOD), ERK)
+#    SRC_INT = $(SRC_ERK)
+#  else
+#    #Unknown option for INTMETHOD - error
+#		MYERRMSG += Unknown option $(INTMETHOD) for INTMETHOD.
+#  endif
+#endif
+
+#----------------------------------------------------------------------
+# Choose the math library to use
+# Whether to use the math.f source file or the system lapack library
+
+ifndef ($(MATHLIB))
+  MATHLIB = FILE
+endif
+
+ifeq ($(MATHLIB), FILE)
+  SRC_FSEVEN += $(FSEVEN)math.f
+else
+  ifeq ($(MATHLIB), SYSMATHLIB)
+    LIBS += $(SYSMATHLIB)
+  else
+    #Unknown option for MATHLIB - error
+		MYERRMSG += Unknown option $(MATHLIB) for MATHLIB.
+  endif
+endif
+
+#----------------------------------------------------------------------
+# Sources for the main program
+
+SRC_S3D = \
+  $(SRC_MODULES) $(SRC_DRIVERS) $(SRC_INT) \
+  $(SRC_INIT) $(SRC_SOLVE) \
+  $(SRC_BC) $(SRC_FSEVEN) 
+
+
+# Extract the list of object files
+OBJ_S3D2 = $(SRC_S3D:.f90=.o)
+#One more pass for converting the .f extensions of f77 files
+#Is there a smarter way?
+OBJ_S3D1 = $(OBJ_S3D2:.f=.o)
+OBJ_S3D  = $(OBJ_S3D1:.c=.o)
+
+#----------------------------------------------------------------------
+# TARGETS:
+
+S3D = ../run/s3d_f90.x
+
+morph = ../run/morph.x
+
+# build the s3d_f90 executable by default.
+
+default: errchk $(S3D)
+
+errchk:
+ifneq ($(MYERRMSG), )
+	$(error Error:!! $(MYERRMSG))
+endif
+
+none: 
+	echo "Doing nothing"
+
+$(S3D): $(OBJ_S3D)
+	$(LINK) $(OBJ_S3D) -o $(S3D) $(LIBS) $(IO_LIB)
+
+morph: $(MORPH)morph.o
+	$(LINK) $(MORPH)morph.o $(LIBS) -o ../run/morph.x
+
+clean: 
+	find .. -name "*.mod" -exec rm {} \; ; find .. -name "*.o" -exec rm {} \;
+
+new: clean default
+
+depend: $(SRC_S3D)
+	$(MAKEDEPEND) $(DIRECTIVES) -I $(FSEVEN_HDR) $(SRC_S3D)
+
+
+#----------------------------------------------------------------------
+#Auto create the dependencies using make depend
+# Do not add anything below the following line
+# They will be removed when you do 'make depend'
+# DO NOT DELETE THIS LINE - used by make depend
+bc_m.xyz = ../source/modules/bc_m.o
+chemkin_m.xyz = ../source/modules/chemkin_m.o
+derivative_m.xyz = ../source/modules/derivative_m.o
+grid_m.xyz = ../source/modules/grid_m.o
+param_m.xyz = ../source/modules/param_m.o
+reference_m.xyz = ../source/modules/reference_m.o
+runtime_m.xyz = ../source/modules/runtime_m.o
+thermchem_m.xyz = ../source/modules/thermchem_m.o
+topology_m.xyz = ../source/modules/topology_m.o
+variables_m.xyz = ../source/modules/variables_m.o
+work_m.xyz = ../source/modules/work_m.o
+
+
+
+
+../source/drivers/main.o: \
+         $(chemkin_m.xyz) $(param_m.xyz) $(runtime_m.xyz) $(topology_m.xyz)
+
+../source/drivers/post_driver.o: \
+         $(chemkin_m.xyz) $(derivative_m.xyz) $(grid_m.xyz) $(param_m.xyz) \
+         $(reference_m.xyz) $(runtime_m.xyz) $(thermchem_m.xyz) \
+         $(topology_m.xyz) $(variables_m.xyz) $(work_m.xyz)
+
+../source/drivers/solve_driver.o: \
+         $(bc_m.xyz) $(derivative_m.xyz) $(grid_m.xyz) $(param_m.xyz) \
+         $(reference_m.xyz) $(rk_m.xyz) $(runtime_m.xyz) $(thermchem_m.xyz) \
+         $(topology_m.xyz) $(variables_m.xyz) $(work_m.xyz)
+../source/f77_files/ck2f90_tools.o: \
+         ../source/include//ckstrt.h
+
+../source/f77_files/ck2f90_tools.o: \
+        
+../source/f77_files/ckheadersetup.o: \
+         ../source/include//ckstrt.h
+
+../source/f77_files/ckheadersetup.o: \
+         $(topology_m.xyz)
+../source/f77_files/cklib.o: \
+         ../source/include//ckstrt.h
+
+../source/f77_files/cklib.o: \
+        
+
+../source/f77_files/getrates.o: \
+         $(thermchem_m.xyz)
+
+../source/f90_files/init/init_pressure_wave_test.o: \
+         $(chemkin_m.xyz) $(grid_m.xyz) $(param_m.xyz) $(reference_m.xyz) \
+         $(topology_m.xyz) $(work_m.xyz)
+
+../source/f90_files/solve/calc_cfl_limit.o: \
+         $(grid_m.xyz) $(param_m.xyz) $(topology_m.xyz) $(work_m.xyz)
+
+../source/f90_files/solve/init_field.o: \
+         $(bc_m.xyz) $(grid_m.xyz) $(param_m.xyz) $(reference_m.xyz) \
+         $(rk_m.xyz) $(runtime_m.xyz) $(thermchem_m.xyz) $(topology_m.xyz) \
+         $(variables_m.xyz)
+
+../source/f90_files/solve/io.o: \
+         $(bc_m.xyz) $(grid_m.xyz) $(param_m.xyz) $(reference_m.xyz) \
+         $(rk_m.xyz) $(runtime_m.xyz) $(thermchem_m.xyz) $(topology_m.xyz) \
+         $(variables_m.xyz)
+
+../source/f90_files/solve/terminate_run.o: \
+         $(param_m.xyz) $(topology_m.xyz)