Commit 6759f047 authored by Ron Rahaman's avatar Ron Rahaman
Browse files

Merge commit '318068f4' as '3rd_party/nek5000_parRSB'

parents 8fc1c5e1 318068f4
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS]
- Browser [e.g. chrome, safari]
- Version [e.g. 22]
**Smartphone (please complete the following information):**
- Device: [e.g. iPhone6]
- OS: [e.g. iOS8.1]
- Browser [e.g. stock browser, safari]
- Version [e.g. 22]
**Additional context**
Add any other context about the problem here.
---
name: Custom issue template
about: Describe this issue template's purpose here.
title: ''
labels: ''
assignees: ''
---
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: ''
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.
Copyright (c) 2019, UCHICAGO ARGONNE, LLC.
The UChicago Argonne, LLC as Operator of Argonne National
Laboratory holds copyright in the Software. The copyright holder
reserves all rights except those expressly granted to licensees,
and U.S. Government license rights.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the disclaimer below.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the disclaimer (as noted below)
in the documentation and/or other materials provided with the
distribution.
3. Neither the name of ANL nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
UCHICAGO ARGONNE, LLC, THE U.S. DEPARTMENT OF
ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Additional BSD Notice
---------------------
1. This notice is required to be provided under our contract with
the U.S. Department of Energy (DOE). This work was produced at
Argonne National Laboratory under Contract
No. DE-AC02-06CH11357 with the DOE.
2. Neither the United States Government nor UCHICAGO ARGONNE,
LLC nor any of their employees, makes any warranty,
express or implied, or assumes any liability or responsibility for the
accuracy, completeness, or usefulness of any information, apparatus,
product, or process disclosed, or represents that its use would not
infringe privately-owned rights.
3. Also, reference herein to any specific commercial products, process,
or services by trade name, trademark, manufacturer or otherwise does
not necessarily constitute or imply its endorsement, recommendation,
or favoring by the United States Government or UCHICAGO ARGONNE LLC.
The views and opinions of authors expressed
herein do not necessarily state or reflect those of the United States
Government or UCHICAGO ARGONNE, LLC, and shall
not be used for advertising or product endorsement purposes.
## General build parameters ##
DEBUG ?= 0
MPI ?= 1
CC ?= mpicc
CFLAGS ?= -g -O0
BLAS ?= 0
UNDERSCORE ?= 0
## Genmap algorithmic parameters ##
# ALGO = 0 (Lanczos),1 (RQI),2 (FMG)
ALGO ?= 0
RCB_PRE_STEP ?= 1
PAUL ?= 1
GRAMMIAN ?= 0
## Don't touch what follows ##
MKFILEPATH =$(abspath $(lastword $(MAKEFILE_LIST)))
SRCROOT_ ?=$(patsubst %/,%,$(dir $(MKFILEPATH)))
SRCROOT =$(realpath $(SRCROOT_))
SRCDIR =$(SRCROOT)/src
SORTDIR =$(SRCROOT)/src/sort
PRECONDDIR=$(SRCROOT)/src/precond
GENCONDIR =$(SRCROOT)/src/gencon
BUILDDIR =$(SRCROOT)/build
EXAMPLEDIR=$(SRCROOT)/examples
TESTDIR =$(SRCROOT)/tests
TARGET=parRSB
LIB=$(BUILDDIR)/lib/lib$(TARGET).a
INCFLAGS=-I$(SRCDIR) -I$(SORTDIR) -I$(PRECONDDIR) -I$(GENCONDIR) \
-I$(GSLIBPATH)/include
LDFLAGS:=-L$(BUILDDIR)/lib -l$(TARGET) -L$(GSLIBPATH)/lib -lgs -lm
SRCS =$(wildcard $(SRCDIR)/*.c)
SORTSRCS =$(wildcard $(SORTDIR)/*.c)
PRECONDSRCS=$(wildcard $(PRECONDDIR)/*.c)
GENCONSRCS =$(wildcard $(GENCONDIR)/*.c)
TESTSRCS =$(wildcard $(TESTDIR)/*.c)
EXAMPLESRCS=$(wildcard $(EXAMPLEDIR)/*.c)
SRCOBJS =$(patsubst $(SRCROOT)/%.c,$(BUILDDIR)/%.o,$(SRCS))
SRCOBJS+=$(patsubst $(SRCROOT)/%.c,$(BUILDDIR)/%.o,$(SORTSRCS))
SRCOBJS+=$(patsubst $(SRCROOT)/%.c,$(BUILDDIR)/%.o,$(PRECONDSRCS))
SRCOBJS+=$(patsubst $(SRCROOT)/%.c,$(BUILDDIR)/%.o,$(GENCONSRCS))
TESTOBJS=$(patsubst $(SRCROOT)/%.c,$(BUILDDIR)/%,$(TESTSRCS))
EXAMPLEOBJS=$(patsubst $(SRCROOT)/%.c,$(BUILDDIR)/%,$(EXAMPLESRCS))
PP=
ifneq ($(DEBUG),0)
PP += -g -DGENMAP_DEBUG
endif
ifeq ($(ALGO),0)
PP += -DGENMAP_LANCZOS
else ifeq ($(ALGO),1)
PP += -DGENMAP_RQI
endif
ifneq ($(RCB_PRE_STEP),0)
PP += -DGENMAP_RCB_PRE_STEP
endif
ifneq ($(PAUL),0)
PP += -DGENMAP_PAUL
endif
ifneq ($(GRAMMIAN),0)
PP += -DGENMAP_GRAMMIAN
endif
ifneq ($(UNDERSCORE),0)
PP += -DGENMAP_UNDERSCORE
endif
ifneq ($(BLAS),0)
PP += -DGENMAP_BLAS
LDFLAGS += -L$(BLASLIBPATH) -lblasLapack
endif
ifneq ($(MPI),0)
PP += -DMPI
endif
INSTALLDIR=
ifneq (,$(strip $(DESTDIR)))
INSTALLDIR=$(realpath $(DESTDIR))
endif
.PHONY: default
default: check lib install
.PHONY: all
all: check lib tests examples install
.PHONY: install
install: lib
ifneq ($(INSTALLDIR),)
@mkdir -p $(INSTALLDIR)/lib 2>/dev/null
@cp -v $(LIB) $(INSTALLDIR)/lib 2>/dev/null
@mkdir -p $(INSTALLDIR)/include 2>/dev/null
@cp $(SRCDIR)/*.h $(SORTDIR)/*.h $(PRECONDDIR)/*.h \
$(INSTALLDIR)/include 2>/dev/null
endif
.PHONY: lib
lib: $(SRCOBJS)
@mkdir -p $(BUILDDIR)/lib
@$(AR) cr $(LIB) $(SRCOBJS)
@ranlib $(LIB)
.PHONY: check
check:
ifeq ($(GSLIBPATH),)
$(error Specify GSLIBPATH=<path to gslib>/build)
endif
$(BUILDDIR)/src/%.o: $(SRCROOT)/src/%.c
$(CC) $(CFLAGS) $(PP) $(INCFLAGS) -c $< -o $@
.PHONY: examples
examples: install $(EXAMPLEOBJS)
$(BUILDDIR)/examples/%: $(SRCROOT)/examples/%.c
$(CC) $(CFLAGS) $(PP) $(INCFLAGS) $< -o $@ $(LDFLAGS)
.PHONY: tests
tests: install $(TESTOBJS)
@cp $(TESTDIR)/run-tests.sh $(BUILDDIR)/tests/
@cd $(BUILDDIR)/tests && ./run-tests.sh
$(BUILDDIR)/tests/%: $(SRCROOT)/tests/%.c
$(CC) $(CFLAGS) $(PP) $(INCFLAGS) $< -o $@ $(LDFLAGS)
.PHONY: clean
clean:
@rm -rf $(BUILDDIR) $(EXAMPLE) $(EXAMPLE).o
print-%:
$(info [ variable name]: $*)
$(info [ origin]: $(origin $*))
$(info [ value]: $(value $*))
$(info [expanded value]: $($*))
$(info)
@true
$(shell mkdir -p $(BUILDDIR)/src/sort)
$(shell mkdir -p $(BUILDDIR)/src/precond)
$(shell mkdir -p $(BUILDDIR)/src/gencon)
$(shell mkdir -p $(BUILDDIR)/tests)
$(shell mkdir -p $(BUILDDIR)/examples)
# parRSB
* Computes high quality partitionings using recursive spectral bisection (RSB)
* Requires MPI and [gslib](https://github.com/gslib/gslib) (requires version 1.0.3 or later)
## Build Instruction
```sh
make CC=mpicc GSLIBPATH=<path to gslib>/build all
```
## Run Example
```sh
cd example
mpirun -np 4 ./example 2 case01.co2
```
## C Interface
```C
int parRSB_partMesh(int *part, long long *vtx, int nel, int nve, int *options, MPI_Comm comm);
```
See `example/example.c`.
### Parameters
```text
part (out) ... Paritition vector of the local elements (size = nel).
vtx (in) ... Vertices of local elements (dense unique IDs are required).
nel (in) ... Numer of local elements.
nve (in) ... Number of vertices of a single element (has to be the same for all).
opt (in) ... Additional parameters (to use defaults set opt[0] = 0).
comm (in) ... MPI Communicator (size determines number of partitions).
```
Note, any initial distribution of mesh elements is valid.
#ifndef _CONREADER_H_
#define _CONREADER_H_
#include <math.h>
struct con {
int nv;
int nelg;
int nel;
long long *el, *vl;
};
void conFree(struct con *c) {
free(c->el);
free(c->vl);
}
int conRead(const char *fname, struct con *c, MPI_Comm comm) {
int i, j, ierr;
int myid, np;
MPI_Comm_rank(comm, &myid);
MPI_Comm_size(comm, &np);
MPI_File fh;
ierr = MPI_File_open(comm, fname, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if(ierr != 0) {
if(myid == 0) printf("ERROR: cannot open con file %s!\n", fname);
return 1;
}
char hdr[132];
MPI_File_read_all(fh, hdr, 132, MPI_BYTE, MPI_STATUS_IGNORE);
char ver[6];
int nelgt, nelgv, nv;
sscanf(hdr, "%s %d %d %d", ver, &nelgt, &nelgv, &nv);
float byte_test;
MPI_File_read_all(fh, &byte_test, 4, MPI_BYTE, MPI_STATUS_IGNORE);
if(fabs(byte_test - 6.543210) > 1e-7) {
if(myid == 0) printf("ERROR byte_test failed! %f\n", byte_test);
return 1;
}
int nelr = nelgt / np;
for(i = 0; i < nelgt % np; ++i) if(np-i == myid) nelr++;
int nelr_;
MPI_Scan(&nelr, &nelr_, 1, MPI_INT, MPI_SUM, comm);
long long off = sizeof(hdr) + sizeof(int);
off += (long long)(nelr_ - nelr) * (nv + 1) * sizeof(int);
MPI_File_set_view(fh, off, MPI_BYTE, MPI_BYTE, "native", MPI_INFO_NULL);
int *buf = (int*) malloc(nelr * (nv + 1) * sizeof(int));
ierr = MPI_File_read_all(fh, buf, (nv + 1) * nelr, MPI_INT,
MPI_STATUS_IGNORE);
if(ierr != 0) {
if(myid == 0) printf("ERROR: failure while reading!\n");
return 1;
}
MPI_File_close(&fh);
c->el = (long long*) malloc(nelr * sizeof(long long));
c->vl = (long long*) malloc(nv * nelr * sizeof(long long));
for(i = 0; i < nelr; ++i) {
c->el[i] = buf[i * (nv + 1)];
for(j = 0; j < nv; ++j) c->vl[i * nv + j] = buf[i * (nv + 1) + 1 + j];
}
c->nv = nv;
c->nelg = nelgv;
c->nel = nelr;
free(buf);
return 0;
}
#endif
#include <stdio.h>
#include <stdlib.h>
#include <gencon-impl.h>
int main(int argc,char *argv[]){
MPI_Init(&argc,&argv);
struct comm comm; comm_init(&comm,MPI_COMM_WORLD);
if(argc<2){
if(comm.id==0)
printf("Usage: ./%s foo.re2 [tol]\n",argv[0]);
MPI_Finalize();
exit(1);
}
Mesh mesh;
read_geometry(&mesh,argv[1],&comm);
findMinNeighborDistance(mesh);
double tol=(argc>2)?atof(argv[2]):0.2;
findSegments(mesh,&comm,tol,0);
setGlobalID(mesh,&comm);
sendBack(mesh,&comm);
matchPeriodicFaces(mesh,&comm);
char co2FileName[BUFSIZ]; strncpy(co2FileName,argv[1],BUFSIZ);
int len=strlen(co2FileName); assert(len>4 && len<BUFSIZ);
co2FileName[len-2]='o',co2FileName[len-3]='c';
write_connectivity(mesh,co2FileName,&comm);
mesh_free(mesh);
comm_free(&comm);
MPI_Finalize();
return 0;
}
/*
Parition mesh using Nek5000's vertex connectivity (con) file.
*/
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#include <gencon.h>
#include <genmap.h>
#include <parRSB.h>
#include "quality.h"
#define MAXNV 8 /* maximum number of vertices per element */
typedef struct {
int proc;
long long vtx[MAXNV];
} elm_data;
#define EXIT_ERROR() do{\
MPI_Finalize();\
return EXIT_FAILURE;\
} while(0)
int main(int argc, char *argv[]) {
MPI_Init(&argc, &argv);
int rank,size;
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
MPI_Comm_size(MPI_COMM_WORLD,&size);
if(argc!=3){
if(rank==0)
printf("Usage: %s <#nread> <mesh file>\n",argv[0]);
EXIT_ERROR();
}
int n_read = atoi(argv[1]);
char* mesh_name = argv[2];
char geom_name[BUFSIZ];
strncpy(geom_name,mesh_name,BUFSIZ);
strncat(geom_name,".re2",BUFSIZ);
char conn_name[BUFSIZ];
strncpy(conn_name,mesh_name,BUFSIZ);
strncat(conn_name,".co2",BUFSIZ);
int color = MPI_UNDEFINED;
if(rank<n_read) color=1;
MPI_Comm comm_read;
MPI_Comm_split(MPI_COMM_WORLD, color, 0, &comm_read);
long long *vl=NULL;
double *coord=NULL;
int nelt=0,ndim,nv;
/* Read mesh data */
if(color==1){
struct comm comm; comm_init(&comm, comm_read);
Mesh mesh;
read_geometry(&mesh,geom_name,&comm);
read_connectivity(mesh,conn_name,&comm);
get_vertex_ids(&vl,mesh);
get_vertex_coordinates(&coord,mesh);
ndim = get_mesh_dim(mesh);
nelt = get_mesh_nel(mesh);
mesh_free(mesh);
comm_free(&comm);
}
MPI_Bcast(&ndim,1,MPI_INT,0,MPI_COMM_WORLD);
nv=(ndim==3)?8:4;
/* Partition the mesh */
int options[3];
options[0] = 1; /* use custom options */
options[1] = 2; /* debug level */
options[2] = 0; /* not used */
int *part=(int*)calloc(nelt,sizeof(int));
int ierr=parRSB_partMesh(part,vl,coord,nelt,nv,options,MPI_COMM_WORLD);
if(ierr){
if(vl) free(vl);
if(coord) free(coord);
if(part) free(part);
EXIT_ERROR();
}
/* Redistribute data */
struct array elements;
array_init(elm_data,&elements,nelt);
elm_data *data;
int e,n;
for(data=elements.ptr,e=0; e<nelt; ++e) {
data[e].proc=part[e];
for(n=0; n<nv; ++n) {
data[e].vtx[n] = vl[e*nv+n];
}
}
elements.n=nelt;
struct comm comm; comm_init(&comm,MPI_COMM_WORLD);
struct crystal cr; crystal_init(&cr, &comm);
sarray_transfer(elm_data,&elements,proc,0,&cr);
crystal_free(&cr);
comm_free(&comm);
nelt=elements.n;
vl=(long long*) malloc(nv*nelt*sizeof(long long));
for(data=elements.ptr, e=0; e<nelt; ++e) {
for(n=0; n<nv; ++n){
vl[e*nv+n]=data[e].vtx[n];
}
}
array_free(&elements);
printPartStat(vl, nelt, nv, MPI_COMM_WORLD);
if(part) free(part);
if(vl) free(vl);
if(coord) free(coord);
}
void printPartStat(long long *vtx, int nel, int nv, comm_ext ce) {
int i, j;
struct comm comm;
int np, id;
int Nmsg;
int *Ncomm;
int nelMin, nelMax;
int ncMin, ncMax, ncSum;
int nsMin, nsMax, nsSum;
int nssMin, nssMax, nssSum;
struct gs_data *gsh;
int b;
int numPoints;
long long *data;
comm_init(&comm, ce);
np = comm.np;
id = comm.id;
if(np == 1) return;