Merge pull request 'EnhancedSolver port complete' (#3) from AdityaUjeniya/NuSiF-Solver:main into main
Reviewed-on: #3
This commit is contained in:
commit
d6aa1cf113
@ -22,6 +22,7 @@ SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)
|
||||
ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC))
|
||||
OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC))
|
||||
OBJ += $(BUILD_DIR)/comm-$(COMM_TYPE).o
|
||||
OBJ += $(BUILD_DIR)/solver-$(SOLVER).o
|
||||
SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h)
|
||||
CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES)
|
||||
|
||||
@ -38,9 +39,20 @@ $(BUILD_DIR)/%.s: %.c
|
||||
$(info ===> GENERATE ASM $@)
|
||||
$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
|
||||
|
||||
.PHONY: clean distclean tags info asm format
|
||||
.PHONY: clean distclean vis vis_clean tags info asm format
|
||||
|
||||
clean:
|
||||
vis:
|
||||
$(info ===> GENERATE VISUALIZATION)
|
||||
@gnuplot -e "filename='pressure.dat'" ./surface.plot
|
||||
@gnuplot -e "filename='velocity.dat'" ./vector.plot
|
||||
@gnuplot -e "filename='residual.dat'" ./residual.plot
|
||||
|
||||
vis_clean:
|
||||
$(info ===> CLEAN VISUALIZATION)
|
||||
@rm -f *.dat
|
||||
@rm -f *.png
|
||||
|
||||
clean: vis_clean
|
||||
$(info ===> CLEAN)
|
||||
@rm -rf $(BUILD_DIR)
|
||||
@rm -f tags
|
||||
|
@ -7,10 +7,10 @@
|
||||
|
||||
name canal # name of flow setup
|
||||
|
||||
bcN 1 # flags for boundary conditions
|
||||
bcE 3 # 1 = no-slip 3 = outflow
|
||||
bcS 1 # 2 = free-slip 4 = periodic
|
||||
bcW 3 #
|
||||
bcTop 1 # flags for boundary conditions
|
||||
bcBottom 1 # 1 = no-slip 3 = outflow
|
||||
bcLeft 3 # 2 = free-slip 4 = periodic
|
||||
bcRight 3 #
|
||||
|
||||
gx 0.0 # Body forces (e.g. gravity)
|
||||
gy 0.0 #
|
||||
@ -27,15 +27,22 @@ p_init 0.0 # initial value for pressure
|
||||
xlength 30.0 # domain size in x-direction
|
||||
ylength 4.0 # domain size in y-direction
|
||||
imax 200 # number of interior cells in x-direction
|
||||
jmax 50 # number of interior cells in y-direction
|
||||
jmax 40 # number of interior cells in y-direction
|
||||
|
||||
# Time Data:
|
||||
# ---------
|
||||
|
||||
te 100.0 # final time
|
||||
te 60.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 2 # Multigrid levels
|
||||
presmooth 5 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
|
@ -1,7 +1,11 @@
|
||||
# Supported: GCC, CLANG, ICC
|
||||
TAG ?= CLANG
|
||||
# Supported: GCC, CLANG, ICX
|
||||
TAG ?= ICX
|
||||
# Supported: true, false
|
||||
ENABLE_MPI ?= true
|
||||
ENABLE_OPENMP ?= false
|
||||
# Supported: rb, mg
|
||||
SOLVER ?= mg
|
||||
# Supported: v1, v2, v3
|
||||
COMM_TYPE ?= v3
|
||||
|
||||
#Feature options
|
||||
|
@ -26,21 +26,28 @@ p_init 0.0 # initial value for pressure
|
||||
|
||||
xlength 1.0 # domain size in x-direction
|
||||
ylength 1.0 # domain size in y-direction
|
||||
imax 80 # number of interior cells in x-direction
|
||||
jmax 80 # number of interior cells in y-direction
|
||||
imax 128 # number of interior cells in x-direction
|
||||
jmax 128 # number of interior cells in y-direction
|
||||
|
||||
# Time Data:
|
||||
# ---------
|
||||
|
||||
te 10.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 2 # Multigrid levels
|
||||
presmooth 20 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
itermax 1000 # maximal number of pressure iteration in one time step
|
||||
eps 0.001 # stopping tolerance for pressure iteration
|
||||
omg 1.9 # relaxation parameter for SOR iteration
|
||||
omg 1.7 # relaxation parameter for SOR iteration
|
||||
gamma 0.9 # upwind differencing factor gamma
|
||||
#===============================================================================
|
||||
|
20
BasicSolver/2D-mpi/include_ICX.mk
Normal file
20
BasicSolver/2D-mpi/include_ICX.mk
Normal file
@ -0,0 +1,20 @@
|
||||
ifeq ($(ENABLE_MPI),true)
|
||||
CC = mpiicx
|
||||
DEFINES = -D_MPI
|
||||
else
|
||||
CC = icx
|
||||
endif
|
||||
|
||||
GCC = gcc
|
||||
LINKER = $(CC)
|
||||
|
||||
ifeq ($(ENABLE_OPENMP),true)
|
||||
OPENMP = -qopenmp
|
||||
endif
|
||||
|
||||
VERSION = --version
|
||||
CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) -Wno-unused-command-line-argument
|
||||
LFLAGS = $(OPENMP)
|
||||
DEFINES += -D_GNU_SOURCE# -DDEBUG
|
||||
INCLUDES =
|
||||
LIBS =
|
9
BasicSolver/2D-mpi/residual.plot
Normal file
9
BasicSolver/2D-mpi/residual.plot
Normal file
@ -0,0 +1,9 @@
|
||||
set terminal png size 1800,768 enhanced font ,12
|
||||
set output 'residual.png'
|
||||
set datafile separator whitespace
|
||||
set xlabel "Timestep"
|
||||
set ylabel "Residual"
|
||||
|
||||
set logscale y 2
|
||||
|
||||
plot 'residual.dat' using 1:2 title "Residual"
|
@ -4,23 +4,12 @@
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "comm.h"
|
||||
|
||||
#ifdef _MPI
|
||||
// subroutines local to this module
|
||||
static int sum(int* sizes, int position)
|
||||
{
|
||||
int sum = 0;
|
||||
|
||||
for (int i = 0; i < position; i++) {
|
||||
sum += sizes[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
static void gatherArray(
|
||||
Comm* c, int cnt, int* rcvCounts, int* displs, double* src, double* dst)
|
||||
{
|
||||
@ -41,6 +30,52 @@ static void gatherArray(
|
||||
MPI_COMM_WORLD);
|
||||
}
|
||||
#endif // defined _MPI
|
||||
void commCollectResult(Comm* c,
|
||||
double* ug,
|
||||
double* vg,
|
||||
double* pg,
|
||||
double* u,
|
||||
double* v,
|
||||
double* p,
|
||||
int imax,
|
||||
int jmax)
|
||||
{
|
||||
#ifdef _MPI
|
||||
int *rcvCounts, *displs;
|
||||
int cnt = c->jmaxLocal * (imax + 2);
|
||||
|
||||
if (c->rank == 0) {
|
||||
rcvCounts = (int*)malloc(c->size * sizeof(int));
|
||||
displs = (int*)malloc(c->size * sizeof(int));
|
||||
for (int i = 0; i < c->size; ++i) {
|
||||
rcvCounts[i] = 0;
|
||||
displs[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (c->rank == 0 && c->size == 1) {
|
||||
cnt = (c->jmaxLocal + 2) * (imax + 2);
|
||||
} else if (c->rank == 0 || c->rank == (c->size - 1)) {
|
||||
cnt = (c->jmaxLocal + 1) * (imax + 2);
|
||||
}
|
||||
|
||||
MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD);
|
||||
|
||||
if (c->rank == 0) {
|
||||
displs[0] = 0;
|
||||
int cursor = rcvCounts[0];
|
||||
|
||||
for (int i = 1; i < c->size; i++) {
|
||||
displs[i] = cursor;
|
||||
cursor += rcvCounts[i];
|
||||
}
|
||||
}
|
||||
|
||||
gatherArray(c, cnt, rcvCounts, displs, p, pg);
|
||||
gatherArray(c, cnt, rcvCounts, displs, u, ug);
|
||||
gatherArray(c, cnt, rcvCounts, displs, v, vg);
|
||||
#endif
|
||||
}
|
||||
|
||||
// exported subroutines
|
||||
int commIsBoundary(Comm* c, int direction)
|
||||
@ -67,6 +102,7 @@ int commIsBoundary(Comm* c, int direction)
|
||||
|
||||
void commExchange(Comm* c, double* grid)
|
||||
{
|
||||
// printf("Rank : %d In exchange \n", c->rank);
|
||||
#ifdef _MPI
|
||||
MPI_Request requests[4] = { MPI_REQUEST_NULL,
|
||||
MPI_REQUEST_NULL,
|
||||
@ -126,56 +162,65 @@ void commShift(Comm* c, double* f, double* g)
|
||||
#endif
|
||||
}
|
||||
|
||||
void commCollectResult(Comm* c,
|
||||
double* ug,
|
||||
double* vg,
|
||||
double* pg,
|
||||
double* u,
|
||||
double* v,
|
||||
double* p,
|
||||
int jmax,
|
||||
int imax)
|
||||
{
|
||||
#ifdef _MPI
|
||||
int *rcvCounts, *displs;
|
||||
int cnt = c->jmaxLocal * (imax + 2);
|
||||
|
||||
if (c->rank == 0) {
|
||||
rcvCounts = (int*)malloc(c->size * sizeof(int));
|
||||
displs = (int*)malloc(c->size * sizeof(int));
|
||||
}
|
||||
|
||||
if (c->rank == 0 && c->size == 1) {
|
||||
cnt = (c->jmaxLocal + 2) * (imax + 2);
|
||||
} else if (c->rank == 0 || c->rank == (c->size - 1)) {
|
||||
cnt = (c->jmaxLocal + 1) * (imax + 2);
|
||||
}
|
||||
|
||||
MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD);
|
||||
|
||||
if (c->rank == 0) {
|
||||
displs[0] = 0;
|
||||
int cursor = rcvCounts[0];
|
||||
|
||||
for (int i = 1; i < c->size; i++) {
|
||||
displs[i] = cursor;
|
||||
cursor += rcvCounts[i];
|
||||
}
|
||||
}
|
||||
|
||||
gatherArray(c, cnt, rcvCounts, displs, p, pg);
|
||||
gatherArray(c, cnt, rcvCounts, displs, u, ug);
|
||||
gatherArray(c, cnt, rcvCounts, displs, v, vg);
|
||||
#endif
|
||||
}
|
||||
|
||||
void commPartition(Comm* c, int jmax, int imax)
|
||||
{
|
||||
#ifdef _MPI
|
||||
c->imaxLocal = imax;
|
||||
c->jmaxLocal = sizeOfRank(c->rank, c->size, jmax);
|
||||
c->jmaxLocal = sizeOfRank(c->coords[JDIM], c->size, jmax);
|
||||
|
||||
c->neighbours[BOTTOM] = c->rank == 0 ? -1 : c->rank - 1;
|
||||
c->neighbours[TOP] = c->rank == (c->size - 1) ? -1 : c->rank + 1;
|
||||
c->neighbours[LEFT] = -1;
|
||||
c->neighbours[RIGHT] = -1;
|
||||
|
||||
c->coords[IDIM] = 0;
|
||||
c->coords[JDIM] = c->rank;
|
||||
|
||||
c->dims[IDIM] = 1;
|
||||
c->dims[JDIM] = c->size;
|
||||
#else
|
||||
c->imaxLocal = imax;
|
||||
c->jmaxLocal = jmax;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
|
||||
{
|
||||
|
||||
#if defined _MPI
|
||||
newcomm->comm = MPI_COMM_NULL;
|
||||
int result = MPI_Comm_dup(MPI_COMM_WORLD, &newcomm->comm);
|
||||
|
||||
if (result == MPI_ERR_COMM) {
|
||||
printf("\nNull communicator. Duplication failed !!\n");
|
||||
}
|
||||
|
||||
newcomm->rank = oldcomm->rank;
|
||||
newcomm->size = oldcomm->size;
|
||||
|
||||
newcomm->imaxLocal = imaxLocal / 2;
|
||||
newcomm->jmaxLocal = jmaxLocal / 2;
|
||||
|
||||
newcomm->neighbours[BOTTOM] = newcomm->rank == 0 ? -1 : newcomm->rank - 1;
|
||||
newcomm->neighbours[TOP] = newcomm->rank == (newcomm->size - 1) ? -1
|
||||
: newcomm->rank + 1;
|
||||
newcomm->neighbours[LEFT] = -1;
|
||||
newcomm->neighbours[RIGHT] = -1;
|
||||
|
||||
newcomm->coords[IDIM] = 0;
|
||||
newcomm->coords[JDIM] = newcomm->rank;
|
||||
|
||||
newcomm->dims[IDIM] = 1;
|
||||
newcomm->dims[JDIM] = newcomm->size;
|
||||
|
||||
#endif
|
||||
newcomm->imaxLocal = imaxLocal;
|
||||
newcomm->jmaxLocal = jmaxLocal;
|
||||
}
|
||||
|
||||
void commFreeCommunicator(Comm* comm)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Comm_free(&comm->comm);
|
||||
#endif
|
||||
}
|
@ -4,18 +4,18 @@
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include "comm.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "comm.h"
|
||||
#include <string.h>
|
||||
|
||||
#ifdef _MPI
|
||||
// subroutines local to this module
|
||||
static int sum(int* sizes, int position)
|
||||
static int sum(int* sizes, int init, int offset, int coord)
|
||||
{
|
||||
int sum = 0;
|
||||
|
||||
for (int i = 0; i < position; i++) {
|
||||
for (int i = init - offset; coord > 0; i -= offset, --coord) {
|
||||
sum += sizes[i];
|
||||
}
|
||||
|
||||
@ -79,8 +79,8 @@ static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax
|
||||
int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
|
||||
int coords[NDIMS];
|
||||
MPI_Cart_coords(c->comm, i, NDIMS, coords);
|
||||
int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
|
||||
sum(newSizesI, coords[IDIM]) };
|
||||
int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]),
|
||||
sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) };
|
||||
printf(
|
||||
"Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
|
||||
"Starts(i,j): %d %d\n",
|
||||
@ -252,8 +252,8 @@ void commPartition(Comm* c, int jmax, int imax)
|
||||
MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]);
|
||||
MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
|
||||
|
||||
int imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
|
||||
int jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
|
||||
int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax);
|
||||
int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax);
|
||||
|
||||
c->imaxLocal = imaxLocal;
|
||||
c->jmaxLocal = jmaxLocal;
|
||||
@ -285,3 +285,58 @@ void commPartition(Comm* c, int jmax, int imax)
|
||||
c->jmaxLocal = jmax;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
|
||||
{
|
||||
#if defined _MPI
|
||||
newcomm->comm = MPI_COMM_NULL;
|
||||
int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
|
||||
|
||||
if (result == MPI_ERR_COMM) {
|
||||
printf("\nNull communicator. Duplication failed !!\n");
|
||||
}
|
||||
|
||||
newcomm->rank = oldcomm->rank;
|
||||
newcomm->size = oldcomm->size;
|
||||
|
||||
memcpy(&newcomm->neighbours, &oldcomm->neighbours, sizeof(oldcomm->neighbours));
|
||||
memcpy(&newcomm->coords, &oldcomm->coords, sizeof(oldcomm->coords));
|
||||
memcpy(&newcomm->dims, &oldcomm->dims, sizeof(oldcomm->dims));
|
||||
|
||||
newcomm->imaxLocal = imaxLocal/2;
|
||||
newcomm->jmaxLocal = jmaxLocal/2;
|
||||
|
||||
MPI_Datatype jBufferType;
|
||||
MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
|
||||
MPI_Type_commit(&jBufferType);
|
||||
|
||||
MPI_Datatype iBufferType;
|
||||
MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
|
||||
MPI_Type_commit(&iBufferType);
|
||||
|
||||
newcomm->bufferTypes[LEFT] = iBufferType;
|
||||
newcomm->bufferTypes[RIGHT] = iBufferType;
|
||||
newcomm->bufferTypes[BOTTOM] = jBufferType;
|
||||
newcomm->bufferTypes[TOP] = jBufferType;
|
||||
|
||||
newcomm->sdispls[LEFT] = (imaxLocal + 2) + 1;
|
||||
newcomm->sdispls[RIGHT] = (imaxLocal + 2) + imaxLocal;
|
||||
newcomm->sdispls[BOTTOM] = (imaxLocal + 2) + 1;
|
||||
newcomm->sdispls[TOP] = jmaxLocal * (imaxLocal + 2) + 1;
|
||||
|
||||
newcomm->rdispls[LEFT] = (imaxLocal + 2);
|
||||
newcomm->rdispls[RIGHT] = (imaxLocal + 2) + (imaxLocal + 1);
|
||||
newcomm->rdispls[BOTTOM] = 1;
|
||||
newcomm->rdispls[TOP] = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
|
||||
#else
|
||||
newcomm->imaxLocal = imaxLocal;
|
||||
newcomm->jmaxLocal = jmaxLocal;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commFreeCommunicator(Comm* comm)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Comm_free(&comm->comm);
|
||||
#endif
|
||||
}
|
@ -11,11 +11,11 @@
|
||||
|
||||
#ifdef _MPI
|
||||
// subroutines local to this module
|
||||
static int sum(int* sizes, int position)
|
||||
static int sum(int* sizes, int init, int offset, int coord)
|
||||
{
|
||||
int sum = 0;
|
||||
|
||||
for (int i = 0; i < position; i++) {
|
||||
for (int i = init - offset; coord > 0; i -= offset, --coord) {
|
||||
sum += sizes[i];
|
||||
}
|
||||
|
||||
@ -79,8 +79,8 @@ static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax
|
||||
int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
|
||||
int coords[NDIMS];
|
||||
MPI_Cart_coords(c->comm, i, NDIMS, coords);
|
||||
int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
|
||||
sum(newSizesI, coords[IDIM]) };
|
||||
int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]),
|
||||
sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) };
|
||||
printf(
|
||||
"Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
|
||||
"Starts(i,j): %d %d\n",
|
||||
@ -139,7 +139,6 @@ void commExchange(Comm* c, double* grid)
|
||||
{
|
||||
#ifdef _MPI
|
||||
int counts[NDIRS] = { 1, 1, 1, 1 };
|
||||
|
||||
MPI_Neighbor_alltoallw(grid,
|
||||
counts,
|
||||
c->sdispls,
|
||||
@ -233,8 +232,8 @@ void commPartition(Comm* c, int jmax, int imax)
|
||||
MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]);
|
||||
MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
|
||||
|
||||
int imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
|
||||
int jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
|
||||
int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax);
|
||||
int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax);
|
||||
|
||||
c->imaxLocal = imaxLocal;
|
||||
c->jmaxLocal = jmaxLocal;
|
||||
@ -267,3 +266,55 @@ void commPartition(Comm* c, int jmax, int imax)
|
||||
c->jmaxLocal = jmax;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
|
||||
{
|
||||
#if defined _MPI
|
||||
|
||||
int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
|
||||
|
||||
if (result == MPI_ERR_COMM) {
|
||||
printf("\nNull communicator. Duplication failed !!\n");
|
||||
}
|
||||
|
||||
newcomm->rank = oldcomm->rank;
|
||||
newcomm->size = oldcomm->size;
|
||||
|
||||
|
||||
newcomm->imaxLocal = imaxLocal / 2;
|
||||
newcomm->jmaxLocal = jmaxLocal / 2;
|
||||
|
||||
MPI_Datatype jBufferType;
|
||||
MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
|
||||
MPI_Type_commit(&jBufferType);
|
||||
|
||||
MPI_Datatype iBufferType;
|
||||
MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
|
||||
MPI_Type_commit(&iBufferType);
|
||||
|
||||
newcomm->bufferTypes[LEFT] = iBufferType;
|
||||
newcomm->bufferTypes[RIGHT] = iBufferType;
|
||||
newcomm->bufferTypes[BOTTOM] = jBufferType;
|
||||
newcomm->bufferTypes[TOP] = jBufferType;
|
||||
|
||||
newcomm->sdispls[LEFT] = (imaxLocal + 2) + 1;
|
||||
newcomm->sdispls[RIGHT] = (imaxLocal + 2) + imaxLocal;
|
||||
newcomm->sdispls[BOTTOM] = (imaxLocal + 2) + 1;
|
||||
newcomm->sdispls[TOP] = jmaxLocal * (imaxLocal + 2) + 1;
|
||||
|
||||
newcomm->rdispls[LEFT] = (imaxLocal + 2);
|
||||
newcomm->rdispls[RIGHT] = (imaxLocal + 2) + (imaxLocal + 1);
|
||||
newcomm->rdispls[BOTTOM] = 1;
|
||||
newcomm->rdispls[TOP] = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
|
||||
#else
|
||||
newcomm->imaxLocal = imaxLocal;
|
||||
newcomm->jmaxLocal = jmaxLocal;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commFreeCommunicator(Comm* comm)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Comm_free(&comm->comm);
|
||||
#endif
|
||||
}
|
@ -41,6 +41,8 @@ extern void commExchange(Comm*, double*);
|
||||
extern void commShift(Comm* c, double* f, double* g);
|
||||
extern void commReduction(double* v, int op);
|
||||
extern int commIsBoundary(Comm* c, int direction);
|
||||
extern void commUpdateDatatypes(Comm*, Comm*, int, int);
|
||||
extern void commFreeCommunicator(Comm*);
|
||||
extern void commCollectResult(Comm* c,
|
||||
double* ug,
|
||||
double* vg,
|
||||
|
@ -273,17 +273,18 @@ void setSpecialBoundaryCondition(Discretization* s)
|
||||
if (commIsBoundary(&s->comm, LEFT)) {
|
||||
double ylength = s->grid.ylength;
|
||||
double dy = s->grid.dy;
|
||||
int rest = s->grid.jmax % s->comm.size;
|
||||
int yc = s->comm.rank * (s->grid.jmax / s->comm.size) +
|
||||
int rest = s->grid.jmax % s->comm.dims[JDIM];
|
||||
int yc = s->comm.rank * (s->grid.jmax / s->comm.dims[JDIM]) +
|
||||
MIN(rest, s->comm.rank);
|
||||
double ys = dy * (yc + 0.5);
|
||||
double y;
|
||||
|
||||
/* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */
|
||||
// printf("RANK %d yc: %d ys: %f\n", s->comm.rank, yc, ys);
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
y = ys + dy * (j - 0.5);
|
||||
U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -16,6 +16,24 @@
|
||||
#include "solver.h"
|
||||
#include "timing.h"
|
||||
|
||||
static FILE* initResidualWriter()
|
||||
{
|
||||
FILE* fp;
|
||||
fp = fopen("residual.dat", "w");
|
||||
|
||||
if (fp == NULL) {
|
||||
printf("Error!\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
return fp;
|
||||
}
|
||||
|
||||
static void writeResidual(FILE* fp, double ts, double res)
|
||||
{
|
||||
fprintf(fp, "%f, %f\n", ts, res);
|
||||
}
|
||||
|
||||
static void writeResults(Discretization* s)
|
||||
{
|
||||
#ifdef _MPI
|
||||
@ -26,7 +44,9 @@ static void writeResults(Discretization* s)
|
||||
double* pg = allocate(64, bytesize);
|
||||
|
||||
commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->grid.imax, s->grid.jmax);
|
||||
writeResult(s, ug, vg, pg);
|
||||
if (commIsMaster(&s->comm)) {
|
||||
writeResult(s, ug, vg, pg);
|
||||
}
|
||||
|
||||
free(ug);
|
||||
free(vg);
|
||||
@ -47,6 +67,9 @@ int main(int argc, char** argv)
|
||||
commInit(&d.comm, argc, argv);
|
||||
initParameter(&p);
|
||||
|
||||
FILE* fp;
|
||||
if (commIsMaster(&d.comm)) fp = initResidualWriter();
|
||||
|
||||
if (argc != 2) {
|
||||
printf("Usage: %s <configFile>\n", argv[0]);
|
||||
exit(EXIT_SUCCESS);
|
||||
@ -77,16 +100,21 @@ int main(int argc, char** argv)
|
||||
double tau = d.tau;
|
||||
double te = d.te;
|
||||
double t = 0.0;
|
||||
double res = 0.0;
|
||||
|
||||
timeStart = getTimeStamp();
|
||||
while (t <= te) {
|
||||
|
||||
if (tau > 0.0) computeTimestep(&d);
|
||||
setBoundaryConditions(&d);
|
||||
setSpecialBoundaryCondition(&d);
|
||||
computeFG(&d);
|
||||
computeRHS(&d);
|
||||
solve(&s, d.p, d.rhs);
|
||||
res = solve(&s, d.p, d.rhs);
|
||||
adaptUV(&d);
|
||||
|
||||
if (commIsMaster(&d.comm)) writeResidual(fp, t, res);
|
||||
|
||||
t += d.dt;
|
||||
|
||||
#ifdef VERBOSE
|
||||
@ -104,7 +132,7 @@ int main(int argc, char** argv)
|
||||
if (commIsMaster(s.comm)) {
|
||||
printf("Solution took %.2fs\n", timeStop - timeStart);
|
||||
}
|
||||
|
||||
if (commIsMaster(&d.comm)) fclose(fp);
|
||||
writeResults(&d);
|
||||
commFinalize(s.comm);
|
||||
return EXIT_SUCCESS;
|
||||
|
@ -14,13 +14,16 @@
|
||||
|
||||
void initParameter(Parameter* param)
|
||||
{
|
||||
param->xlength = 1.0;
|
||||
param->ylength = 1.0;
|
||||
param->imax = 100;
|
||||
param->jmax = 100;
|
||||
param->itermax = 1000;
|
||||
param->eps = 0.0001;
|
||||
param->omg = 1.8;
|
||||
param->xlength = 1.0;
|
||||
param->ylength = 1.0;
|
||||
param->imax = 100;
|
||||
param->jmax = 100;
|
||||
param->itermax = 1000;
|
||||
param->eps = 0.0001;
|
||||
param->omg = 1.8;
|
||||
param->levels = 5;
|
||||
param->presmooth = 5;
|
||||
param->postsmooth = 5;
|
||||
}
|
||||
|
||||
void readParameter(Parameter* param, const char* filename)
|
||||
@ -72,6 +75,9 @@ void readParameter(Parameter* param, const char* filename)
|
||||
PARSE_INT(bcRight);
|
||||
PARSE_INT(bcBottom);
|
||||
PARSE_INT(bcTop);
|
||||
PARSE_INT(levels);
|
||||
PARSE_INT(presmooth);
|
||||
PARSE_INT(postsmooth);
|
||||
PARSE_REAL(u_init);
|
||||
PARSE_REAL(v_init);
|
||||
PARSE_REAL(p_init);
|
||||
|
@ -18,6 +18,7 @@ typedef struct {
|
||||
char* name;
|
||||
int bcLeft, bcRight, bcBottom, bcTop;
|
||||
double u_init, v_init, p_init;
|
||||
int levels, presmooth, postsmooth;
|
||||
} Parameter;
|
||||
|
||||
void initParameter(Parameter*);
|
||||
|
@ -8,7 +8,7 @@
|
||||
#include <mpi.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <stdlib.h>
|
||||
#include "progress.h"
|
||||
|
||||
static double _end;
|
||||
|
@ -10,5 +10,4 @@
|
||||
extern void initProgress(double);
|
||||
extern void printProgress(double);
|
||||
extern void stopProgress();
|
||||
|
||||
#endif
|
||||
|
303
BasicSolver/2D-mpi/src/solver-mg.c
Normal file
303
BasicSolver/2D-mpi/src/solver-mg.c
Normal file
@ -0,0 +1,303 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "allocate.h"
|
||||
#include "solver.h"
|
||||
#include "util.h"
|
||||
|
||||
#define FINEST_LEVEL 0
|
||||
#define COARSEST_LEVEL (s->levels - 1)
|
||||
// #define S(i, j) s[(j) * (imaxLocal + 2) + (i)]
|
||||
#define E(i, j) e[(j) * (imaxLocal + 2) + (i)]
|
||||
#define R(i, j) r[(j) * (imaxLocal + 2) + (i)]
|
||||
#define OLD(i, j) old[(j) * (imaxLocal + 2) + (i)]
|
||||
|
||||
static void restrictMG(Solver* s, int level, Comm* comm)
|
||||
{
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
|
||||
double* r = s->r[level + 1];
|
||||
double* old = s->r[level];
|
||||
|
||||
#ifdef _MPI
|
||||
commExchange(comm, old);
|
||||
#endif
|
||||
|
||||
for (int j = 1; j < (jmaxLocal / 2) + 1; j++) {
|
||||
for (int i = 1; i < (imaxLocal / 2) + 1; i++) {
|
||||
R(i, j) = (OLD(2 * i - 1, 2 * j - 1) + OLD(2 * i, 2 * j - 1) * 2 +
|
||||
OLD(2 * i + 1, 2 * j - 1) + OLD(2 * i - 1, 2 * j) * 2 +
|
||||
OLD(2 * i, 2 * j) * 4 + OLD(2 * i + 1, 2 * j) * 2 +
|
||||
OLD(2 * i - 1, 2 * j + 1) + OLD(2 * i, 2 * j + 1) * 2 +
|
||||
OLD(2 * i + 1, 2 * j + 1)) /
|
||||
16.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void prolongate(Solver* s, int level, Comm* comm)
|
||||
{
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
|
||||
double* old = s->r[level + 1];
|
||||
double* e = s->r[level];
|
||||
|
||||
for (int j = 2; j < jmaxLocal + 1; j += 2) {
|
||||
for (int i = 2; i < imaxLocal + 1; i += 2) {
|
||||
E(i, j) = OLD(i / 2, j / 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void correct(Solver* s, double* p, int level, Comm* comm)
|
||||
{
|
||||
double* e = s->e[level];
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; ++j) {
|
||||
for (int i = 1; i < imaxLocal + 1; ++i) {
|
||||
P(i, j) += E(i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void setBoundaryCondition(Solver* s, double* p, int imaxLocal, int jmaxLocal)
|
||||
{
|
||||
#ifdef _MPI
|
||||
if (commIsBoundary(s->comm, BOTTOM)) { // set bottom bc
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, 0) = P(i, 1);
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, TOP)) { // set top bc
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, jmaxLocal + 1) = P(i, jmaxLocal);
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, LEFT)) { // set left bc
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
P(0, j) = P(1, j);
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, RIGHT)) { // set right bc
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
P(imaxLocal + 1, j) = P(imaxLocal, j);
|
||||
}
|
||||
}
|
||||
#else
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, 0) = P(i, 1);
|
||||
P(i, jmaxLocal + 1) = P(i, jmaxLocal);
|
||||
}
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
P(0, j) = P(1, j);
|
||||
P(imaxLocal + 1, j) = P(imaxLocal, j);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static void smooth(Solver* s, double* p, double* rhs, int level, Comm* comm)
|
||||
{
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
|
||||
int imax = s->grid->imax;
|
||||
int jmax = s->grid->jmax;
|
||||
|
||||
double dx2 = s->grid->dx * s->grid->dx;
|
||||
double dy2 = s->grid->dy * s->grid->dy;
|
||||
double idx2 = 1.0 / dx2;
|
||||
double idy2 = 1.0 / dy2;
|
||||
|
||||
double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2);
|
||||
double* r = s->r[level];
|
||||
|
||||
double res = 1.0;
|
||||
int pass, jsw, isw;
|
||||
|
||||
jsw = 1;
|
||||
|
||||
for (pass = 0; pass < 2; pass++) {
|
||||
isw = jsw;
|
||||
|
||||
#ifdef _MPI
|
||||
commExchange(comm, p);
|
||||
#endif
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = isw; i < imaxLocal + 1; i += 2) {
|
||||
|
||||
P(i, j) -= factor *
|
||||
(RHS(i, j) -
|
||||
((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
|
||||
(P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2));
|
||||
}
|
||||
isw = 3 - isw;
|
||||
}
|
||||
jsw = 3 - jsw;
|
||||
}
|
||||
}
|
||||
|
||||
static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm)
|
||||
{
|
||||
int imax = s->grid->imax;
|
||||
int jmax = s->grid->jmax;
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
|
||||
double dx2 = s->grid->dx * s->grid->dx;
|
||||
double dy2 = s->grid->dy * s->grid->dy;
|
||||
double idx2 = 1.0 / dx2;
|
||||
double idy2 = 1.0 / dy2;
|
||||
double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2);
|
||||
double* r = s->r[level];
|
||||
double res = 1.0;
|
||||
int pass, jsw, isw;
|
||||
|
||||
jsw = 1;
|
||||
|
||||
for (pass = 0; pass < 2; pass++) {
|
||||
isw = jsw;
|
||||
|
||||
#ifdef _MPI
|
||||
commExchange(comm, p);
|
||||
#endif
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = isw; i < imaxLocal + 1; i += 2) {
|
||||
|
||||
R(i, j) = RHS(i, j) -
|
||||
((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
|
||||
(P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2);
|
||||
|
||||
res += (R(i, j) * R(i, j));
|
||||
}
|
||||
isw = 3 - isw;
|
||||
}
|
||||
jsw = 3 - jsw;
|
||||
}
|
||||
|
||||
#ifdef _MPI
|
||||
commReduction(&res, SUM);
|
||||
#endif
|
||||
|
||||
res = res / (double)(imax * jmax);
|
||||
#ifdef DEBUG
|
||||
if (commIsMaster(s->comm)) {
|
||||
printf("%d Residuum: %e\n", it, res);
|
||||
}
|
||||
#endif
|
||||
return res;
|
||||
}
|
||||
|
||||
static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm)
|
||||
{
|
||||
double res = 0.0;
|
||||
|
||||
// coarsest level
|
||||
if (level == COARSEST_LEVEL) {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
smooth(s, p, rhs, level, comm);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
// pre-smoothing
|
||||
for (int i = 0; i < s->presmooth; i++) {
|
||||
smooth(s, p, rhs, level, comm);
|
||||
if (level == FINEST_LEVEL)
|
||||
setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal);
|
||||
}
|
||||
|
||||
// calculate residuals
|
||||
res = calculateResidual(s, p, rhs, level, comm);
|
||||
|
||||
// restrict
|
||||
restrictMG(s, level, comm);
|
||||
|
||||
// Create a new comm object withupdated imaxLocal and jmaxLocal
|
||||
// along with their updated bufferTypes, sdispls, rdispls
|
||||
Comm newcomm;
|
||||
commUpdateDatatypes(s->comm, &newcomm, comm->imaxLocal, comm->jmaxLocal);
|
||||
|
||||
// MGSolver on residual and error.
|
||||
multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm);
|
||||
|
||||
commFreeCommunicator(&newcomm);
|
||||
|
||||
// prolongate
|
||||
prolongate(s, level, comm);
|
||||
|
||||
// correct p on finer level using residual
|
||||
correct(s, p, level, comm);
|
||||
|
||||
if (level == FINEST_LEVEL)
|
||||
setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal);
|
||||
|
||||
// post-smoothing
|
||||
for (int i = 0; i < s->postsmooth; i++) {
|
||||
smooth(s, p, rhs, level, comm);
|
||||
if (level == FINEST_LEVEL)
|
||||
setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
{
|
||||
s->eps = p->eps;
|
||||
s->omega = p->omg;
|
||||
s->itermax = p->itermax;
|
||||
s->levels = p->levels;
|
||||
s->grid = &d->grid;
|
||||
s->comm = &d->comm;
|
||||
s->presmooth = p->presmooth;
|
||||
s->postsmooth = p->postsmooth;
|
||||
|
||||
int imax = s->grid->imax;
|
||||
int jmax = s->grid->jmax;
|
||||
int levels = s->levels;
|
||||
printf("Using Multigrid solver with %d levels\n", levels);
|
||||
|
||||
s->r = malloc(levels * sizeof(double*));
|
||||
s->e = malloc(levels * sizeof(double*));
|
||||
|
||||
size_t size = (imax + 2) * (jmax + 2) * sizeof(double);
|
||||
|
||||
for (int j = 0; j < levels; j++) {
|
||||
s->r[j] = allocate(64, size);
|
||||
s->e[j] = allocate(64, size);
|
||||
|
||||
for (int i = 0; i < (imax + 2) * (jmax + 2); i++) {
|
||||
s->r[j][i] = 0.0;
|
||||
s->e[j][i] = 0.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
double solve(Solver* s, double* p, double* rhs)
|
||||
{
|
||||
double res = multiGrid(s, p, rhs, 0, s->comm);
|
||||
|
||||
#ifdef VERBOSE
|
||||
if (commIsMaster(s->comm)) {
|
||||
printf("Residuum: %.6f\n", res);
|
||||
}
|
||||
#endif
|
||||
|
||||
return res;
|
||||
}
|
@ -23,7 +23,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
s->comm = &d->comm;
|
||||
}
|
||||
|
||||
void solve(Solver* s, double* p, double* rhs)
|
||||
double solve(Solver* s, double* p, double* rhs)
|
||||
{
|
||||
int imax = s->grid->imax;
|
||||
int jmax = s->grid->jmax;
|
||||
@ -101,4 +101,6 @@ void solve(Solver* s, double* p, double* rhs)
|
||||
printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
|
||||
}
|
||||
#endif
|
||||
|
||||
return res;
|
||||
}
|
@ -9,6 +9,7 @@
|
||||
#include "comm.h"
|
||||
#include "discretization.h"
|
||||
#include "grid.h"
|
||||
#include "mpi.h"
|
||||
#include "parameter.h"
|
||||
|
||||
typedef struct {
|
||||
@ -17,10 +18,12 @@ typedef struct {
|
||||
/* parameters */
|
||||
double eps, omega;
|
||||
int itermax;
|
||||
int levels, presmooth, postsmooth;
|
||||
double **r, **e;
|
||||
/* communication */
|
||||
Comm* comm;
|
||||
} Solver;
|
||||
|
||||
void initSolver(Solver*, Discretization*, Parameter*);
|
||||
void solve(Solver*, double*, double*);
|
||||
double solve(Solver*, double*, double*);
|
||||
#endif
|
||||
|
@ -38,9 +38,22 @@ $(BUILD_DIR)/%.s: %.c
|
||||
$(info ===> GENERATE ASM $@)
|
||||
$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
|
||||
|
||||
.PHONY: clean distclean tags info asm format
|
||||
.PHONY: clean distclean vis vis_clean tags info asm format
|
||||
|
||||
clean:
|
||||
vis:
|
||||
$(info ===> GENERATE VISUALIZATION)
|
||||
@gnuplot -e "filename='pressure.dat'" ./surface.plot
|
||||
@gnuplot -e "filename='velocity.dat'" ./vector.plot
|
||||
@gnuplot -e "filename='residual.dat'" ./residual.plot
|
||||
|
||||
vis_clean:
|
||||
$(info ===> CLEAN VISUALIZATION)
|
||||
@rm -f *.dat
|
||||
@rm -f *.png
|
||||
@rm -f ./vis_files/*.dat
|
||||
@rm -f ./vis_files/*.gif
|
||||
|
||||
clean: vis_clean
|
||||
$(info ===> CLEAN)
|
||||
@rm -rf $(BUILD_DIR)
|
||||
@rm -f tags
|
||||
|
@ -36,6 +36,13 @@ te 100.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 3 # Multigrid levels
|
||||
presmooth 5 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
|
@ -1,12 +1,12 @@
|
||||
# Supported: GCC, CLANG, ICC
|
||||
TAG ?= CLANG
|
||||
TAG ?= ICC
|
||||
ENABLE_OPENMP ?= false
|
||||
# Supported: sor, rb, mg
|
||||
SOLVER ?= rb
|
||||
SOLVER ?= mg
|
||||
# Run in debug settings
|
||||
DEBUG ?= false
|
||||
|
||||
#Feature options
|
||||
OPTIONS += -DARRAY_ALIGNMENT=64
|
||||
#OPTIONS += -DVERBOSE
|
||||
OPTIONS += -DVERBOSE
|
||||
#OPTIONS += -DDEBUG
|
||||
|
@ -36,6 +36,13 @@ te 10.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 2 # Multigrid levels
|
||||
presmooth 20 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Solver Data:
|
||||
# -----------------------
|
||||
|
||||
@ -44,5 +51,4 @@ eps 0.001 # stopping tolerance for pressure iteration
|
||||
rho 0.5
|
||||
omg 1.7 # relaxation parameter for SOR iteration
|
||||
gamma 0.9 # upwind differencing factor gamma
|
||||
levels 5 # Multigrid levels
|
||||
#===============================================================================
|
||||
|
9
BasicSolver/2D-seq/residual.plot
Normal file
9
BasicSolver/2D-seq/residual.plot
Normal file
@ -0,0 +1,9 @@
|
||||
set terminal png size 1800,768 enhanced font ,12
|
||||
set output 'residual.png'
|
||||
set datafile separator whitespace
|
||||
set xlabel "Timestep"
|
||||
set ylabel "Residual"
|
||||
|
||||
set logscale y 2
|
||||
|
||||
plot 'residual.dat' using 1:2 title "Residual"
|
@ -20,8 +20,10 @@ int main(int argc, char** argv)
|
||||
Parameter p;
|
||||
Discretization d;
|
||||
Solver s;
|
||||
|
||||
|
||||
initParameter(&p);
|
||||
FILE* fp;
|
||||
fp = initResidualWriter();
|
||||
|
||||
if (argc != 2) {
|
||||
printf("Usage: %s <configFile>\n", argv[0]);
|
||||
@ -32,7 +34,7 @@ int main(int argc, char** argv)
|
||||
printParameter(&p);
|
||||
initDiscretization(&d, &p);
|
||||
initSolver(&s, &d, &p);
|
||||
|
||||
|
||||
#ifndef VERBOSE
|
||||
initProgress(d.te);
|
||||
#endif
|
||||
@ -41,8 +43,9 @@ int main(int argc, char** argv)
|
||||
double te = d.te;
|
||||
double t = 0.0;
|
||||
int nt = 0;
|
||||
double res = 0.0;
|
||||
|
||||
timeStart = getTimeStamp();
|
||||
timeStart = getTimeStamp();
|
||||
while (t <= te) {
|
||||
if (tau > 0.0) computeTimestep(&d);
|
||||
setBoundaryConditions(&d);
|
||||
@ -50,17 +53,21 @@ int main(int argc, char** argv)
|
||||
computeFG(&d);
|
||||
computeRHS(&d);
|
||||
if (nt % 100 == 0) normalizePressure(&d);
|
||||
solve(&s, d.p, d.rhs);
|
||||
res = solve(&s, d.p, d.rhs);
|
||||
adaptUV(&d);
|
||||
|
||||
writeResidual(fp, t, res);
|
||||
|
||||
t += d.dt;
|
||||
nt++;
|
||||
|
||||
#ifdef VERBOSE
|
||||
printf("TIME %f , TIMESTEP %f\n", t, solver.dt);
|
||||
printf("TIME %f , TIMESTEP %f\n", t, d.dt);
|
||||
#else
|
||||
printProgress(t);
|
||||
#endif
|
||||
}
|
||||
fclose(fp);
|
||||
timeStop = getTimeStamp();
|
||||
stopProgress();
|
||||
printf("Solution took %.2fs\n", timeStop - timeStart);
|
||||
|
@ -24,6 +24,8 @@ void initParameter(Parameter* param)
|
||||
param->gamma = 0.9;
|
||||
param->tau = 0.5;
|
||||
param->levels = 5;
|
||||
param->presmooth = 5;
|
||||
param->postsmooth = 5;
|
||||
}
|
||||
|
||||
void readParameter(Parameter* param, const char* filename)
|
||||
@ -79,6 +81,8 @@ void readParameter(Parameter* param, const char* filename)
|
||||
PARSE_REAL(u_init);
|
||||
PARSE_REAL(v_init);
|
||||
PARSE_REAL(p_init);
|
||||
PARSE_INT(presmooth);
|
||||
PARSE_INT(postsmooth);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -18,6 +18,7 @@ typedef struct {
|
||||
char* name;
|
||||
int bcLeft, bcRight, bcBottom, bcTop;
|
||||
double u_init, v_init, p_init;
|
||||
int presmooth, postsmooth;
|
||||
} Parameter;
|
||||
|
||||
void initParameter(Parameter*);
|
||||
|
@ -49,3 +49,22 @@ void stopProgress()
|
||||
printf("\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
FILE* initResidualWriter()
|
||||
{
|
||||
FILE* fp;
|
||||
fp = fopen("residual.dat", "w");
|
||||
|
||||
if (fp == NULL) {
|
||||
printf("Error!\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
return fp;
|
||||
|
||||
}
|
||||
|
||||
void writeResidual(FILE* fp, double ts, double res)
|
||||
{
|
||||
fprintf(fp, "%f, %f\n", ts, res);
|
||||
}
|
||||
|
@ -10,5 +10,7 @@
|
||||
extern void initProgress(double);
|
||||
extern void printProgress(double);
|
||||
extern void stopProgress(void);
|
||||
extern FILE* initResidualWriter(void);
|
||||
extern void writeResidual(FILE*, double, double);
|
||||
|
||||
#endif
|
||||
|
@ -84,6 +84,37 @@ static double smooth(Solver* s, double* p, double* rhs, int level, int imax, int
|
||||
|
||||
jsw = 1;
|
||||
|
||||
for (pass = 0; pass < 2; pass++) {
|
||||
isw = jsw;
|
||||
|
||||
for (int j = 1; j < jmax + 1; j++) {
|
||||
for (int i = isw; i < imax + 1; i += 2) {
|
||||
|
||||
P(i, j) -= factor * (RHS(i, j) -
|
||||
((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
|
||||
(P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2));
|
||||
|
||||
}
|
||||
isw = 3 - isw;
|
||||
}
|
||||
jsw = 3 - jsw;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static double calculateResidual(Solver* s, double* p, double* rhs, int level, int imax, int jmax)
|
||||
{
|
||||
double dx2 = s->grid->dx * s->grid->dx;
|
||||
double dy2 = s->grid->dy * s->grid->dy;
|
||||
double idx2 = 1.0 / dx2;
|
||||
double idy2 = 1.0 / dy2;
|
||||
double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2);
|
||||
double* r = s->r[level];
|
||||
double res = 1.0;
|
||||
int pass, jsw, isw;
|
||||
|
||||
jsw = 1;
|
||||
|
||||
for (pass = 0; pass < 2; pass++) {
|
||||
isw = jsw;
|
||||
|
||||
@ -94,7 +125,6 @@ static double smooth(Solver* s, double* p, double* rhs, int level, int imax, int
|
||||
((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
|
||||
(P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2);
|
||||
|
||||
P(i, j) -= (factor * R(i, j));
|
||||
res += (R(i, j) * R(i, j));
|
||||
}
|
||||
isw = 3 - isw;
|
||||
@ -110,7 +140,7 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax,
|
||||
{
|
||||
double res = 0.0;
|
||||
|
||||
// coarsest level TODO: Use direct solver?
|
||||
// coarsest level
|
||||
if (level == COARSEST_LEVEL) {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
smooth(s, p, rhs, level, imax, jmax);
|
||||
@ -118,17 +148,18 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax,
|
||||
return res;
|
||||
}
|
||||
|
||||
// pre-smoothing TODO: Make smoothing steps configurable?
|
||||
for (int i = 0; i < 5; i++) {
|
||||
// pre-smoothing
|
||||
for (int i = 0; i < s->presmooth; i++) {
|
||||
smooth(s, p, rhs, level, imax, jmax);
|
||||
if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax);
|
||||
}
|
||||
|
||||
res = calculateResidual(s, p, rhs, level, imax, jmax);
|
||||
|
||||
// restrict
|
||||
restrictMG(s, level, imax, jmax);
|
||||
|
||||
// MGSolver on residual and error.
|
||||
// TODO: What if there is a rest?
|
||||
multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, imax / 2, jmax / 2);
|
||||
|
||||
// prolongate
|
||||
@ -139,8 +170,8 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax,
|
||||
if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax);
|
||||
|
||||
// post-smoothing
|
||||
for (int i = 0; i < 5; i++) {
|
||||
res = smooth(s, p, rhs, level, imax, jmax);
|
||||
for (int i = 0; i < s->postsmooth; i++) {
|
||||
smooth(s, p, rhs, level, imax, jmax);
|
||||
if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax);
|
||||
}
|
||||
|
||||
@ -154,6 +185,8 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
s->itermax = p->itermax;
|
||||
s->levels = p->levels;
|
||||
s->grid = &d->grid;
|
||||
s->presmooth = p->presmooth;
|
||||
s->postsmooth = p->postsmooth;
|
||||
|
||||
int imax = s->grid->imax;
|
||||
int jmax = s->grid->jmax;
|
||||
@ -176,11 +209,13 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
}
|
||||
}
|
||||
|
||||
void solve(Solver* s, double* p, double* rhs)
|
||||
double solve(Solver* s, double* p, double* rhs)
|
||||
{
|
||||
double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax);
|
||||
|
||||
#ifdef VERBOSE
|
||||
printf("Residuum: %.6f\n", res);
|
||||
#endif
|
||||
|
||||
return res;
|
||||
}
|
||||
|
@ -15,7 +15,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
s->omega = p->omg;
|
||||
}
|
||||
|
||||
void solve(Solver* solver, double* p, double* rhs)
|
||||
double solve(Solver* solver, double* p, double* rhs)
|
||||
{
|
||||
int imax = solver->grid->imax;
|
||||
int jmax = solver->grid->jmax;
|
||||
@ -73,4 +73,6 @@ void solve(Solver* solver, double* p, double* rhs)
|
||||
#ifdef VERBOSE
|
||||
printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
|
||||
#endif
|
||||
|
||||
return res;
|
||||
}
|
||||
|
@ -15,7 +15,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
s->omega = p->omg;
|
||||
}
|
||||
|
||||
void solve(Solver* solver, double* p, double* rhs)
|
||||
double solve(Solver* solver, double* p, double* rhs)
|
||||
{
|
||||
int imax = solver->grid->imax;
|
||||
int jmax = solver->grid->jmax;
|
||||
@ -65,4 +65,6 @@ void solve(Solver* solver, double* p, double* rhs)
|
||||
#ifdef VERBOSE
|
||||
printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
|
||||
#endif
|
||||
|
||||
return res;
|
||||
}
|
||||
|
@ -18,9 +18,10 @@ typedef struct {
|
||||
int itermax;
|
||||
int levels;
|
||||
double **r, **e;
|
||||
int presmooth, postsmooth;
|
||||
} Solver;
|
||||
|
||||
extern void initSolver(Solver*, Discretization*, Parameter*);
|
||||
extern void solve(Solver*, double*, double*);
|
||||
extern double solve(Solver*, double*, double*);
|
||||
|
||||
#endif
|
||||
|
@ -22,6 +22,7 @@ SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)
|
||||
ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC))
|
||||
OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC))
|
||||
OBJ += $(BUILD_DIR)/vtkWriter-$(VTK_OUTPUT_FMT).o
|
||||
OBJ += $(BUILD_DIR)/solver-$(SOLVER).o
|
||||
SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h)
|
||||
ifeq ($(VTK_OUTPUT_FMT),mpi)
|
||||
DEFINES += -D_VTK_WRITER_MPI
|
||||
@ -42,9 +43,19 @@ $(BUILD_DIR)/%.s: %.c
|
||||
$(info ===> GENERATE ASM $@)
|
||||
$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
|
||||
|
||||
.PHONY: clean distclean tags info asm format
|
||||
.PHONY: clean distclean vis vis_clean tags info asm format
|
||||
|
||||
clean:
|
||||
vis:
|
||||
$(info ===> GENERATE VISUALIZATION)
|
||||
@gnuplot -e "filename='residual.dat'" ./residual.plot
|
||||
|
||||
vis_clean:
|
||||
$(info ===> CLEAN VISUALIZATION)
|
||||
@rm -f *.dat
|
||||
@rm -f *.vtk
|
||||
@rm -f *.png
|
||||
|
||||
clean: vis_clean
|
||||
$(info ===> CLEAN)
|
||||
@rm -rf $(BUILD_DIR)
|
||||
@rm -f tags
|
||||
|
@ -32,21 +32,28 @@ xlength 30.0 # domain size in x-direction
|
||||
ylength 4.0 # domain size in y-direction
|
||||
zlength 4.0 # domain size in z-direction
|
||||
imax 200 # number of interior cells in x-direction
|
||||
jmax 50 # number of interior cells in y-direction
|
||||
kmax 50 # number of interior cells in z-direction
|
||||
jmax 40 # number of interior cells in y-direction
|
||||
kmax 40 # number of interior cells in z-direction
|
||||
|
||||
# Time Data:
|
||||
# ---------
|
||||
|
||||
te 100.0 # final time
|
||||
te 60.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 2 # Multigrid levels
|
||||
presmooth 20 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
itermax 500 # maximal number of pressure iteration in one time step
|
||||
eps 0.0001 # stopping tolerance for pressure iteration
|
||||
omg 1.3 # relaxation parameter for SOR iteration
|
||||
omg 1.7 # relaxation parameter for SOR iteration
|
||||
gamma 0.9 # upwind differencing factor gamma
|
||||
#===============================================================================
|
||||
|
@ -1,7 +1,10 @@
|
||||
# Supported: GCC, CLANG, ICC
|
||||
TAG ?= CLANG
|
||||
TAG ?= ICC
|
||||
# Supported: true, false
|
||||
ENABLE_MPI ?= true
|
||||
ENABLE_OPENMP ?= false
|
||||
# Supported: rb, mg
|
||||
SOLVER ?= mg
|
||||
# Supported: seq, mpi
|
||||
VTK_OUTPUT_FMT ?= seq
|
||||
|
||||
|
@ -18,7 +18,7 @@ gx 0.0 # Body forces (e.g. gravity)
|
||||
gy 0.0 #
|
||||
gz 0.0 #
|
||||
|
||||
re 1000.0 # Reynolds number
|
||||
re 1000.0 # Reynolds number
|
||||
|
||||
u_init 0.0 # initial value for velocity in x-direction
|
||||
v_init 0.0 # initial value for velocity in y-direction
|
||||
@ -42,6 +42,13 @@ te 10.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 3 # Multigrid levels
|
||||
presmooth 20 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
|
9
BasicSolver/3D-mpi/residual.plot
Normal file
9
BasicSolver/3D-mpi/residual.plot
Normal file
@ -0,0 +1,9 @@
|
||||
set terminal png size 1800,768 enhanced font ,12
|
||||
set output 'residual.png'
|
||||
set datafile separator whitespace
|
||||
set xlabel "Timestep"
|
||||
set ylabel "Residual"
|
||||
|
||||
set logscale y 2
|
||||
|
||||
plot 'residual.dat' using 1:2 title "Residual"
|
@ -167,11 +167,12 @@ static void assembleResult(Comm* c,
|
||||
MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
|
||||
}
|
||||
|
||||
static int sum(int* sizes, int position)
|
||||
// subroutines local to this module
|
||||
static int sum(int* sizes, int init, int offset, int coord)
|
||||
{
|
||||
int sum = 0;
|
||||
|
||||
for (int i = 0; i < position; i++) {
|
||||
for (int i = init - offset; coord > 0; i -= offset, --coord) {
|
||||
sum += sizes[i];
|
||||
}
|
||||
|
||||
@ -336,7 +337,6 @@ void commCollectResult(Comm* c,
|
||||
int imaxLocal = c->imaxLocal;
|
||||
int jmaxLocal = c->jmaxLocal;
|
||||
int kmaxLocal = c->kmaxLocal;
|
||||
|
||||
#if defined(_MPI)
|
||||
int offset[c->size * NDIMS];
|
||||
int imaxLocalAll[c->size];
|
||||
@ -351,9 +351,13 @@ void commCollectResult(Comm* c,
|
||||
for (int i = 0; i < c->size; i++) {
|
||||
int coords[NCORDS];
|
||||
MPI_Cart_coords(c->comm, i, NDIMS, coords);
|
||||
offset[i * NDIMS + IDIM] = sum(imaxLocalAll, coords[ICORD]);
|
||||
offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, coords[JCORD]);
|
||||
offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, coords[KCORD]);
|
||||
offset[i * NDIMS + IDIM] = sum(imaxLocalAll,
|
||||
i,
|
||||
c->dims[IDIM] * c->dims[JDIM],
|
||||
coords[ICORD]);
|
||||
offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, i, c->dims[IDIM], coords[JCORD]);
|
||||
offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, i, 1, coords[KCORD]);
|
||||
|
||||
printf("Rank: %d, Coords(k,j,i): %d %d %d, Size(k,j,i): %d %d %d, "
|
||||
"Offset(k,j,i): %d %d %d\n",
|
||||
i,
|
||||
@ -562,9 +566,9 @@ void commPartition(Comm* c, int kmax, int jmax, int imax)
|
||||
MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]);
|
||||
MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords);
|
||||
|
||||
c->imaxLocal = sizeOfRank(c->rank, dims[ICORD], imax);
|
||||
c->jmaxLocal = sizeOfRank(c->rank, dims[JCORD], jmax);
|
||||
c->kmaxLocal = sizeOfRank(c->rank, dims[KCORD], kmax);
|
||||
c->imaxLocal = sizeOfRank(c->coords[KDIM], dims[ICORD], imax);
|
||||
c->jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JCORD], jmax);
|
||||
c->kmaxLocal = sizeOfRank(c->coords[IDIM], dims[KCORD], kmax);
|
||||
|
||||
// setup buffer types for communication
|
||||
setupCommunication(c, LEFT, BULK);
|
||||
@ -597,3 +601,47 @@ void commFinalize(Comm* c)
|
||||
MPI_Finalize();
|
||||
#endif
|
||||
}
|
||||
|
||||
void commUpdateDatatypes(
|
||||
Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal)
|
||||
{
|
||||
#if defined _MPI
|
||||
|
||||
int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
|
||||
|
||||
if (result == MPI_ERR_COMM) {
|
||||
printf("\nNull communicator. Duplication failed !!\n");
|
||||
}
|
||||
|
||||
newcomm->rank = oldcomm->rank;
|
||||
newcomm->size = oldcomm->size;
|
||||
|
||||
newcomm->imaxLocal = imaxLocal / 2;
|
||||
newcomm->jmaxLocal = jmaxLocal / 2;
|
||||
newcomm->kmaxLocal = kmaxLocal / 2;
|
||||
|
||||
setupCommunication(newcomm, LEFT, BULK);
|
||||
setupCommunication(newcomm, LEFT, HALO);
|
||||
setupCommunication(newcomm, RIGHT, BULK);
|
||||
setupCommunication(newcomm, RIGHT, HALO);
|
||||
setupCommunication(newcomm, BOTTOM, BULK);
|
||||
setupCommunication(newcomm, BOTTOM, HALO);
|
||||
setupCommunication(newcomm, TOP, BULK);
|
||||
setupCommunication(newcomm, TOP, HALO);
|
||||
setupCommunication(newcomm, FRONT, BULK);
|
||||
setupCommunication(newcomm, FRONT, HALO);
|
||||
setupCommunication(newcomm, BACK, BULK);
|
||||
setupCommunication(newcomm, BACK, HALO);
|
||||
#else
|
||||
newcomm->imaxLocal = imaxLocal;
|
||||
newcomm->jmaxLocal = jmaxLocal;
|
||||
newcomm->kmaxLocal = kmaxLocal;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commFreeCommunicator(Comm* comm)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Comm_free(&comm->comm);
|
||||
#endif
|
||||
}
|
@ -23,6 +23,7 @@ typedef enum dimension { KDIM = 0, JDIM, IDIM, NDIMS } Dimension;
|
||||
enum layer { HALO = 0, BULK };
|
||||
enum op { MAX = 0, SUM };
|
||||
|
||||
|
||||
typedef struct {
|
||||
int rank;
|
||||
int size;
|
||||
@ -45,6 +46,9 @@ extern void commShift(Comm* c, double* f, double* g, double* h);
|
||||
extern void commReduction(double* v, int op);
|
||||
extern int commIsBoundary(Comm* c, Direction direction);
|
||||
extern void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax);
|
||||
extern void commFreeCommunicator(Comm* comm);
|
||||
extern void commUpdateDatatypes(
|
||||
Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal);
|
||||
extern void commCollectResult(Comm* c,
|
||||
double* ug,
|
||||
double* vg,
|
||||
|
729
BasicSolver/3D-mpi/src/discretization.c
Normal file
729
BasicSolver/3D-mpi/src/discretization.c
Normal file
@ -0,0 +1,729 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <float.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "allocate.h"
|
||||
#include "discretization.h"
|
||||
#include "parameter.h"
|
||||
#include "util.h"
|
||||
|
||||
#define P(i, j, k) \
|
||||
p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define F(i, j, k) \
|
||||
f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define G(i, j, k) \
|
||||
g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define H(i, j, k) \
|
||||
h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define U(i, j, k) \
|
||||
u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define V(i, j, k) \
|
||||
v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define W(i, j, k) \
|
||||
w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define RHS(i, j, k) \
|
||||
rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
|
||||
static void printConfig(Discretization* s)
|
||||
{
|
||||
if (commIsMaster(&s->comm)) {
|
||||
printf("Parameters for #%s#\n", s->problem);
|
||||
printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n",
|
||||
s->bcLeft,
|
||||
s->bcRight,
|
||||
s->bcBottom,
|
||||
s->bcTop,
|
||||
s->bcFront,
|
||||
s->bcBack);
|
||||
printf("\tReynolds number: %.2f\n", s->re);
|
||||
printf("\tGx Gy: %.2f %.2f %.2f\n", s->gx, s->gy, s->gz);
|
||||
printf("Geometry data:\n");
|
||||
printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n",
|
||||
s->grid.xlength,
|
||||
s->grid.ylength,
|
||||
s->grid.zlength);
|
||||
printf("\tCells (x, y, z): %d, %d, %d\n",
|
||||
s->grid.imax,
|
||||
s->grid.jmax,
|
||||
s->grid.kmax);
|
||||
printf("\tCell size (dx, dy, dz): %f, %f, %f\n",
|
||||
s->grid.dx,
|
||||
s->grid.dy,
|
||||
s->grid.dz);
|
||||
printf("Timestep parameters:\n");
|
||||
printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te);
|
||||
printf("\tdt bound: %.6f\n", s->dtBound);
|
||||
printf("\tTau factor: %.2f\n", s->tau);
|
||||
printf("Iterative parameters:\n");
|
||||
printf("\tMax iterations: %d\n", s->itermax);
|
||||
printf("\tepsilon (stopping tolerance) : %f\n", s->eps);
|
||||
printf("\tgamma factor: %f\n", s->gamma);
|
||||
printf("\tomega (SOR relaxation): %f\n", s->omega);
|
||||
}
|
||||
commPrintConfig(&s->comm);
|
||||
}
|
||||
|
||||
void initDiscretization(Discretization* s, Parameter* params)
|
||||
{
|
||||
s->problem = params->name;
|
||||
s->bcLeft = params->bcLeft;
|
||||
s->bcRight = params->bcRight;
|
||||
s->bcBottom = params->bcBottom;
|
||||
s->bcTop = params->bcTop;
|
||||
s->bcFront = params->bcFront;
|
||||
s->bcBack = params->bcBack;
|
||||
|
||||
s->grid.imax = params->imax;
|
||||
s->grid.jmax = params->jmax;
|
||||
s->grid.kmax = params->kmax;
|
||||
s->grid.xlength = params->xlength;
|
||||
s->grid.ylength = params->ylength;
|
||||
s->grid.zlength = params->zlength;
|
||||
s->grid.dx = params->xlength / params->imax;
|
||||
s->grid.dy = params->ylength / params->jmax;
|
||||
s->grid.dz = params->zlength / params->kmax;
|
||||
|
||||
s->eps = params->eps;
|
||||
s->omega = params->omg;
|
||||
s->itermax = params->itermax;
|
||||
s->re = params->re;
|
||||
s->gx = params->gx;
|
||||
s->gy = params->gy;
|
||||
s->gz = params->gz;
|
||||
s->dt = params->dt;
|
||||
s->te = params->te;
|
||||
s->tau = params->tau;
|
||||
s->gamma = params->gamma;
|
||||
|
||||
/* allocate arrays */
|
||||
int imaxLocal = s->comm.imaxLocal;
|
||||
int jmaxLocal = s->comm.jmaxLocal;
|
||||
int kmaxLocal = s->comm.kmaxLocal;
|
||||
size_t size = (imaxLocal + 2) * (jmaxLocal + 2) * (kmaxLocal + 2);
|
||||
|
||||
s->u = allocate(64, size * sizeof(double));
|
||||
s->v = allocate(64, size * sizeof(double));
|
||||
s->w = allocate(64, size * sizeof(double));
|
||||
s->p = allocate(64, size * sizeof(double));
|
||||
s->rhs = allocate(64, size * sizeof(double));
|
||||
s->f = allocate(64, size * sizeof(double));
|
||||
s->g = allocate(64, size * sizeof(double));
|
||||
s->h = allocate(64, size * sizeof(double));
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
s->u[i] = params->u_init;
|
||||
s->v[i] = params->v_init;
|
||||
s->w[i] = params->w_init;
|
||||
s->p[i] = params->p_init;
|
||||
s->rhs[i] = 0.0;
|
||||
s->f[i] = 0.0;
|
||||
s->g[i] = 0.0;
|
||||
s->h[i] = 0.0;
|
||||
}
|
||||
|
||||
double dx = s->grid.dx;
|
||||
double dy = s->grid.dy;
|
||||
double dz = s->grid.dz;
|
||||
|
||||
double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz);
|
||||
s->dtBound = 0.5 * s->re * 1.0 / invSqrSum;
|
||||
|
||||
#ifdef VERBOSE
|
||||
printConfig(s);
|
||||
#endif /* VERBOSE */
|
||||
}
|
||||
|
||||
void setBoundaryConditions(Discretization* s)
|
||||
{
|
||||
int imaxLocal = s->comm.imaxLocal;
|
||||
int jmaxLocal = s->comm.jmaxLocal;
|
||||
int kmaxLocal = s->comm.kmaxLocal;
|
||||
|
||||
double* u = s->u;
|
||||
double* v = s->v;
|
||||
double* w = s->w;
|
||||
|
||||
if (commIsBoundary(&s->comm, TOP)) {
|
||||
switch (s->bcTop) {
|
||||
case NOSLIP:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, jmaxLocal + 1, k) = -U(i, jmaxLocal, k);
|
||||
V(i, jmaxLocal, k) = 0.0;
|
||||
W(i, jmaxLocal + 1, k) = -W(i, jmaxLocal, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case SLIP:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k);
|
||||
V(i, jmaxLocal, k) = 0.0;
|
||||
W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case OUTFLOW:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k);
|
||||
V(i, jmaxLocal, k) = V(i, jmaxLocal - 1, k);
|
||||
W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case PERIODIC:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&s->comm, BOTTOM)) {
|
||||
switch (s->bcBottom) {
|
||||
case NOSLIP:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, 0, k) = -U(i, 1, k);
|
||||
V(i, 0, k) = 0.0;
|
||||
W(i, 0, k) = -W(i, 1, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case SLIP:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, 0, k) = U(i, 1, k);
|
||||
V(i, 0, k) = 0.0;
|
||||
W(i, 0, k) = W(i, 1, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case OUTFLOW:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, 0, k) = U(i, 1, k);
|
||||
V(i, 0, k) = V(i, 1, k);
|
||||
W(i, 0, k) = W(i, 1, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case PERIODIC:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&s->comm, LEFT)) {
|
||||
switch (s->bcLeft) {
|
||||
case NOSLIP:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(0, j, k) = 0.0;
|
||||
V(0, j, k) = -V(1, j, k);
|
||||
W(0, j, k) = -W(1, j, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case SLIP:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(0, j, k) = 0.0;
|
||||
V(0, j, k) = V(1, j, k);
|
||||
W(0, j, k) = W(1, j, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case OUTFLOW:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(0, j, k) = U(1, j, k);
|
||||
V(0, j, k) = V(1, j, k);
|
||||
W(0, j, k) = W(1, j, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case PERIODIC:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&s->comm, RIGHT)) {
|
||||
switch (s->bcRight) {
|
||||
case NOSLIP:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(imaxLocal, j, k) = 0.0;
|
||||
V(imaxLocal + 1, j, k) = -V(imaxLocal, j, k);
|
||||
W(imaxLocal + 1, j, k) = -W(imaxLocal, j, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case SLIP:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(imaxLocal, j, k) = 0.0;
|
||||
V(imaxLocal + 1, j, k) = V(imaxLocal, j, k);
|
||||
W(imaxLocal + 1, j, k) = W(imaxLocal, j, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case OUTFLOW:
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(imaxLocal, j, k) = U(imaxLocal - 1, j, k);
|
||||
V(imaxLocal + 1, j, k) = V(imaxLocal, j, k);
|
||||
W(imaxLocal + 1, j, k) = W(imaxLocal, j, k);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case PERIODIC:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&s->comm, FRONT)) {
|
||||
switch (s->bcFront) {
|
||||
case NOSLIP:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, j, 0) = -U(i, j, 1);
|
||||
V(i, j, 0) = -V(i, j, 1);
|
||||
W(i, j, 0) = 0.0;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case SLIP:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, j, 0) = U(i, j, 1);
|
||||
V(i, j, 0) = V(i, j, 1);
|
||||
W(i, j, 0) = 0.0;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case OUTFLOW:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, j, 0) = U(i, j, 1);
|
||||
V(i, j, 0) = V(i, j, 1);
|
||||
W(i, j, 0) = W(i, j, 1);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case PERIODIC:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&s->comm, BACK)) {
|
||||
switch (s->bcBack) {
|
||||
case NOSLIP:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, j, kmaxLocal + 1) = -U(i, j, kmaxLocal);
|
||||
V(i, j, kmaxLocal + 1) = -V(i, j, kmaxLocal);
|
||||
W(i, j, kmaxLocal) = 0.0;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case SLIP:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal);
|
||||
V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal);
|
||||
W(i, j, kmaxLocal) = 0.0;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case OUTFLOW:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal);
|
||||
V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal);
|
||||
W(i, j, kmaxLocal) = W(i, j, kmaxLocal - 1);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case PERIODIC:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void computeRHS(Discretization* s)
|
||||
{
|
||||
int imaxLocal = s->comm.imaxLocal;
|
||||
int jmaxLocal = s->comm.jmaxLocal;
|
||||
int kmaxLocal = s->comm.kmaxLocal;
|
||||
|
||||
double idx = 1.0 / s->grid.dx;
|
||||
double idy = 1.0 / s->grid.dy;
|
||||
double idz = 1.0 / s->grid.dz;
|
||||
double idt = 1.0 / s->dt;
|
||||
|
||||
double* rhs = s->rhs;
|
||||
double* f = s->f;
|
||||
double* g = s->g;
|
||||
double* h = s->h;
|
||||
|
||||
commShift(&s->comm, f, g, h);
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx +
|
||||
(G(i, j, k) - G(i, j - 1, k)) * idy +
|
||||
(H(i, j, k) - H(i, j, k - 1)) * idz) *
|
||||
idt;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void setSpecialBoundaryCondition(Discretization* s)
|
||||
{
|
||||
int imaxLocal = s->comm.imaxLocal;
|
||||
int jmaxLocal = s->comm.jmaxLocal;
|
||||
int kmaxLocal = s->comm.kmaxLocal;
|
||||
|
||||
double* u = s->u;
|
||||
|
||||
if (strcmp(s->problem, "dcavity") == 0) {
|
||||
if (commIsBoundary(&s->comm, TOP)) {
|
||||
for (int k = 1; k < kmaxLocal; k++) {
|
||||
for (int i = 1; i < imaxLocal; i++) {
|
||||
U(i, jmaxLocal + 1, k) = 2.0 - U(i, jmaxLocal, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (strcmp(s->problem, "canal") == 0) {
|
||||
if (commIsBoundary(&s->comm, LEFT)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(0, j, k) = 2.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static double maxElement(Discretization* s, double* m)
|
||||
{
|
||||
int size = (s->comm.imaxLocal + 2) * (s->comm.jmaxLocal + 2) *
|
||||
(s->comm.kmaxLocal + 2);
|
||||
double maxval = DBL_MIN;
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
maxval = MAX(maxval, fabs(m[i]));
|
||||
}
|
||||
commReduction(&maxval, MAX);
|
||||
return maxval;
|
||||
}
|
||||
|
||||
void normalizePressure(Discretization* s)
|
||||
{
|
||||
int imaxLocal = s->comm.imaxLocal;
|
||||
int jmaxLocal = s->comm.jmaxLocal;
|
||||
int kmaxLocal = s->comm.kmaxLocal;
|
||||
|
||||
double* p = s->p;
|
||||
double avgP = 0.0;
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
avgP += P(i, j, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
commReduction(&avgP, SUM);
|
||||
avgP /= (s->grid.imax * s->grid.jmax * s->grid.kmax);
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, j, k) = P(i, j, k) - avgP;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void computeTimestep(Discretization* s)
|
||||
{
|
||||
double dt = s->dtBound;
|
||||
double dx = s->grid.dx;
|
||||
double dy = s->grid.dy;
|
||||
double dz = s->grid.dz;
|
||||
|
||||
double umax = maxElement(s, s->u);
|
||||
double vmax = maxElement(s, s->v);
|
||||
double wmax = maxElement(s, s->w);
|
||||
|
||||
if (umax > 0) {
|
||||
dt = (dt > dx / umax) ? dx / umax : dt;
|
||||
}
|
||||
if (vmax > 0) {
|
||||
dt = (dt > dy / vmax) ? dy / vmax : dt;
|
||||
}
|
||||
if (wmax > 0) {
|
||||
dt = (dt > dz / wmax) ? dz / wmax : dt;
|
||||
}
|
||||
|
||||
s->dt = dt * s->tau;
|
||||
}
|
||||
|
||||
|
||||
|
||||
void computeFG(Discretization* s)
|
||||
{
|
||||
int imaxLocal = s->comm.imaxLocal;
|
||||
int jmaxLocal = s->comm.jmaxLocal;
|
||||
int kmaxLocal = s->comm.kmaxLocal;
|
||||
|
||||
double* u = s->u;
|
||||
double* v = s->v;
|
||||
double* w = s->w;
|
||||
double* f = s->f;
|
||||
double* g = s->g;
|
||||
double* h = s->h;
|
||||
|
||||
double gx = s->gx;
|
||||
double gy = s->gy;
|
||||
double gz = s->gz;
|
||||
double dt = s->dt;
|
||||
|
||||
double gamma = s->gamma;
|
||||
double inverseRe = 1.0 / s->re;
|
||||
double inverseDx = 1.0 / s->grid.dx;
|
||||
double inverseDy = 1.0 / s->grid.dy;
|
||||
double inverseDz = 1.0 / s->grid.dz;
|
||||
double du2dx, dv2dy, dw2dz;
|
||||
double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz;
|
||||
double du2dx2, du2dy2, du2dz2;
|
||||
double dv2dx2, dv2dy2, dv2dz2;
|
||||
double dw2dx2, dw2dy2, dw2dz2;
|
||||
|
||||
commExchange(&s->comm, u);
|
||||
commExchange(&s->comm, v);
|
||||
commExchange(&s->comm, w);
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
du2dx = inverseDx * 0.25 *
|
||||
((U(i, j, k) + U(i + 1, j, k)) *
|
||||
(U(i, j, k) + U(i + 1, j, k)) -
|
||||
(U(i, j, k) + U(i - 1, j, k)) *
|
||||
(U(i, j, k) + U(i - 1, j, k))) +
|
||||
gamma * inverseDx * 0.25 *
|
||||
(fabs(U(i, j, k) + U(i + 1, j, k)) *
|
||||
(U(i, j, k) - U(i + 1, j, k)) +
|
||||
fabs(U(i, j, k) + U(i - 1, j, k)) *
|
||||
(U(i, j, k) - U(i - 1, j, k)));
|
||||
|
||||
duvdy = inverseDy * 0.25 *
|
||||
((V(i, j, k) + V(i + 1, j, k)) *
|
||||
(U(i, j, k) + U(i, j + 1, k)) -
|
||||
(V(i, j - 1, k) + V(i + 1, j - 1, k)) *
|
||||
(U(i, j, k) + U(i, j - 1, k))) +
|
||||
gamma * inverseDy * 0.25 *
|
||||
(fabs(V(i, j, k) + V(i + 1, j, k)) *
|
||||
(U(i, j, k) - U(i, j + 1, k)) +
|
||||
fabs(V(i, j - 1, k) + V(i + 1, j - 1, k)) *
|
||||
(U(i, j, k) - U(i, j - 1, k)));
|
||||
|
||||
duwdz = inverseDz * 0.25 *
|
||||
((W(i, j, k) + W(i + 1, j, k)) *
|
||||
(U(i, j, k) + U(i, j, k + 1)) -
|
||||
(W(i, j, k - 1) + W(i + 1, j, k - 1)) *
|
||||
(U(i, j, k) + U(i, j, k - 1))) +
|
||||
gamma * inverseDz * 0.25 *
|
||||
(fabs(W(i, j, k) + W(i + 1, j, k)) *
|
||||
(U(i, j, k) - U(i, j, k + 1)) +
|
||||
fabs(W(i, j, k - 1) + W(i + 1, j, k - 1)) *
|
||||
(U(i, j, k) - U(i, j, k - 1)));
|
||||
|
||||
du2dx2 = inverseDx * inverseDx *
|
||||
(U(i + 1, j, k) - 2.0 * U(i, j, k) + U(i - 1, j, k));
|
||||
du2dy2 = inverseDy * inverseDy *
|
||||
(U(i, j + 1, k) - 2.0 * U(i, j, k) + U(i, j - 1, k));
|
||||
du2dz2 = inverseDz * inverseDz *
|
||||
(U(i, j, k + 1) - 2.0 * U(i, j, k) + U(i, j, k - 1));
|
||||
F(i, j, k) = U(i, j, k) + dt * (inverseRe * (du2dx2 + du2dy2 + du2dz2) -
|
||||
du2dx - duvdy - duwdz + gx);
|
||||
|
||||
duvdx = inverseDx * 0.25 *
|
||||
((U(i, j, k) + U(i, j + 1, k)) *
|
||||
(V(i, j, k) + V(i + 1, j, k)) -
|
||||
(U(i - 1, j, k) + U(i - 1, j + 1, k)) *
|
||||
(V(i, j, k) + V(i - 1, j, k))) +
|
||||
gamma * inverseDx * 0.25 *
|
||||
(fabs(U(i, j, k) + U(i, j + 1, k)) *
|
||||
(V(i, j, k) - V(i + 1, j, k)) +
|
||||
fabs(U(i - 1, j, k) + U(i - 1, j + 1, k)) *
|
||||
(V(i, j, k) - V(i - 1, j, k)));
|
||||
|
||||
dv2dy = inverseDy * 0.25 *
|
||||
((V(i, j, k) + V(i, j + 1, k)) *
|
||||
(V(i, j, k) + V(i, j + 1, k)) -
|
||||
(V(i, j, k) + V(i, j - 1, k)) *
|
||||
(V(i, j, k) + V(i, j - 1, k))) +
|
||||
gamma * inverseDy * 0.25 *
|
||||
(fabs(V(i, j, k) + V(i, j + 1, k)) *
|
||||
(V(i, j, k) - V(i, j + 1, k)) +
|
||||
fabs(V(i, j, k) + V(i, j - 1, k)) *
|
||||
(V(i, j, k) - V(i, j - 1, k)));
|
||||
|
||||
dvwdz = inverseDz * 0.25 *
|
||||
((W(i, j, k) + W(i, j + 1, k)) *
|
||||
(V(i, j, k) + V(i, j, k + 1)) -
|
||||
(W(i, j, k - 1) + W(i, j + 1, k - 1)) *
|
||||
(V(i, j, k) + V(i, j, k + 1))) +
|
||||
gamma * inverseDz * 0.25 *
|
||||
(fabs(W(i, j, k) + W(i, j + 1, k)) *
|
||||
(V(i, j, k) - V(i, j, k + 1)) +
|
||||
fabs(W(i, j, k - 1) + W(i, j + 1, k - 1)) *
|
||||
(V(i, j, k) - V(i, j, k + 1)));
|
||||
|
||||
dv2dx2 = inverseDx * inverseDx *
|
||||
(V(i + 1, j, k) - 2.0 * V(i, j, k) + V(i - 1, j, k));
|
||||
dv2dy2 = inverseDy * inverseDy *
|
||||
(V(i, j + 1, k) - 2.0 * V(i, j, k) + V(i, j - 1, k));
|
||||
dv2dz2 = inverseDz * inverseDz *
|
||||
(V(i, j, k + 1) - 2.0 * V(i, j, k) + V(i, j, k - 1));
|
||||
G(i, j, k) = V(i, j, k) + dt * (inverseRe * (dv2dx2 + dv2dy2 + dv2dz2) -
|
||||
duvdx - dv2dy - dvwdz + gy);
|
||||
|
||||
duwdx = inverseDx * 0.25 *
|
||||
((U(i, j, k) + U(i, j, k + 1)) *
|
||||
(W(i, j, k) + W(i + 1, j, k)) -
|
||||
(U(i - 1, j, k) + U(i - 1, j, k + 1)) *
|
||||
(W(i, j, k) + W(i - 1, j, k))) +
|
||||
gamma * inverseDx * 0.25 *
|
||||
(fabs(U(i, j, k) + U(i, j, k + 1)) *
|
||||
(W(i, j, k) - W(i + 1, j, k)) +
|
||||
fabs(U(i - 1, j, k) + U(i - 1, j, k + 1)) *
|
||||
(W(i, j, k) - W(i - 1, j, k)));
|
||||
|
||||
dvwdy = inverseDy * 0.25 *
|
||||
((V(i, j, k) + V(i, j, k + 1)) *
|
||||
(W(i, j, k) + W(i, j + 1, k)) -
|
||||
(V(i, j - 1, k + 1) + V(i, j - 1, k)) *
|
||||
(W(i, j, k) + W(i, j - 1, k))) +
|
||||
gamma * inverseDy * 0.25 *
|
||||
(fabs(V(i, j, k) + V(i, j, k + 1)) *
|
||||
(W(i, j, k) - W(i, j + 1, k)) +
|
||||
fabs(V(i, j - 1, k + 1) + V(i, j - 1, k)) *
|
||||
(W(i, j, k) - W(i, j - 1, k)));
|
||||
|
||||
dw2dz = inverseDz * 0.25 *
|
||||
((W(i, j, k) + W(i, j, k + 1)) *
|
||||
(W(i, j, k) + W(i, j, k + 1)) -
|
||||
(W(i, j, k) + W(i, j, k - 1)) *
|
||||
(W(i, j, k) + W(i, j, k - 1))) +
|
||||
gamma * inverseDz * 0.25 *
|
||||
(fabs(W(i, j, k) + W(i, j, k + 1)) *
|
||||
(W(i, j, k) - W(i, j, k + 1)) +
|
||||
fabs(W(i, j, k) + W(i, j, k - 1)) *
|
||||
(W(i, j, k) - W(i, j, k - 1)));
|
||||
|
||||
dw2dx2 = inverseDx * inverseDx *
|
||||
(W(i + 1, j, k) - 2.0 * W(i, j, k) + W(i - 1, j, k));
|
||||
dw2dy2 = inverseDy * inverseDy *
|
||||
(W(i, j + 1, k) - 2.0 * W(i, j, k) + W(i, j - 1, k));
|
||||
dw2dz2 = inverseDz * inverseDz *
|
||||
(W(i, j, k + 1) - 2.0 * W(i, j, k) + W(i, j, k - 1));
|
||||
H(i, j, k) = W(i, j, k) + dt * (inverseRe * (dw2dx2 + dw2dy2 + dw2dz2) -
|
||||
duwdx - dvwdy - dw2dz + gz);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ----------------------------- boundary of F ---------------------------
|
||||
*/
|
||||
if (commIsBoundary(&s->comm, LEFT)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
F(0, j, k) = U(0, j, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&s->comm, RIGHT)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
F(imaxLocal, j, k) = U(imaxLocal, j, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ----------------------------- boundary of G ---------------------------
|
||||
*/
|
||||
if (commIsBoundary(&s->comm, BOTTOM)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
G(i, 0, k) = V(i, 0, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&s->comm, TOP)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
G(i, jmaxLocal, k) = V(i, jmaxLocal, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ----------------------------- boundary of H ---------------------------
|
||||
*/
|
||||
if (commIsBoundary(&s->comm, FRONT)) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
H(i, j, 0) = W(i, j, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&s->comm, BACK)) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
H(i, j, kmaxLocal) = W(i, j, kmaxLocal);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void adaptUV(Discretization* s)
|
||||
{
|
||||
int imaxLocal = s->comm.imaxLocal;
|
||||
int jmaxLocal = s->comm.jmaxLocal;
|
||||
int kmaxLocal = s->comm.kmaxLocal;
|
||||
|
||||
double* p = s->p;
|
||||
double* u = s->u;
|
||||
double* v = s->v;
|
||||
double* w = s->w;
|
||||
double* f = s->f;
|
||||
double* g = s->g;
|
||||
double* h = s->h;
|
||||
|
||||
double factorX = s->dt / s->grid.dx;
|
||||
double factorY = s->dt / s->grid.dy;
|
||||
double factorZ = s->dt / s->grid.dz;
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, j, k) = F(i, j, k) - (P(i + 1, j, k) - P(i, j, k)) * factorX;
|
||||
V(i, j, k) = G(i, j, k) - (P(i, j + 1, k) - P(i, j, k)) * factorY;
|
||||
W(i, j, k) = H(i, j, k) - (P(i, j, k + 1) - P(i, j, k)) * factorZ;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
44
BasicSolver/3D-mpi/src/discretization.h
Normal file
44
BasicSolver/3D-mpi/src/discretization.h
Normal file
@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#ifndef __DISCRETIZATION_H_
|
||||
#define __DISCRETIZATION_H_
|
||||
|
||||
#include "grid.h"
|
||||
#include "parameter.h"
|
||||
#include "comm.h"
|
||||
|
||||
enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
|
||||
|
||||
typedef struct {
|
||||
/* geometry and grid information */
|
||||
Grid grid;
|
||||
/* arrays */
|
||||
double *p, *rhs;
|
||||
double *f, *g, *h;
|
||||
double *u, *v, *w;
|
||||
/* parameters */
|
||||
double eps, omega;
|
||||
double re, tau, gamma;
|
||||
double gx, gy, gz;
|
||||
/* time stepping */
|
||||
int itermax;
|
||||
double dt, te;
|
||||
double dtBound;
|
||||
char* problem;
|
||||
int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
|
||||
Comm comm;
|
||||
} Discretization;
|
||||
|
||||
extern void initDiscretization(Discretization*, Parameter*);
|
||||
extern void computeRHS(Discretization*);
|
||||
extern void normalizePressure(Discretization*);
|
||||
extern void computeTimestep(Discretization*);
|
||||
extern void setBoundaryConditions(Discretization*);
|
||||
extern void setSpecialBoundaryCondition(Discretization*);
|
||||
extern void computeFG(Discretization*);
|
||||
extern void adaptUV(Discretization*);
|
||||
#endif
|
@ -9,7 +9,7 @@
|
||||
#include <unistd.h>
|
||||
|
||||
#include "allocate.h"
|
||||
#include "comm.h"
|
||||
#include "discretization.h"
|
||||
#include "parameter.h"
|
||||
#include "progress.h"
|
||||
#include "solver.h"
|
||||
@ -21,9 +21,12 @@ int main(int argc, char** argv)
|
||||
double timeStart, timeStop;
|
||||
Parameter p;
|
||||
Solver s;
|
||||
Discretization d;
|
||||
|
||||
commInit(&s.comm, argc, argv);
|
||||
commInit(&d.comm, argc, argv);
|
||||
initParameter(&p);
|
||||
FILE* fp;
|
||||
if (commIsMaster(&d.comm)) fp = initResidualWriter();
|
||||
|
||||
if (argc != 2) {
|
||||
printf("Usage: %s <configFile>\n", argv[0]);
|
||||
@ -31,33 +34,44 @@ int main(int argc, char** argv)
|
||||
}
|
||||
|
||||
readParameter(&p, argv[1]);
|
||||
commPartition(&s.comm, p.kmax, p.jmax, p.imax);
|
||||
if (commIsMaster(&s.comm)) {
|
||||
commPartition(&d.comm, p.kmax, p.jmax, p.imax);
|
||||
|
||||
if (commIsMaster(&d.comm)) {
|
||||
printParameter(&p);
|
||||
}
|
||||
initSolver(&s, &p);
|
||||
|
||||
initDiscretization(&d, &p);
|
||||
initSolver(&s, &d, &p);
|
||||
|
||||
#ifndef VERBOSE
|
||||
initProgress(s.te);
|
||||
initProgress(d.te);
|
||||
#endif
|
||||
|
||||
double tau = s.tau;
|
||||
double te = s.te;
|
||||
double tau = d.tau;
|
||||
double te = d.te;
|
||||
double t = 0.0;
|
||||
int nt = 0;
|
||||
double res = 0.0;
|
||||
|
||||
timeStart = getTimeStamp();
|
||||
while (t <= te) {
|
||||
if (tau > 0.0) computeTimestep(&s);
|
||||
setBoundaryConditions(&s);
|
||||
setSpecialBoundaryCondition(&s);
|
||||
computeFG(&s);
|
||||
computeRHS(&s);
|
||||
solve(&s);
|
||||
adaptUV(&s);
|
||||
t += s.dt;
|
||||
if (tau > 0.0) computeTimestep(&d);
|
||||
setBoundaryConditions(&d);
|
||||
setSpecialBoundaryCondition(&d);
|
||||
computeFG(&d);
|
||||
computeRHS(&d);
|
||||
if (nt % 100 == 0) normalizePressure(&d);
|
||||
res = solve(&s, d.p, d.rhs);
|
||||
adaptUV(&d);
|
||||
|
||||
if (commIsMaster(&d.comm)) writeResidual(fp, t, res);
|
||||
|
||||
t += d.dt;
|
||||
nt++;
|
||||
|
||||
#ifdef VERBOSE
|
||||
if (commIsMaster(&s.comm)) {
|
||||
printf("TIME %f , TIMESTEP %f\n", t, s.dt);
|
||||
if (commIsMaster(s.comm)) {
|
||||
printf("TIME %f , TIMESTEP %f\n", t, d.dt);
|
||||
}
|
||||
#else
|
||||
printProgress(t);
|
||||
@ -67,7 +81,7 @@ int main(int argc, char** argv)
|
||||
#ifndef VERBOSE
|
||||
stopProgress();
|
||||
#endif
|
||||
if (commIsMaster(&s.comm)) {
|
||||
if (commIsMaster(s.comm)) {
|
||||
printf("Solution took %.2fs\n", timeStop - timeStart);
|
||||
}
|
||||
|
||||
@ -75,14 +89,16 @@ int main(int argc, char** argv)
|
||||
#ifdef _VTK_WRITER_MPI
|
||||
VtkOptions opts = { .grid = s.grid, .comm = s.comm };
|
||||
vtkOpen(&opts, s.problem);
|
||||
vtkScalar(&opts, "pressure", s.p);
|
||||
vtkVector(&opts, "velocity", (VtkVector) { s.u, s.v, s.w });
|
||||
vtkScalar(&opts, "pressure", d.p);
|
||||
vtkVector(&opts, "velocity", (VtkVector) { d.u, d.v, d.w });
|
||||
vtkClose(&opts);
|
||||
#else
|
||||
if (commIsMaster(&d.comm)) fclose(fp);
|
||||
|
||||
double *pg, *ug, *vg, *wg;
|
||||
|
||||
if (commIsMaster(&s.comm)) {
|
||||
size_t bytesize = s.grid.imax * s.grid.jmax * s.grid.kmax * sizeof(double);
|
||||
if (commIsMaster(s.comm)) {
|
||||
size_t bytesize = s.grid->imax * s.grid->jmax * s.grid->kmax * sizeof(double);
|
||||
|
||||
pg = allocate(64, bytesize);
|
||||
ug = allocate(64, bytesize);
|
||||
@ -90,34 +106,35 @@ int main(int argc, char** argv)
|
||||
wg = allocate(64, bytesize);
|
||||
}
|
||||
|
||||
commCollectResult(&s.comm,
|
||||
commCollectResult(s.comm,
|
||||
ug,
|
||||
vg,
|
||||
wg,
|
||||
pg,
|
||||
s.u,
|
||||
s.v,
|
||||
s.w,
|
||||
s.p,
|
||||
s.grid.kmax,
|
||||
s.grid.jmax,
|
||||
s.grid.imax);
|
||||
d.u,
|
||||
d.v,
|
||||
d.w,
|
||||
d.p,
|
||||
s.grid->kmax,
|
||||
s.grid->jmax,
|
||||
s.grid->imax);
|
||||
|
||||
if (commIsMaster(&s.comm)) {
|
||||
if (commIsMaster(s.comm)) {
|
||||
VtkOptions opts = { .grid = s.grid };
|
||||
vtkOpen(&opts, s.problem);
|
||||
vtkScalar(&opts, "pressure", pg);
|
||||
vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg });
|
||||
vtkClose(&opts);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
timeStop = getTimeStamp();
|
||||
|
||||
if (commIsMaster(&s.comm)) {
|
||||
if (commIsMaster(s.comm)) {
|
||||
printf("Result output took %.2fs\n", timeStop - timeStart);
|
||||
}
|
||||
|
||||
commFinalize(&s.comm);
|
||||
commFinalize(s.comm);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
@ -14,18 +14,22 @@
|
||||
|
||||
void initParameter(Parameter* param)
|
||||
{
|
||||
param->xlength = 1.0;
|
||||
param->ylength = 1.0;
|
||||
param->zlength = 1.0;
|
||||
param->imax = 100;
|
||||
param->jmax = 100;
|
||||
param->kmax = 100;
|
||||
param->itermax = 1000;
|
||||
param->eps = 0.0001;
|
||||
param->omg = 1.7;
|
||||
param->re = 100.0;
|
||||
param->gamma = 0.9;
|
||||
param->tau = 0.5;
|
||||
param->xlength = 1.0;
|
||||
param->ylength = 1.0;
|
||||
param->zlength = 1.0;
|
||||
param->imax = 100;
|
||||
param->jmax = 100;
|
||||
param->kmax = 100;
|
||||
param->itermax = 1000;
|
||||
param->eps = 0.0001;
|
||||
param->omg = 1.7;
|
||||
param->re = 100.0;
|
||||
param->gamma = 0.9;
|
||||
param->tau = 0.5;
|
||||
param->levels = 5;
|
||||
param->presmooth = 5;
|
||||
param->postsmooth = 5;
|
||||
|
||||
}
|
||||
|
||||
void readParameter(Parameter* param, const char* filename)
|
||||
@ -65,6 +69,9 @@ void readParameter(Parameter* param, const char* filename)
|
||||
PARSE_INT(jmax);
|
||||
PARSE_INT(kmax);
|
||||
PARSE_INT(itermax);
|
||||
PARSE_INT(levels);
|
||||
PARSE_INT(presmooth);
|
||||
PARSE_INT(postsmooth);
|
||||
PARSE_REAL(eps);
|
||||
PARSE_REAL(omg);
|
||||
PARSE_REAL(re);
|
||||
|
@ -18,6 +18,7 @@ typedef struct {
|
||||
char* name;
|
||||
int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
|
||||
double u_init, v_init, w_init, p_init;
|
||||
int levels, presmooth, postsmooth;
|
||||
} Parameter;
|
||||
|
||||
void initParameter(Parameter*);
|
||||
|
@ -4,12 +4,12 @@
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include "progress.h"
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "progress.h"
|
||||
|
||||
static double _end;
|
||||
static int _current;
|
||||
|
||||
@ -48,3 +48,18 @@ void stopProgress()
|
||||
printf("\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
FILE* initResidualWriter()
|
||||
{
|
||||
FILE* fp;
|
||||
fp = fopen("residual.dat", "w");
|
||||
|
||||
if (fp == NULL) {
|
||||
printf("Error!\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
return fp;
|
||||
}
|
||||
|
||||
void writeResidual(FILE* fp, double ts, double res) { fprintf(fp, "%f, %f\n", ts, res); }
|
@ -4,11 +4,14 @@
|
||||
* Use of this source code is governed by a MIT-style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#ifndef __PROGRESS_H_
|
||||
#define __PROGRESS_H_
|
||||
|
||||
extern void initProgress(double);
|
||||
extern void printProgress(double);
|
||||
extern void stopProgress(void);
|
||||
|
||||
extern FILE* initResidualWriter(void);
|
||||
extern void writeResidual(FILE*, double, double);
|
||||
#endif
|
||||
|
403
BasicSolver/3D-mpi/src/solver-mg.c
Normal file
403
BasicSolver/3D-mpi/src/solver-mg.c
Normal file
@ -0,0 +1,403 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "allocate.h"
|
||||
#include "solver.h"
|
||||
#include "util.h"
|
||||
|
||||
#define FINEST_LEVEL 0
|
||||
#define COARSEST_LEVEL (s->levels - 1)
|
||||
#define S(i, j, k) \
|
||||
s[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define E(i, j, k) \
|
||||
e[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define R(i, j, k) \
|
||||
r[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define OLD(i, j, k) \
|
||||
old[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
|
||||
static void restrictMG(Solver* s, int level, Comm* comm)
|
||||
{
|
||||
double* r = s->r[level + 1];
|
||||
double* old = s->r[level];
|
||||
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
int kmaxLocal = comm->kmaxLocal;
|
||||
|
||||
commExchange(comm, old);
|
||||
|
||||
for (int k = 1; k < comm->kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < comm->jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < comm->imaxLocal + 1; ++i) {
|
||||
R(i, j, k) = (OLD(2 * i - 1, 2 * j - 1, 2 * k) +
|
||||
OLD(2 * i, 2 * j - 1, 2 * k) * 2 +
|
||||
OLD(2 * i + 1, 2 * j - 1, 2 * k) +
|
||||
OLD(2 * i - 1, 2 * j, 2 * k) * 2 +
|
||||
OLD(2 * i, 2 * j, 2 * k) * 8 +
|
||||
OLD(2 * i + 1, 2 * j, 2 * k) * 2 +
|
||||
OLD(2 * i - 1, 2 * j + 1, 2 * k) +
|
||||
OLD(2 * i, 2 * j + 1, 2 * k) * 2 +
|
||||
OLD(2 * i + 1, 2 * j + 1, 2 * k) +
|
||||
|
||||
OLD(2 * i - 1, 2 * j - 1, 2 * k - 1) +
|
||||
OLD(2 * i, 2 * j - 1, 2 * k - 1) * 2 +
|
||||
OLD(2 * i + 1, 2 * j - 1, 2 * k - 1) +
|
||||
OLD(2 * i - 1, 2 * j, 2 * k - 1) * 2 +
|
||||
OLD(2 * i, 2 * j, 2 * k - 1) * 4 +
|
||||
OLD(2 * i + 1, 2 * j, 2 * k - 1) * 2 +
|
||||
OLD(2 * i - 1, 2 * j + 1, 2 * k - 1) +
|
||||
OLD(2 * i, 2 * j + 1, 2 * k - 1) * 2 +
|
||||
OLD(2 * i + 1, 2 * j + 1, 2 * k - 1) +
|
||||
|
||||
OLD(2 * i - 1, 2 * j - 1, 2 * k + 1) +
|
||||
OLD(2 * i, 2 * j - 1, 2 * k + 1) * 2 +
|
||||
OLD(2 * i + 1, 2 * j - 1, 2 * k + 1) +
|
||||
OLD(2 * i - 1, 2 * j, 2 * k + 1) * 2 +
|
||||
OLD(2 * i, 2 * j, 2 * k + 1) * 4 +
|
||||
OLD(2 * i + 1, 2 * j, 2 * k + 1) * 2 +
|
||||
OLD(2 * i - 1, 2 * j + 1, 2 * k + 1) +
|
||||
OLD(2 * i, 2 * j + 1, 2 * k + 1) * 2 +
|
||||
OLD(2 * i + 1, 2 * j + 1, 2 * k + 1)) /
|
||||
64.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void prolongate(Solver* s, int level, Comm* comm)
|
||||
{
|
||||
double* old = s->r[level + 1];
|
||||
double* e = s->r[level];
|
||||
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
int kmaxLocal = comm->kmaxLocal;
|
||||
|
||||
for (int k = 2; k < kmaxLocal + 1; k += 2) {
|
||||
for (int j = 2; j < jmaxLocal + 1; j += 2) {
|
||||
for (int i = 2; i < imaxLocal + 1; i += 2) {
|
||||
E(i, j, k) = OLD(i / 2, j / 2, k / 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void correct(Solver* s, double* p, int level, Comm* comm)
|
||||
{
|
||||
double* e = s->e[level];
|
||||
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
int kmaxLocal = comm->kmaxLocal;
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; ++k) {
|
||||
for (int j = 1; j < jmaxLocal + 1; ++j) {
|
||||
for (int i = 1; i < imaxLocal + 1; ++i) {
|
||||
P(i, j, k) += E(i, j, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void setBoundaryCondition(
|
||||
Solver* s, double* p, int imaxLocal, int jmaxLocal, int kmaxLocal)
|
||||
{
|
||||
#ifdef _MPI
|
||||
if (commIsBoundary(s->comm, FRONT)) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, j, 0) = P(i, j, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, BACK)) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, BOTTOM)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, 0, k) = P(i, 1, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, TOP)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, LEFT)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
P(0, j, k) = P(1, j, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, RIGHT)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
P(imaxLocal + 1, j, k) = P(imaxLocal, j, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, j, 0) = P(i, j, 1);
|
||||
P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal);
|
||||
}
|
||||
}
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, 0, k) = P(i, 1, k);
|
||||
P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k);
|
||||
}
|
||||
}
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
P(0, j, k) = P(1, j, k);
|
||||
P(imaxLocal + 1, j, k) = P(imaxLocal, j, k);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm)
|
||||
{
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
int kmaxLocal = comm->kmaxLocal;
|
||||
|
||||
double eps = s->eps;
|
||||
int itermax = s->itermax;
|
||||
double dx2 = s->grid->dx * s->grid->dx;
|
||||
double dy2 = s->grid->dy * s->grid->dy;
|
||||
double dz2 = s->grid->dz * s->grid->dz;
|
||||
double idx2 = 1.0 / dx2;
|
||||
double idy2 = 1.0 / dy2;
|
||||
double idz2 = 1.0 / dz2;
|
||||
double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
|
||||
(dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
|
||||
double* r = s->r[level];
|
||||
double epssq = eps * eps;
|
||||
int it = 0;
|
||||
int pass, ksw, jsw, isw;
|
||||
double res = 1.0;
|
||||
|
||||
ksw = 1;
|
||||
|
||||
for (pass = 0; pass < 2; pass++) {
|
||||
jsw = ksw;
|
||||
|
||||
commExchange(comm, p);
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
isw = jsw;
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = isw; i < imaxLocal + 1; i += 2) {
|
||||
|
||||
P(i, j, k) -=
|
||||
factor *
|
||||
(RHS(i, j, k) -
|
||||
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
|
||||
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
|
||||
idy2 +
|
||||
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
|
||||
idz2));
|
||||
}
|
||||
isw = 3 - isw;
|
||||
}
|
||||
jsw = 3 - jsw;
|
||||
}
|
||||
ksw = 3 - ksw;
|
||||
}
|
||||
}
|
||||
|
||||
static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm)
|
||||
{
|
||||
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
int kmaxLocal = comm->kmaxLocal;
|
||||
|
||||
double eps = s->eps;
|
||||
int itermax = s->itermax;
|
||||
double dx2 = s->grid->dx * s->grid->dx;
|
||||
double dy2 = s->grid->dy * s->grid->dy;
|
||||
double dz2 = s->grid->dz * s->grid->dz;
|
||||
double idx2 = 1.0 / dx2;
|
||||
double idy2 = 1.0 / dy2;
|
||||
double idz2 = 1.0 / dz2;
|
||||
double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
|
||||
(dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
|
||||
double* r = s->r[level];
|
||||
double epssq = eps * eps;
|
||||
int it = 0;
|
||||
int pass, ksw, jsw, isw;
|
||||
double res = 1.0;
|
||||
|
||||
ksw = 1;
|
||||
|
||||
for (pass = 0; pass < 2; pass++) {
|
||||
jsw = ksw;
|
||||
|
||||
commExchange(comm, p);
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
isw = jsw;
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = isw; i < imaxLocal + 1; i += 2) {
|
||||
|
||||
R(i,
|
||||
j,
|
||||
k) = (RHS(i, j, k) -
|
||||
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) *
|
||||
idx2 +
|
||||
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
|
||||
idy2 +
|
||||
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
|
||||
idz2));
|
||||
|
||||
res += (R(i, j, k) * R(i, j, k));
|
||||
}
|
||||
isw = 3 - isw;
|
||||
}
|
||||
jsw = 3 - jsw;
|
||||
}
|
||||
ksw = 3 - ksw;
|
||||
}
|
||||
|
||||
commReduction(&res, SUM);
|
||||
|
||||
res = res / (double)(imaxLocal * jmaxLocal * kmaxLocal);
|
||||
#ifdef DEBUG
|
||||
if (commIsMaster(s->comm)) {
|
||||
printf("%d Residuum: %e\n", it, res);
|
||||
}
|
||||
#endif
|
||||
return res;
|
||||
}
|
||||
|
||||
static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm)
|
||||
{
|
||||
int imaxLocal = comm->imaxLocal;
|
||||
int jmaxLocal = comm->jmaxLocal;
|
||||
int kmaxLocal = comm->kmaxLocal;
|
||||
|
||||
double res = 0.0;
|
||||
|
||||
// coarsest level
|
||||
if (level == COARSEST_LEVEL) {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
smooth(s, p, rhs, level, comm);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
// pre-smoothing
|
||||
for (int i = 0; i < s->presmooth; i++) {
|
||||
smooth(s, p, rhs, level, comm);
|
||||
if (level == FINEST_LEVEL)
|
||||
setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal);
|
||||
}
|
||||
|
||||
res = calculateResidual(s, p, rhs, level, comm);
|
||||
|
||||
// restrict
|
||||
restrictMG(s, level, comm);
|
||||
|
||||
Comm newcomm;
|
||||
commUpdateDatatypes(s->comm,
|
||||
&newcomm,
|
||||
imaxLocal,
|
||||
jmaxLocal,
|
||||
kmaxLocal);
|
||||
|
||||
// MGSolver on residual and error.
|
||||
multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm);
|
||||
|
||||
commFreeCommunicator(&newcomm);
|
||||
|
||||
// prolongate
|
||||
prolongate(s, level, comm);
|
||||
|
||||
// correct p on finer level using residual
|
||||
correct(s, p, level, comm);
|
||||
if (level == FINEST_LEVEL)
|
||||
setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal);
|
||||
|
||||
// post-smoothing
|
||||
for (int i = 0; i < s->postsmooth; i++) {
|
||||
smooth(s, p, rhs, level, comm);
|
||||
if (level == FINEST_LEVEL)
|
||||
setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
{
|
||||
s->eps = p->eps;
|
||||
s->omega = p->omg;
|
||||
s->itermax = p->itermax;
|
||||
s->levels = p->levels;
|
||||
s->grid = &d->grid;
|
||||
s->presmooth = p->presmooth;
|
||||
s->postsmooth = p->postsmooth;
|
||||
s->comm = &d->comm;
|
||||
s->problem = p->name;
|
||||
|
||||
int imax = s->grid->imax;
|
||||
int jmax = s->grid->jmax;
|
||||
int kmax = s->grid->kmax;
|
||||
int levels = s->levels;
|
||||
if (commIsMaster(s->comm)) printf("Using Multigrid solver with %d levels\n", levels);
|
||||
|
||||
s->r = malloc(levels * sizeof(double*));
|
||||
s->e = malloc(levels * sizeof(double*));
|
||||
|
||||
size_t size = (imax + 2) * (jmax + 2) * (kmax + 2);
|
||||
|
||||
for (int j = 0; j < levels; j++) {
|
||||
s->r[j] = allocate(64, size * sizeof(double));
|
||||
s->e[j] = allocate(64, size * sizeof(double));
|
||||
|
||||
for (size_t i = 0; i < size; i++) {
|
||||
s->r[j][i] = 0.0;
|
||||
s->e[j][i] = 0.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
double solve(Solver* s, double* p, double* rhs)
|
||||
{
|
||||
double res = multiGrid(s, p, rhs, 0, s->comm);
|
||||
|
||||
#ifdef VERBOSE
|
||||
if (commIsMaster(s->comm)) {
|
||||
printf("Residuum: %.6f\n", res);
|
||||
}
|
||||
#endif
|
||||
|
||||
return res;
|
||||
}
|
175
BasicSolver/3D-mpi/src/solver-rb.c
Normal file
175
BasicSolver/3D-mpi/src/solver-rb.c
Normal file
@ -0,0 +1,175 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <float.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "allocate.h"
|
||||
#include "comm.h"
|
||||
#include "parameter.h"
|
||||
#include "solver.h"
|
||||
#include "util.h"
|
||||
|
||||
void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
{
|
||||
s->eps = p->eps;
|
||||
s->omega = p->omg;
|
||||
s->itermax = p->itermax;
|
||||
s->levels = p->levels;
|
||||
s->grid = &d->grid;
|
||||
s->presmooth = p->presmooth;
|
||||
s->postsmooth = p->postsmooth;
|
||||
s->comm = &d->comm;
|
||||
s->problem = p->name;
|
||||
}
|
||||
|
||||
double solve(Solver* s, double* p, double* rhs)
|
||||
{
|
||||
int imaxLocal = s->comm->imaxLocal;
|
||||
int jmaxLocal = s->comm->jmaxLocal;
|
||||
int kmaxLocal = s->comm->kmaxLocal;
|
||||
|
||||
int imax = s->grid->imax;
|
||||
int jmax = s->grid->jmax;
|
||||
int kmax = s->grid->kmax;
|
||||
|
||||
double eps = s->eps;
|
||||
int itermax = s->itermax;
|
||||
double dx2 = s->grid->dx * s->grid->dx;
|
||||
double dy2 = s->grid->dy * s->grid->dy;
|
||||
double dz2 = s->grid->dz * s->grid->dz;
|
||||
double idx2 = 1.0 / dx2;
|
||||
double idy2 = 1.0 / dy2;
|
||||
double idz2 = 1.0 / dz2;
|
||||
|
||||
double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
|
||||
(dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
|
||||
double epssq = eps * eps;
|
||||
int it = 0;
|
||||
double res = 1.0;
|
||||
int pass, ksw, jsw, isw;
|
||||
|
||||
while ((res >= epssq) && (it < itermax)) {
|
||||
ksw = 1;
|
||||
|
||||
for (pass = 0; pass < 2; pass++) {
|
||||
jsw = ksw;
|
||||
commExchange(s->comm, p);
|
||||
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
isw = jsw;
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = isw; i < imaxLocal + 1; i += 2) {
|
||||
|
||||
double r =
|
||||
RHS(i, j, k) -
|
||||
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
|
||||
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
|
||||
idy2 +
|
||||
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
|
||||
idz2);
|
||||
|
||||
P(i, j, k) -= (factor * r);
|
||||
res += (r * r);
|
||||
}
|
||||
isw = 3 - isw;
|
||||
}
|
||||
jsw = 3 - jsw;
|
||||
}
|
||||
ksw = 3 - ksw;
|
||||
}
|
||||
#ifdef _MPI
|
||||
if (commIsBoundary(s->comm, FRONT)) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, j, 0) = P(i, j, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, BACK)) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, BOTTOM)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, 0, k) = P(i, 1, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, TOP)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, LEFT)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
P(0, j, k) = P(1, j, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(s->comm, RIGHT)) {
|
||||
for (int k = 1; k < kmaxLocal + 1; k++) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
P(imaxLocal + 1, j, k) = P(imaxLocal, j, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
for (int j = 1; j < jmax + 1; j++) {
|
||||
for (int i = 1; i < imax + 1; i++) {
|
||||
P(i, j, 0) = P(i, j, 1);
|
||||
P(i, j, kmax + 1) = P(i, j, kmax);
|
||||
}
|
||||
}
|
||||
|
||||
for (int k = 1; k < kmax + 1; k++) {
|
||||
for (int i = 1; i < imax + 1; i++) {
|
||||
P(i, 0, k) = P(i, 1, k);
|
||||
P(i, jmax + 1, k) = P(i, jmax, k);
|
||||
}
|
||||
}
|
||||
|
||||
for (int k = 1; k < kmax + 1; k++) {
|
||||
for (int j = 1; j < jmax + 1; j++) {
|
||||
P(0, j, k) = P(1, j, k);
|
||||
P(imax + 1, j, k) = P(imax, j, k);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
commReduction(&res, SUM);
|
||||
res = res / (double)(imax * jmax * kmax);
|
||||
#ifdef DEBUG
|
||||
if (commIsMaster(&s->comm)) {
|
||||
printf("%d Residuum: %e\n", it, res);
|
||||
}
|
||||
#endif
|
||||
commExchange(s->comm, p);
|
||||
it++;
|
||||
}
|
||||
|
||||
#ifdef VERBOSE
|
||||
if (commIsMaster(s->comm)) {
|
||||
printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
|
||||
}
|
||||
#endif
|
||||
|
||||
return res;
|
||||
}
|
@ -9,12 +9,11 @@
|
||||
#include "comm.h"
|
||||
#include "grid.h"
|
||||
#include "parameter.h"
|
||||
|
||||
enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
|
||||
#include "discretization.h"
|
||||
|
||||
typedef struct {
|
||||
/* geometry and grid information */
|
||||
Grid grid;
|
||||
Grid* grid;
|
||||
/* arrays */
|
||||
double *p, *rhs;
|
||||
double *f, *g, *h;
|
||||
@ -30,16 +29,11 @@ typedef struct {
|
||||
char* problem;
|
||||
int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
|
||||
/* communication */
|
||||
Comm comm;
|
||||
double **r, **e;
|
||||
int levels, presmooth, postsmooth;
|
||||
Comm* comm;
|
||||
} Solver;
|
||||
|
||||
extern void initSolver(Solver*, Parameter*);
|
||||
extern void computeRHS(Solver*);
|
||||
extern void solve(Solver*);
|
||||
extern void normalizePressure(Solver*);
|
||||
extern void computeTimestep(Solver*);
|
||||
extern void setBoundaryConditions(Solver*);
|
||||
extern void setSpecialBoundaryCondition(Solver*);
|
||||
extern void computeFG(Solver*);
|
||||
extern void adaptUV(Solver*);
|
||||
extern double solve(Solver* , double* , double* );
|
||||
extern void initSolver(Solver*, Discretization*, Parameter*);
|
||||
#endif
|
||||
|
@ -14,10 +14,10 @@
|
||||
|
||||
void testInit(Solver* s)
|
||||
{
|
||||
int imaxLocal = s->comm.imaxLocal;
|
||||
int jmaxLocal = s->comm.jmaxLocal;
|
||||
int kmaxLocal = s->comm.kmaxLocal;
|
||||
int myrank = s->comm.rank;
|
||||
int imaxLocal = s->comm->imaxLocal;
|
||||
int jmaxLocal = s->comm->jmaxLocal;
|
||||
int kmaxLocal = s->comm->kmaxLocal;
|
||||
int myrank = s->comm->rank;
|
||||
double* p = s->p;
|
||||
double* f = s->f;
|
||||
double* g = s->g;
|
||||
@ -76,11 +76,11 @@ static char* direction2String(Direction dir)
|
||||
|
||||
static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir)
|
||||
{
|
||||
int imaxLocal = s->comm.imaxLocal;
|
||||
int jmaxLocal = s->comm.jmaxLocal;
|
||||
int kmaxLocal = s->comm.kmaxLocal;
|
||||
int imaxLocal = s->comm->imaxLocal;
|
||||
int jmaxLocal = s->comm->jmaxLocal;
|
||||
int kmaxLocal = s->comm->kmaxLocal;
|
||||
char filename[50];
|
||||
snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm.rank);
|
||||
snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm->rank);
|
||||
FILE* fh = fopen(filename, "w");
|
||||
|
||||
for (int y = 0; y < ymax; y++) {
|
||||
@ -116,9 +116,9 @@ static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir)
|
||||
|
||||
void testPrintHalo(Solver* s, double* a)
|
||||
{
|
||||
int imaxLocal = s->comm.imaxLocal;
|
||||
int jmaxLocal = s->comm.jmaxLocal;
|
||||
int kmaxLocal = s->comm.kmaxLocal;
|
||||
int imaxLocal = s->comm->imaxLocal;
|
||||
int jmaxLocal = s->comm->jmaxLocal;
|
||||
int kmaxLocal = s->comm->kmaxLocal;
|
||||
|
||||
printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, BOTTOM);
|
||||
printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, TOP);
|
||||
|
@ -19,4 +19,13 @@
|
||||
#define ABS(a) ((a) >= 0 ? (a) : -(a))
|
||||
#endif
|
||||
|
||||
#define P(i, j, k) p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define F(i, j, k) f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define G(i, j, k) g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define H(i, j, k) h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define U(i, j, k) u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define V(i, j, k) v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define W(i, j, k) w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
#define RHS(i, j, k) rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
|
||||
|
||||
#endif // __UTIL_H_
|
||||
|
@ -46,18 +46,18 @@ static void writeHeader(VtkOptions* o)
|
||||
cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n");
|
||||
cursor += sprintf(cursor,
|
||||
"DIMENSIONS %d %d %d\n",
|
||||
o->grid.imax,
|
||||
o->grid.jmax,
|
||||
o->grid.kmax);
|
||||
o->grid->imax,
|
||||
o->grid->jmax,
|
||||
o->grid->kmax);
|
||||
cursor += sprintf(cursor,
|
||||
"ORIGIN %f %f %f\n",
|
||||
o->grid.dx * 0.5,
|
||||
o->grid.dy * 0.5,
|
||||
o->grid.dz * 0.5);
|
||||
cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz);
|
||||
o->grid->dx * 0.5,
|
||||
o->grid->dy * 0.5,
|
||||
o->grid->dz * 0.5);
|
||||
cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz);
|
||||
cursor += sprintf(cursor,
|
||||
"POINT_DATA %d\n",
|
||||
o->grid.imax * o->grid.jmax * o->grid.kmax);
|
||||
o->grid->imax * o->grid->jmax * o->grid->kmax);
|
||||
|
||||
if (commIsMaster(&o->comm)) {
|
||||
MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
|
||||
@ -67,7 +67,6 @@ static void writeHeader(VtkOptions* o)
|
||||
void vtkOpen(VtkOptions* o, char* problem)
|
||||
{
|
||||
char filename[50];
|
||||
|
||||
snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size);
|
||||
MPI_File_open(o->comm.comm,
|
||||
filename,
|
||||
@ -98,7 +97,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
|
||||
}
|
||||
|
||||
int offsets[NDIMS];
|
||||
commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax);
|
||||
commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax);
|
||||
|
||||
// set global view in file
|
||||
MPI_Offset disp;
|
||||
@ -108,7 +107,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
|
||||
MPI_File_get_size(o->fh, &disp);
|
||||
|
||||
MPI_Type_create_subarray(NDIMS,
|
||||
(int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax },
|
||||
(int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax },
|
||||
(int[NDIMS]) { o->comm.kmaxLocal, o->comm.jmaxLocal, o->comm.imaxLocal },
|
||||
offsets,
|
||||
MPI_ORDER_C,
|
||||
@ -177,7 +176,7 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec)
|
||||
}
|
||||
|
||||
int offsets[NDIMS];
|
||||
commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax);
|
||||
commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax);
|
||||
|
||||
// set global view in file
|
||||
MPI_Offset disp;
|
||||
@ -190,7 +189,7 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec)
|
||||
MPI_Type_commit(&vectorType);
|
||||
|
||||
MPI_Type_create_subarray(NDIMS,
|
||||
(int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax },
|
||||
(int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax },
|
||||
(int[NDIMS]) { kmaxLocal, jmaxLocal, imaxLocal },
|
||||
offsets,
|
||||
MPI_ORDER_C,
|
||||
|
@ -41,14 +41,14 @@ static void writeHeader(VtkOptions* o)
|
||||
}
|
||||
|
||||
fprintf(o->fh, "DATASET STRUCTURED_POINTS\n");
|
||||
fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid.imax, o->grid.jmax, o->grid.kmax);
|
||||
fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid->imax, o->grid->jmax, o->grid->kmax);
|
||||
fprintf(o->fh,
|
||||
"ORIGIN %f %f %f\n",
|
||||
o->grid.dx * 0.5,
|
||||
o->grid.dy * 0.5,
|
||||
o->grid.dz * 0.5);
|
||||
fprintf(o->fh, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz);
|
||||
fprintf(o->fh, "POINT_DATA %d\n", o->grid.imax * o->grid.jmax * o->grid.kmax);
|
||||
o->grid->dx * 0.5,
|
||||
o->grid->dy * 0.5,
|
||||
o->grid->dz * 0.5);
|
||||
fprintf(o->fh, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz);
|
||||
fprintf(o->fh, "POINT_DATA %d\n", o->grid->imax * o->grid->jmax * o->grid->kmax);
|
||||
}
|
||||
|
||||
void vtkOpen(VtkOptions* o, char* problem)
|
||||
@ -64,9 +64,9 @@ void vtkOpen(VtkOptions* o, char* problem)
|
||||
|
||||
static void writeScalar(VtkOptions* o, double* s)
|
||||
{
|
||||
int imax = o->grid.imax;
|
||||
int jmax = o->grid.jmax;
|
||||
int kmax = o->grid.kmax;
|
||||
int imax = o->grid->imax;
|
||||
int jmax = o->grid->jmax;
|
||||
int kmax = o->grid->kmax;
|
||||
|
||||
for (int k = 0; k < kmax; k++) {
|
||||
for (int j = 0; j < jmax; j++) {
|
||||
@ -105,9 +105,9 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
|
||||
|
||||
static void writeVector(VtkOptions* o, VtkVector vec)
|
||||
{
|
||||
int imax = o->grid.imax;
|
||||
int jmax = o->grid.jmax;
|
||||
int kmax = o->grid.kmax;
|
||||
int imax = o->grid->imax;
|
||||
int jmax = o->grid->jmax;
|
||||
int kmax = o->grid->kmax;
|
||||
|
||||
for (int k = 0; k < kmax; k++) {
|
||||
for (int j = 0; j < jmax; j++) {
|
||||
|
@ -14,7 +14,7 @@
|
||||
typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat;
|
||||
|
||||
typedef struct VtkOptions {
|
||||
Grid grid;
|
||||
Grid* grid;
|
||||
#ifdef _VTK_WRITER_MPI
|
||||
MPI_File fh;
|
||||
#else
|
||||
|
@ -40,6 +40,18 @@ $(BUILD_DIR)/%.s: %.c
|
||||
|
||||
.PHONY: clean distclean tags info asm format
|
||||
|
||||
vis:
|
||||
$(info ===> GENERATE VISUALIZATION)
|
||||
@gnuplot -e "filename='residual.dat'" ./residual.plot
|
||||
|
||||
vis_clean:
|
||||
$(info ===> CLEAN VISUALIZATION)
|
||||
@rm -f *.dat
|
||||
@rm -f *.vtk
|
||||
@rm -f *.png
|
||||
|
||||
clean: vis_clean
|
||||
|
||||
clean:
|
||||
$(info ===> CLEAN)
|
||||
@rm -rf $(BUILD_DIR)
|
||||
|
@ -38,10 +38,17 @@ kmax 50 # number of interior cells in z-direction
|
||||
# Time Data:
|
||||
# ---------
|
||||
|
||||
te 100.0 # final time
|
||||
te 60.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 3 # Multigrid levels
|
||||
presmooth 5 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
|
@ -1,12 +1,12 @@
|
||||
# Supported: GCC, CLANG, ICC
|
||||
TAG ?= CLANG
|
||||
TAG ?= ICC
|
||||
ENABLE_OPENMP ?= false
|
||||
# Supported: sor, mg
|
||||
# Supported: rb, mg
|
||||
SOLVER ?= mg
|
||||
# Run in debug settings
|
||||
DEBUG ?= false
|
||||
|
||||
#Feature options
|
||||
OPTIONS += -DARRAY_ALIGNMENT=64
|
||||
#OPTIONS += -DVERBOSE
|
||||
OPTIONS += -DVERBOSE
|
||||
#OPTIONS += -DDEBUG
|
||||
|
@ -38,10 +38,17 @@ kmax 128 # number of interior cells in z-direction
|
||||
# Time Data:
|
||||
# ---------
|
||||
|
||||
te 2.0 # final time
|
||||
te 10.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 3 # Multigrid levels
|
||||
presmooth 20 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
@ -50,5 +57,4 @@ eps 0.001 # stopping tolerance for pressure iteration
|
||||
rho 0.5
|
||||
omg 1.7 # relaxation parameter for SOR iteration
|
||||
gamma 0.9 # upwind differencing factor gamma
|
||||
levels 5 # Multigrid levels
|
||||
#===============================================================================
|
||||
|
9
BasicSolver/3D-seq/residual.plot
Normal file
9
BasicSolver/3D-seq/residual.plot
Normal file
@ -0,0 +1,9 @@
|
||||
set terminal png size 1800,768 enhanced font ,12
|
||||
set output 'residual.png'
|
||||
set datafile separator whitespace
|
||||
set xlabel "Timestep"
|
||||
set ylabel "Residual"
|
||||
|
||||
set logscale y 2
|
||||
|
||||
plot 'residual.dat' using 1:2 title "Residual"
|
@ -106,7 +106,7 @@ void initDiscretization(Discretization* d, Parameter* p)
|
||||
d->dtBound = 0.5 * d->re * 1.0 / invSqrSum;
|
||||
|
||||
#ifdef VERBOSE
|
||||
printConfig(s);
|
||||
printConfig(d);
|
||||
#endif /* VERBOSE */
|
||||
}
|
||||
|
||||
|
@ -73,6 +73,9 @@ int main(int argc, char** argv)
|
||||
Solver s;
|
||||
initParameter(&p);
|
||||
|
||||
FILE* fp;
|
||||
fp = initResidualWriter();
|
||||
|
||||
if (argc != 2) {
|
||||
printf("Usage: %s <configFile>\n", argv[0]);
|
||||
exit(EXIT_SUCCESS);
|
||||
@ -90,6 +93,7 @@ int main(int argc, char** argv)
|
||||
double te = d.te;
|
||||
double t = 0.0;
|
||||
int nt = 0;
|
||||
double res = 0.0;
|
||||
|
||||
timeStart = getTimeStamp();
|
||||
while (t <= te) {
|
||||
@ -99,13 +103,16 @@ int main(int argc, char** argv)
|
||||
computeFG(&d);
|
||||
computeRHS(&d);
|
||||
if (nt % 100 == 0) normalizePressure(&d);
|
||||
solve(&s, d.p, d.rhs);
|
||||
res = solve(&s, d.p, d.rhs);
|
||||
adaptUV(&d);
|
||||
|
||||
writeResidual(fp, t, res);
|
||||
|
||||
t += d.dt;
|
||||
nt++;
|
||||
|
||||
#ifdef VERBOSE
|
||||
printf("TIME %f , TIMESTEP %f\n", t, solver.dt);
|
||||
printf("TIME %f , TIMESTEP %f\n", t, d.dt);
|
||||
#else
|
||||
printProgress(t);
|
||||
#endif
|
||||
@ -123,7 +130,8 @@ int main(int argc, char** argv)
|
||||
ug = allocate(64, bytesize);
|
||||
vg = allocate(64, bytesize);
|
||||
wg = allocate(64, bytesize);
|
||||
|
||||
|
||||
fclose(fp);
|
||||
createBulkArrays(&d, pg, ug, vg, wg);
|
||||
VtkOptions opts = { .grid = d.grid };
|
||||
vtkOpen(&opts, d.problem);
|
||||
|
@ -14,19 +14,21 @@
|
||||
|
||||
void initParameter(Parameter* param)
|
||||
{
|
||||
param->xlength = 1.0;
|
||||
param->ylength = 1.0;
|
||||
param->zlength = 1.0;
|
||||
param->imax = 100;
|
||||
param->jmax = 100;
|
||||
param->kmax = 100;
|
||||
param->itermax = 1000;
|
||||
param->eps = 0.0001;
|
||||
param->omg = 1.7;
|
||||
param->re = 100.0;
|
||||
param->gamma = 0.9;
|
||||
param->tau = 0.5;
|
||||
param->levels = 5;
|
||||
param->xlength = 1.0;
|
||||
param->ylength = 1.0;
|
||||
param->zlength = 1.0;
|
||||
param->imax = 100;
|
||||
param->jmax = 100;
|
||||
param->kmax = 100;
|
||||
param->itermax = 1000;
|
||||
param->eps = 0.0001;
|
||||
param->omg = 1.7;
|
||||
param->re = 100.0;
|
||||
param->gamma = 0.9;
|
||||
param->tau = 0.5;
|
||||
param->levels = 5;
|
||||
param->presmooth = 5;
|
||||
param->postsmooth = 5;
|
||||
}
|
||||
|
||||
void readParameter(Parameter* param, const char* filename)
|
||||
|
@ -18,6 +18,7 @@ typedef struct {
|
||||
char* name;
|
||||
int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
|
||||
double u_init, v_init, w_init, p_init;
|
||||
int presmooth, postsmooth;
|
||||
} Parameter;
|
||||
|
||||
void initParameter(Parameter*);
|
||||
|
@ -49,3 +49,22 @@ void stopProgress()
|
||||
printf("\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
FILE* initResidualWriter()
|
||||
{
|
||||
FILE* fp;
|
||||
fp = fopen("residual.dat", "w");
|
||||
|
||||
if (fp == NULL) {
|
||||
printf("Error!\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
return fp;
|
||||
|
||||
}
|
||||
|
||||
void writeResidual(FILE* fp, double ts, double res)
|
||||
{
|
||||
fprintf(fp, "%f, %f\n", ts, res);
|
||||
}
|
@ -10,5 +10,6 @@
|
||||
extern void initProgress(double);
|
||||
extern void printProgress(double);
|
||||
extern void stopProgress(void);
|
||||
|
||||
extern FILE* initResidualWriter(void);
|
||||
extern void writeResidual(FILE*, double, double);
|
||||
#endif
|
||||
|
@ -141,13 +141,62 @@ static double smooth(
|
||||
for (int j = 1; j < jmax + 1; j++) {
|
||||
for (int i = isw; i < imax + 1; i += 2) {
|
||||
|
||||
R(i, j, k) =
|
||||
RHS(i, j, k) -
|
||||
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
|
||||
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * idy2 +
|
||||
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * idz2);
|
||||
P(i, j, k) -=
|
||||
factor *
|
||||
(RHS(i, j, k) -
|
||||
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
|
||||
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
|
||||
idy2 +
|
||||
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
|
||||
idz2));
|
||||
}
|
||||
isw = 3 - isw;
|
||||
}
|
||||
jsw = 3 - jsw;
|
||||
}
|
||||
ksw = 3 - ksw;
|
||||
}
|
||||
}
|
||||
|
||||
static double calculateResidual(
|
||||
Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax)
|
||||
{
|
||||
double eps = s->eps;
|
||||
int itermax = s->itermax;
|
||||
double dx2 = s->grid->dx * s->grid->dx;
|
||||
double dy2 = s->grid->dy * s->grid->dy;
|
||||
double dz2 = s->grid->dz * s->grid->dz;
|
||||
double idx2 = 1.0 / dx2;
|
||||
double idy2 = 1.0 / dy2;
|
||||
double idz2 = 1.0 / dz2;
|
||||
double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
|
||||
(dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
|
||||
double* r = s->r[level];
|
||||
double epssq = eps * eps;
|
||||
int it = 0;
|
||||
int pass, ksw, jsw, isw;
|
||||
double res = 1.0;
|
||||
|
||||
ksw = 1;
|
||||
|
||||
for (pass = 0; pass < 2; pass++) {
|
||||
jsw = ksw;
|
||||
|
||||
for (int k = 1; k < kmax + 1; k++) {
|
||||
isw = jsw;
|
||||
for (int j = 1; j < jmax + 1; j++) {
|
||||
for (int i = isw; i < imax + 1; i += 2) {
|
||||
|
||||
R(i,
|
||||
j,
|
||||
k) = (RHS(i, j, k) -
|
||||
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) *
|
||||
idx2 +
|
||||
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
|
||||
idy2 +
|
||||
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
|
||||
idz2));
|
||||
|
||||
P(i, j, k) -= (factor * R(i, j, k));
|
||||
res += (R(i, j, k) * R(i, j, k));
|
||||
}
|
||||
isw = 3 - isw;
|
||||
@ -167,7 +216,7 @@ static double multiGrid(
|
||||
{
|
||||
double res = 0.0;
|
||||
|
||||
// coarsest level TODO: Use direct solver?
|
||||
// coarsest level
|
||||
if (level == COARSEST_LEVEL) {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
smooth(s, p, rhs, level, imax, jmax, kmax);
|
||||
@ -175,17 +224,18 @@ static double multiGrid(
|
||||
return res;
|
||||
}
|
||||
|
||||
// pre-smoothing TODO: Make smoothing steps configurable?
|
||||
for (int i = 0; i < 5; i++) {
|
||||
// pre-smoothing
|
||||
for (int i = 0; i < s->presmooth; i++) {
|
||||
smooth(s, p, rhs, level, imax, jmax, kmax);
|
||||
if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax);
|
||||
}
|
||||
|
||||
res = calculateResidual(s, p, rhs, level, imax, jmax, kmax);
|
||||
|
||||
// restrict
|
||||
restrictMG(s, level, imax, jmax, kmax);
|
||||
|
||||
// MGSolver on residual and error.
|
||||
// TODO: What if there is a rest?
|
||||
multiGrid(s,
|
||||
s->e[level + 1],
|
||||
s->r[level + 1],
|
||||
@ -202,8 +252,8 @@ static double multiGrid(
|
||||
if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax);
|
||||
|
||||
// post-smoothing
|
||||
for (int i = 0; i < 5; i++) {
|
||||
res = smooth(s, p, rhs, level, imax, jmax, kmax);
|
||||
for (int i = 0; i < s->postsmooth; i++) {
|
||||
smooth(s, p, rhs, level, imax, jmax, kmax);
|
||||
if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax);
|
||||
}
|
||||
|
||||
@ -212,11 +262,13 @@ static double multiGrid(
|
||||
|
||||
void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
{
|
||||
s->eps = p->eps;
|
||||
s->omega = p->omg;
|
||||
s->itermax = p->itermax;
|
||||
s->levels = p->levels;
|
||||
s->grid = &d->grid;
|
||||
s->eps = p->eps;
|
||||
s->omega = p->omg;
|
||||
s->itermax = p->itermax;
|
||||
s->levels = p->levels;
|
||||
s->grid = &d->grid;
|
||||
s->presmooth = p->presmooth;
|
||||
s->postsmooth = p->postsmooth;
|
||||
|
||||
int imax = s->grid->imax;
|
||||
int jmax = s->grid->jmax;
|
||||
@ -240,11 +292,13 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
}
|
||||
}
|
||||
|
||||
void solve(Solver* s, double* p, double* rhs)
|
||||
double solve(Solver* s, double* p, double* rhs)
|
||||
{
|
||||
double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax, s->grid->kmax);
|
||||
|
||||
#ifdef VERBOSE
|
||||
printf("Residuum: %.6f\n", res);
|
||||
#endif
|
||||
|
||||
return res;
|
||||
}
|
||||
|
101
BasicSolver/3D-seq/src/solver-rb.c
Normal file
101
BasicSolver/3D-seq/src/solver-rb.c
Normal file
@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include "solver.h"
|
||||
#include "util.h"
|
||||
|
||||
void initSolver(Solver* s, Discretization* d, Parameter* p)
|
||||
{
|
||||
s->grid = &d->grid;
|
||||
s->itermax = p->itermax;
|
||||
s->eps = p->eps;
|
||||
s->omega = p->omg;
|
||||
}
|
||||
|
||||
double solve(Solver* s, double* p, double* rhs)
|
||||
{
|
||||
int imax = s->grid->imax;
|
||||
int jmax = s->grid->jmax;
|
||||
int kmax = s->grid->kmax;
|
||||
double eps = s->eps;
|
||||
int itermax = s->itermax;
|
||||
double dx2 = s->grid->dx * s->grid->dx;
|
||||
double dy2 = s->grid->dy * s->grid->dy;
|
||||
double dz2 = s->grid->dz * s->grid->dz;
|
||||
double idx2 = 1.0 / dx2;
|
||||
double idy2 = 1.0 / dy2;
|
||||
double idz2 = 1.0 / dz2;
|
||||
double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
|
||||
(dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
|
||||
double epssq = eps * eps;
|
||||
int it = 0;
|
||||
double res = 1.0;
|
||||
int pass, ksw, jsw, isw;
|
||||
|
||||
while ((res >= epssq) && (it < itermax)) {
|
||||
res = 0.0;
|
||||
ksw = 1;
|
||||
|
||||
for (pass = 0; pass < 2; pass++) {
|
||||
jsw = ksw;
|
||||
|
||||
for (int k = 1; k < kmax + 1; k++) {
|
||||
isw = jsw;
|
||||
for (int j = 1; j < jmax + 1; j++) {
|
||||
for (int i = isw; i < imax + 1; i += 2) {
|
||||
|
||||
double r =
|
||||
RHS(i, j, k) -
|
||||
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
|
||||
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
|
||||
idy2 +
|
||||
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
|
||||
idz2);
|
||||
|
||||
P(i, j, k) -= (factor * r);
|
||||
res += (r * r);
|
||||
}
|
||||
isw = 3 - isw;
|
||||
}
|
||||
jsw = 3 - jsw;
|
||||
}
|
||||
ksw = 3 - ksw;
|
||||
}
|
||||
|
||||
for (int j = 1; j < jmax + 1; j++) {
|
||||
for (int i = 1; i < imax + 1; i++) {
|
||||
P(i, j, 0) = P(i, j, 1);
|
||||
P(i, j, kmax + 1) = P(i, j, kmax);
|
||||
}
|
||||
}
|
||||
|
||||
for (int k = 1; k < kmax + 1; k++) {
|
||||
for (int i = 1; i < imax + 1; i++) {
|
||||
P(i, 0, k) = P(i, 1, k);
|
||||
P(i, jmax + 1, k) = P(i, jmax, k);
|
||||
}
|
||||
}
|
||||
|
||||
for (int k = 1; k < kmax + 1; k++) {
|
||||
for (int j = 1; j < jmax + 1; j++) {
|
||||
P(0, j, k) = P(1, j, k);
|
||||
P(imax + 1, j, k) = P(imax, j, k);
|
||||
}
|
||||
}
|
||||
|
||||
res = res / (double)(imax * jmax * kmax);
|
||||
#ifdef DEBUG
|
||||
printf("%d Residuum: %e\n", it, res);
|
||||
#endif
|
||||
it++;
|
||||
}
|
||||
|
||||
#ifdef VERBOSE
|
||||
printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
|
||||
#endif
|
||||
|
||||
return res;
|
||||
}
|
@ -18,9 +18,10 @@ typedef struct {
|
||||
int itermax;
|
||||
int levels;
|
||||
double **r, **e;
|
||||
int presmooth, postsmooth;
|
||||
} Solver;
|
||||
|
||||
extern void initSolver(Solver*, Discretization*, Parameter*);
|
||||
extern void solve(Solver*, double*, double*);
|
||||
extern double solve(Solver*, double*, double*);
|
||||
|
||||
#endif
|
||||
|
@ -70,6 +70,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
fprintf(o->fh, "SCALARS %s float\n", name);
|
||||
fprintf(o->fh, "LOOKUP_TABLE default\n");
|
||||
|
||||
for (int k = 0; k < kmax; k++) {
|
||||
for (int j = 0; j < jmax; j++) {
|
||||
|
88
EnhancedSolver/2D-mpi/Makefile
Normal file
88
EnhancedSolver/2D-mpi/Makefile
Normal file
@ -0,0 +1,88 @@
|
||||
#=======================================================================================
|
||||
# Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
# All rights reserved.
|
||||
# Use of this source code is governed by a MIT-style
|
||||
# license that can be found in the LICENSE file.
|
||||
#=======================================================================================
|
||||
|
||||
#CONFIGURE BUILD SYSTEM
|
||||
TARGET = exe-$(TAG)
|
||||
BUILD_DIR = ./$(TAG)
|
||||
SRC_DIR = ./src
|
||||
MAKE_DIR = ./
|
||||
Q ?= @
|
||||
|
||||
#DO NOT EDIT BELOW
|
||||
include $(MAKE_DIR)/config.mk
|
||||
include $(MAKE_DIR)/include_$(TAG).mk
|
||||
INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR)
|
||||
|
||||
VPATH = $(SRC_DIR)
|
||||
SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c))
|
||||
ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC))
|
||||
OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC))
|
||||
OBJ += $(BUILD_DIR)/comm-$(COMM_TYPE).o
|
||||
OBJ += $(BUILD_DIR)/solver-$(SOLVER).o
|
||||
SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h)
|
||||
CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES)
|
||||
|
||||
${TARGET}: $(BUILD_DIR) $(OBJ)
|
||||
$(info ===> LINKING $(TARGET))
|
||||
$(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS)
|
||||
|
||||
$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk
|
||||
$(info ===> COMPILE $@)
|
||||
$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
|
||||
$(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d
|
||||
|
||||
$(BUILD_DIR)/%.s: %.c
|
||||
$(info ===> GENERATE ASM $@)
|
||||
$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
|
||||
|
||||
.PHONY: clean distclean vis_clean vis tags info asm format
|
||||
|
||||
vis:
|
||||
$(info ===> GENERATE VISUALIZATION)
|
||||
@gnuplot -e "filename='pressure.dat'" ./surface.plot
|
||||
@gnuplot -e "filename='velocity.dat'" ./vector.plot
|
||||
@gnuplot -e "filename='residual.dat'" ./residual.plot
|
||||
|
||||
vis_clean:
|
||||
$(info ===> CLEAN VISUALIZATION)
|
||||
@rm -f *.dat
|
||||
@rm -f *.png
|
||||
@rm -f ./vis_files/*.dat
|
||||
@rm -f ./vis_files/*.gif
|
||||
|
||||
clean: vis_clean
|
||||
$(info ===> CLEAN)
|
||||
@rm -rf $(BUILD_DIR)
|
||||
@rm -f tags
|
||||
|
||||
distclean: clean
|
||||
$(info ===> DIST CLEAN)
|
||||
@rm -f $(TARGET)
|
||||
@rm -f *.dat
|
||||
@rm -f *.png
|
||||
|
||||
info:
|
||||
$(info $(CFLAGS))
|
||||
$(Q)$(CC) $(VERSION)
|
||||
|
||||
asm: $(BUILD_DIR) $(ASM)
|
||||
|
||||
tags:
|
||||
$(info ===> GENERATE TAGS)
|
||||
$(Q)ctags -R
|
||||
|
||||
format:
|
||||
@for src in $(SOURCES) ; do \
|
||||
echo "Formatting $$src" ; \
|
||||
clang-format -i $$src ; \
|
||||
done
|
||||
@echo "Done"
|
||||
|
||||
$(BUILD_DIR):
|
||||
@mkdir $(BUILD_DIR)
|
||||
|
||||
-include $(OBJ:.o=.d)
|
48
EnhancedSolver/2D-mpi/README.md
Normal file
48
EnhancedSolver/2D-mpi/README.md
Normal file
@ -0,0 +1,48 @@
|
||||
# C source skeleton
|
||||
|
||||
## Build
|
||||
|
||||
1. Configure the toolchain and additional options in `config.mk`:
|
||||
```
|
||||
# Supported: GCC, CLANG, ICC
|
||||
TAG ?= GCC
|
||||
ENABLE_OPENMP ?= false
|
||||
|
||||
OPTIONS += -DARRAY_ALIGNMENT=64
|
||||
#OPTIONS += -DVERBOSE_AFFINITY
|
||||
#OPTIONS += -DVERBOSE_DATASIZE
|
||||
#OPTIONS += -DVERBOSE_TIMER
|
||||
```
|
||||
|
||||
The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution.
|
||||
|
||||
|
||||
2. Build with:
|
||||
```
|
||||
make
|
||||
```
|
||||
|
||||
You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set.
|
||||
Intermediate build results are located in the `<TOOLCHAIN>` directory.
|
||||
|
||||
To output the executed commands use:
|
||||
```
|
||||
make Q=
|
||||
```
|
||||
|
||||
3. Clean up with:
|
||||
```
|
||||
make clean
|
||||
```
|
||||
to clean intermediate build results.
|
||||
|
||||
```
|
||||
make distclean
|
||||
```
|
||||
to clean intermediate build results and binary.
|
||||
|
||||
4. (Optional) Generate assembler:
|
||||
```
|
||||
make asm
|
||||
```
|
||||
The assembler files will also be located in the `<TOOLCHAIN>` directory.
|
79
EnhancedSolver/2D-mpi/backstep.par
Normal file
79
EnhancedSolver/2D-mpi/backstep.par
Normal file
@ -0,0 +1,79 @@
|
||||
#==============================================================================
|
||||
# Laminar Canal Flow
|
||||
#==============================================================================
|
||||
|
||||
# Problem specific Data:
|
||||
# ---------------------
|
||||
|
||||
name backstep # name of flow setup
|
||||
|
||||
bcTop 1 # flags for boundary conditions
|
||||
bcBottom 1 # 1 = no-slip 3 = outflow
|
||||
bcLeft 3 # 2 = free-slip 4 = periodic
|
||||
bcRight 3 #
|
||||
|
||||
gx 0.0 # Body forces (e.g. gravity)
|
||||
gy 0.0 #
|
||||
|
||||
re 36000.0 # Reynolds number
|
||||
|
||||
u_init 1.0 # initial value for velocity in x-direction
|
||||
v_init 0.0 # initial value for velocity in y-direction
|
||||
p_init 1.0 # initial value for pressure
|
||||
|
||||
# Geometry Data:
|
||||
# -------------
|
||||
|
||||
xlength 7.0 # domain size in x-direction
|
||||
ylength 1.5 # domain size in y-direction
|
||||
imax 200 # number of interior cells in x-direction
|
||||
jmax 40 # number of interior cells in y-direction
|
||||
|
||||
# Time Data:
|
||||
# ---------
|
||||
|
||||
te 60.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
itermax 500 # maximal number of pressure iteration in one time step
|
||||
eps 0.0001 # stopping tolerance for pressure iteration
|
||||
rho 0.52
|
||||
omg 1.7 # relaxation parameter for SOR iteration
|
||||
gamma 0.9 # upwind differencing factor gamma
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 3 # Multigrid levels
|
||||
presmooth 55 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Particle Tracing Data:
|
||||
# -----------------------
|
||||
|
||||
numberOfParticles 500
|
||||
startTime 0 #if you want to see particles trapped in recirculation zone, startTime should be set to 0
|
||||
injectTimePeriod 1.0
|
||||
writeTimePeriod 0.5
|
||||
|
||||
x1 0.0
|
||||
y1 0.5
|
||||
x2 0.0
|
||||
y2 1.5
|
||||
|
||||
# Obstacle Geometry Data:
|
||||
# -----------------------
|
||||
# Shape 0 disable, 1 Rectangle/Square, 2 Circle
|
||||
|
||||
shape 1
|
||||
xCenter 0.0
|
||||
yCenter 0.0
|
||||
xRectLength 2.0
|
||||
yRectLength 1.0
|
||||
circleRadius 1.0
|
||||
|
||||
#===============================================================================
|
78
EnhancedSolver/2D-mpi/canal.par
Normal file
78
EnhancedSolver/2D-mpi/canal.par
Normal file
@ -0,0 +1,78 @@
|
||||
#==============================================================================
|
||||
# Laminar Canal Flow
|
||||
#==============================================================================
|
||||
|
||||
# Problem specific Data:
|
||||
# ---------------------
|
||||
|
||||
name canal # name of flow setup
|
||||
|
||||
bcTop 1 # flags for boundary conditions
|
||||
bcBottom 1 # 1 = no-slip 3 = outflow
|
||||
bcLeft 3 # 2 = free-slip 4 = periodic
|
||||
bcRight 3 #
|
||||
|
||||
gx 0.0 # Body forces (e.g. gravity)
|
||||
gy 0.0 #
|
||||
|
||||
re 100.0 # Reynolds number
|
||||
|
||||
u_init 1.0 # initial value for velocity in x-direction
|
||||
v_init 0.0 # initial value for velocity in y-direction
|
||||
p_init 0.0 # initial value for pressure
|
||||
|
||||
# Geometry Data:
|
||||
# -------------
|
||||
|
||||
xlength 30.0 # domain size in x-direction
|
||||
ylength 4.0 # domain size in y-direction
|
||||
imax 200 # number of interior cells in x-direction
|
||||
jmax 40 # number of interior cells in y-direction
|
||||
|
||||
# Time Data:
|
||||
# ---------
|
||||
|
||||
te 60.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 3 # Multigrid levels
|
||||
presmooth 5 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
itermax 500 # maximal number of pressure iteration in one time step
|
||||
eps 0.00001 # stopping tolerance for pressure iteration
|
||||
omg 1.7 # relaxation parameter for SOR iteration
|
||||
gamma 0.9 # upwind differencing factor gamma
|
||||
|
||||
# Particle Tracing Data:
|
||||
# -----------------------
|
||||
|
||||
numberOfParticles 60
|
||||
startTime 10.0
|
||||
injectTimePeriod 4.0
|
||||
writeTimePeriod 1.0
|
||||
|
||||
x1 1.0
|
||||
y1 0.0
|
||||
x2 1.0
|
||||
y2 4.0
|
||||
|
||||
# Obstacle Geometry Data:
|
||||
# -----------------------
|
||||
# Shape 0 disable, 1 Rectangle/Square, 2 Circle
|
||||
|
||||
shape 0
|
||||
xCenter 10.0
|
||||
yCenter 2
|
||||
xRectLength 6.0
|
||||
yRectLength 1.0
|
||||
circleRadius 1.0
|
||||
|
||||
#===============================================================================
|
17
EnhancedSolver/2D-mpi/config.mk
Normal file
17
EnhancedSolver/2D-mpi/config.mk
Normal file
@ -0,0 +1,17 @@
|
||||
# Supported: GCC, CLANG, ICC
|
||||
TAG ?= ICC
|
||||
# Supported: true, false
|
||||
ENABLE_MPI ?= true
|
||||
ENABLE_OPENMP ?= false
|
||||
# Supported: rb, mg
|
||||
SOLVER ?= mg
|
||||
# Run in debug settings ?= mg
|
||||
COMM_TYPE ?= v3
|
||||
|
||||
#Feature options
|
||||
OPTIONS += -DARRAY_ALIGNMENT=64
|
||||
OPTIONS += -DVERBOSE
|
||||
# OPTIONS += -DTEST
|
||||
#OPTIONS += -DVERBOSE_AFFINITY
|
||||
#OPTIONS += -DVERBOSE_DATASIZE
|
||||
#OPTIONS += -DVERBOSE_TIMER
|
79
EnhancedSolver/2D-mpi/dcavity.par
Normal file
79
EnhancedSolver/2D-mpi/dcavity.par
Normal file
@ -0,0 +1,79 @@
|
||||
#==============================================================================
|
||||
# Driven Cavity
|
||||
#==============================================================================
|
||||
|
||||
# Problem specific Data:
|
||||
# ---------------------
|
||||
|
||||
name dcavity # name of flow setup
|
||||
|
||||
bcTop 1 # flags for boundary conditions
|
||||
bcBottom 1 # 1 = no-slip 3 = outflow
|
||||
bcLeft 1 # 2 = free-slip 4 = periodic
|
||||
bcRight 1 #
|
||||
|
||||
gx 0.0 # Body forces (e.g. gravity)
|
||||
gy 0.0 #
|
||||
|
||||
re 10.0 # Reynolds number
|
||||
|
||||
u_init 1.0 # initial value for velocity in x-direction
|
||||
v_init 0.0 # initial value for velocity in y-direction
|
||||
p_init 0.0 # initial value for pressure
|
||||
|
||||
# Geometry Data:
|
||||
# -------------
|
||||
|
||||
xlength 1.0 # domain size in x-direction
|
||||
ylength 1.0 # domain size in y-direction
|
||||
imax 128 # number of interior cells in x-direction
|
||||
jmax 128 # number of interior cells in y-direction
|
||||
|
||||
# Time Data:
|
||||
# ---------
|
||||
|
||||
te 10.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
itermax 1000 # maximal number of pressure iteration in one time step
|
||||
eps 0.001 # stopping tolerance for pressure iteration
|
||||
rho 0.5
|
||||
omg 1.7 # relaxation parameter for SOR iteration
|
||||
gamma 0.9 # upwind differencing factor gamma
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 3 # Multigrid levels
|
||||
presmooth 20 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Particle Tracing Data:
|
||||
# -----------------------
|
||||
|
||||
numberOfParticles 200
|
||||
startTime 2.0
|
||||
injectTimePeriod 0.5
|
||||
writeTimePeriod 0.2
|
||||
|
||||
x1 0.1
|
||||
y1 0.9
|
||||
x2 0.9
|
||||
y2 0.9
|
||||
|
||||
|
||||
# Obstacle Geometry Data:
|
||||
# -----------------------
|
||||
# Shape 0 disable, 1 Rectangle/Square, 2 Circle
|
||||
|
||||
shape 0
|
||||
xCenter 0.5
|
||||
yCenter 0.5
|
||||
xRectLength 0.5
|
||||
yRectLength 0.5
|
||||
circleRadius 0.5
|
||||
#===============================================================================
|
21
EnhancedSolver/2D-mpi/include_CLANG.mk
Normal file
21
EnhancedSolver/2D-mpi/include_CLANG.mk
Normal file
@ -0,0 +1,21 @@
|
||||
ifeq ($(ENABLE_MPI),true)
|
||||
CC = mpicc
|
||||
DEFINES = -D_MPI
|
||||
else
|
||||
CC = cc
|
||||
endif
|
||||
|
||||
GCC = cc
|
||||
LINKER = $(CC)
|
||||
|
||||
ifeq ($(ENABLE_OPENMP),true)
|
||||
OPENMP = -fopenmp
|
||||
#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp
|
||||
LIBS = # -lomp
|
||||
endif
|
||||
|
||||
VERSION = --version
|
||||
CFLAGS = -Ofast -std=c17
|
||||
LFLAGS = $(OPENMP) -lm
|
||||
DEFINES += -D_GNU_SOURCE# -DDEBUG
|
||||
INCLUDES = -I/opt/homebrew/include
|
20
EnhancedSolver/2D-mpi/include_GCC.mk
Normal file
20
EnhancedSolver/2D-mpi/include_GCC.mk
Normal file
@ -0,0 +1,20 @@
|
||||
ifeq ($(ENABLE_MPI),true)
|
||||
CC = mpicc
|
||||
DEFINES = -D_MPI
|
||||
else
|
||||
CC = gcc
|
||||
endif
|
||||
|
||||
GCC = gcc
|
||||
LINKER = $(CC)
|
||||
|
||||
ifeq ($(ENABLE_OPENMP),true)
|
||||
OPENMP = -fopenmp
|
||||
endif
|
||||
|
||||
VERSION = --version
|
||||
CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP)
|
||||
LFLAGS = $(OPENMP)
|
||||
DEFINES += -D_GNU_SOURCE
|
||||
INCLUDES =
|
||||
LIBS =
|
79
EnhancedSolver/2D-mpi/karman.par
Normal file
79
EnhancedSolver/2D-mpi/karman.par
Normal file
@ -0,0 +1,79 @@
|
||||
#==============================================================================
|
||||
# Laminar Canal Flow
|
||||
#==============================================================================
|
||||
|
||||
# Problem specific Data:
|
||||
# ---------------------
|
||||
|
||||
name karman # name of flow setup
|
||||
|
||||
bcTop 1 # flags for boundary conditions
|
||||
bcBottom 1 # 1 = no-slip 3 = outflow
|
||||
bcLeft 3 # 2 = free-slip 4 = periodic
|
||||
bcRight 3 #
|
||||
|
||||
gx 0.0 # Body forces (e.g. gravity)
|
||||
gy 0.0 #
|
||||
|
||||
re 5050.0 # Reynolds number
|
||||
|
||||
u_init 1.0 # initial value for velocity in x-direction
|
||||
v_init 0.0 # initial value for velocity in y-direction
|
||||
p_init 0.0 # initial value for pressure
|
||||
|
||||
# Geometry Data:
|
||||
# -------------
|
||||
|
||||
xlength 30.0 # domain size in x-direction
|
||||
ylength 8.0 # domain size in y-direction
|
||||
imax 400 # number of interior cells in x-direction
|
||||
jmax 200 # number of interior cells in y-direction
|
||||
|
||||
# Time Data:
|
||||
# ---------
|
||||
|
||||
te 150.0 # final time
|
||||
dt 0.02 # time stepsize
|
||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||
|
||||
# Pressure Iteration Data:
|
||||
# -----------------------
|
||||
|
||||
itermax 200 # maximal number of pressure iteration in one time step
|
||||
eps 0.001 # stopping tolerance for pressure iteration
|
||||
rho 0.52
|
||||
omg 1.75 # relaxation parameter for SOR iteration
|
||||
gamma 0.9 # upwind differencing factor gamma
|
||||
|
||||
# Multigrid data:
|
||||
# ---------
|
||||
|
||||
levels 3 # Multigrid levels
|
||||
presmooth 15 # Pre-smoothning iterations
|
||||
postsmooth 5 # Post-smoothning iterations
|
||||
|
||||
# Particle Tracing Data:
|
||||
# -----------------------
|
||||
|
||||
numberOfParticles 200
|
||||
startTime 50
|
||||
injectTimePeriod 1.0
|
||||
writeTimePeriod 0.5
|
||||
|
||||
x1 0.0
|
||||
y1 3.8
|
||||
x2 0.0
|
||||
y2 4.1
|
||||
|
||||
# Obstacle Geometry Data:
|
||||
# -----------------------
|
||||
# Shape 0 disable, 1 Rectangle/Square, 2 Circle
|
||||
|
||||
shape 2
|
||||
xCenter 5.0
|
||||
yCenter 4.0
|
||||
xRectLength 2.0
|
||||
yRectLength 1.0
|
||||
circleRadius 1.0
|
||||
|
||||
#===============================================================================
|
9
EnhancedSolver/2D-mpi/residual.plot
Normal file
9
EnhancedSolver/2D-mpi/residual.plot
Normal file
@ -0,0 +1,9 @@
|
||||
set terminal png size 1800,768 enhanced font ,12
|
||||
set output 'residual.png'
|
||||
set datafile separator whitespace
|
||||
set xlabel "Timestep"
|
||||
set ylabel "Residual"
|
||||
|
||||
set logscale y 2
|
||||
|
||||
plot 'residual.dat' using 1:2 title "Residual"
|
61
EnhancedSolver/2D-mpi/src/affinity.c
Normal file
61
EnhancedSolver/2D-mpi/src/affinity.c
Normal file
@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved.
|
||||
* Use of this source code is governed by a MIT-style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#ifdef __linux__
|
||||
#ifdef _OPENMP
|
||||
#include <pthread.h>
|
||||
#include <sched.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define MAX_NUM_THREADS 128
|
||||
#define gettid() syscall(SYS_gettid)
|
||||
|
||||
static int getProcessorID(cpu_set_t* cpu_set)
|
||||
{
|
||||
int processorId;
|
||||
|
||||
for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) {
|
||||
if (CPU_ISSET(processorId, cpu_set)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return processorId;
|
||||
}
|
||||
|
||||
int affinity_getProcessorId()
|
||||
{
|
||||
cpu_set_t cpu_set;
|
||||
CPU_ZERO(&cpu_set);
|
||||
sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set);
|
||||
|
||||
return getProcessorID(&cpu_set);
|
||||
}
|
||||
|
||||
void affinity_pinThread(int processorId)
|
||||
{
|
||||
cpu_set_t cpuset;
|
||||
pthread_t thread;
|
||||
|
||||
thread = pthread_self();
|
||||
CPU_ZERO(&cpuset);
|
||||
CPU_SET(processorId, &cpuset);
|
||||
pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
|
||||
}
|
||||
|
||||
void affinity_pinProcess(int processorId)
|
||||
{
|
||||
cpu_set_t cpuset;
|
||||
|
||||
CPU_ZERO(&cpuset);
|
||||
CPU_SET(processorId, &cpuset);
|
||||
sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
|
||||
}
|
||||
#endif /*_OPENMP*/
|
||||
#endif /*__linux__*/
|
14
EnhancedSolver/2D-mpi/src/affinity.h
Normal file
14
EnhancedSolver/2D-mpi/src/affinity.h
Normal file
@ -0,0 +1,14 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved.
|
||||
* Use of this source code is governed by a MIT-style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#ifndef AFFINITY_H
|
||||
#define AFFINITY_H
|
||||
|
||||
extern int affinity_getProcessorId();
|
||||
extern void affinity_pinProcess(int);
|
||||
extern void affinity_pinThread(int);
|
||||
|
||||
#endif /*AFFINITY_H*/
|
38
EnhancedSolver/2D-mpi/src/allocate.c
Normal file
38
EnhancedSolver/2D-mpi/src/allocate.c
Normal file
@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved.
|
||||
* Use of this source code is governed by a MIT-style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <errno.h>
|
||||
#include <stddef.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "allocate.h"
|
||||
|
||||
void* allocate(size_t alignment, size_t bytesize)
|
||||
{
|
||||
int errorCode;
|
||||
void* ptr;
|
||||
|
||||
errorCode = posix_memalign(&ptr, alignment, bytesize);
|
||||
|
||||
if (errorCode) {
|
||||
if (errorCode == EINVAL) {
|
||||
fprintf(stderr, "Error: Alignment parameter is not a power of two\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (errorCode == ENOMEM) {
|
||||
fprintf(stderr, "Error: Insufficient memory to fulfill the request\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (ptr == NULL) {
|
||||
fprintf(stderr, "Error: posix_memalign failed!\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
return ptr;
|
||||
}
|
13
EnhancedSolver/2D-mpi/src/allocate.h
Normal file
13
EnhancedSolver/2D-mpi/src/allocate.h
Normal file
@ -0,0 +1,13 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved.
|
||||
* Use of this source code is governed by a MIT-style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#ifndef __ALLOCATE_H_
|
||||
#define __ALLOCATE_H_
|
||||
#include <stdlib.h>
|
||||
|
||||
extern void* allocate(size_t alignment, size_t bytesize);
|
||||
|
||||
#endif
|
233
EnhancedSolver/2D-mpi/src/comm-v1.c
Normal file
233
EnhancedSolver/2D-mpi/src/comm-v1.c
Normal file
@ -0,0 +1,233 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "comm.h"
|
||||
|
||||
#ifdef _MPI
|
||||
// subroutines local to this module
|
||||
static int sum(int* sizes, int position)
|
||||
{
|
||||
int sum = 0;
|
||||
|
||||
for (int i = 0; i < position; i += position) {
|
||||
sum += sizes[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
static void gatherArray(
|
||||
Comm* c, int cnt, int* rcvCounts, int* displs, double* src, double* dst)
|
||||
{
|
||||
double* sendbuffer = src + (c->imaxLocal + 2);
|
||||
|
||||
if (c->rank == 0) {
|
||||
sendbuffer = src;
|
||||
}
|
||||
|
||||
MPI_Gatherv(sendbuffer,
|
||||
cnt,
|
||||
MPI_DOUBLE,
|
||||
dst,
|
||||
rcvCounts,
|
||||
displs,
|
||||
MPI_DOUBLE,
|
||||
0,
|
||||
MPI_COMM_WORLD);
|
||||
}
|
||||
#endif // defined _MPI
|
||||
|
||||
// exported subroutines
|
||||
int commIsBoundary(Comm* c, int direction)
|
||||
{
|
||||
#ifdef _MPI
|
||||
switch (direction) {
|
||||
case L:
|
||||
return 1;
|
||||
break;
|
||||
case R:
|
||||
return 1;
|
||||
break;
|
||||
case B:
|
||||
return c->rank == 0;
|
||||
break;
|
||||
case T:
|
||||
return c->rank == (c->size - 1);
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
void commExchange(Comm* c, double* grid)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Request requests[4] = { MPI_REQUEST_NULL,
|
||||
MPI_REQUEST_NULL,
|
||||
MPI_REQUEST_NULL,
|
||||
MPI_REQUEST_NULL };
|
||||
|
||||
/* exchange ghost cells with top neighbor */
|
||||
if (c->rank + 1 < c->size) {
|
||||
int top = c->rank + 1;
|
||||
double* src = grid + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
|
||||
double* dst = grid + (c->jmaxLocal + 1) * (c->imaxLocal + 2) + 1;
|
||||
|
||||
MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, top, 1, MPI_COMM_WORLD, &requests[0]);
|
||||
MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, top, 2, MPI_COMM_WORLD, &requests[1]);
|
||||
}
|
||||
|
||||
/* exchange ghost cells with bottom neighbor */
|
||||
if (c->rank > 0) {
|
||||
int bottom = c->rank - 1;
|
||||
double* src = grid + (c->imaxLocal + 2) + 1;
|
||||
double* dst = grid + 1;
|
||||
|
||||
MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]);
|
||||
MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]);
|
||||
}
|
||||
|
||||
MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
|
||||
#endif
|
||||
}
|
||||
|
||||
void commShift(Comm* c, double* f, double* g)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Request requests[2] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL };
|
||||
|
||||
/* shift G */
|
||||
/* receive ghost cells from bottom neighbor */
|
||||
if (c->rank > 0) {
|
||||
int bottom = c->rank - 1;
|
||||
MPI_Irecv(g + 1,
|
||||
c->imaxLocal,
|
||||
MPI_DOUBLE,
|
||||
bottom,
|
||||
0,
|
||||
MPI_COMM_WORLD,
|
||||
&requests[0]);
|
||||
}
|
||||
|
||||
if (c->rank + 1 < c->size) {
|
||||
int top = c->rank + 1;
|
||||
double* buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
|
||||
/* send ghost cells to top neighbor */
|
||||
MPI_Isend(buf, c->imaxLocal, MPI_DOUBLE, top, 0, MPI_COMM_WORLD, &requests[1]);
|
||||
}
|
||||
|
||||
MPI_Waitall(2, requests, MPI_STATUSES_IGNORE);
|
||||
#endif
|
||||
}
|
||||
|
||||
void commCollectResult(Comm* c,
|
||||
double* ug,
|
||||
double* vg,
|
||||
double* pg,
|
||||
double* u,
|
||||
double* v,
|
||||
double* p,
|
||||
int jmax,
|
||||
int imax)
|
||||
{
|
||||
#ifdef _MPI
|
||||
int *rcvCounts, *displs;
|
||||
int cnt = c->jmaxLocal * (imax + 2);
|
||||
|
||||
if (c->rank == 0) {
|
||||
rcvCounts = (int*)malloc(c->size * sizeof(int));
|
||||
displs = (int*)malloc(c->size * sizeof(int));
|
||||
}
|
||||
|
||||
if (c->rank == 0 && c->size == 1) {
|
||||
cnt = (c->jmaxLocal + 2) * (imax + 2);
|
||||
} else if (c->rank == 0 || c->rank == (c->size - 1)) {
|
||||
cnt = (c->jmaxLocal + 1) * (imax + 2);
|
||||
}
|
||||
|
||||
MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD);
|
||||
|
||||
if (c->rank == 0) {
|
||||
displs[0] = 0;
|
||||
int cursor = rcvCounts[0];
|
||||
|
||||
for (int i = 1; i < c->size; i++) {
|
||||
displs[i] = cursor;
|
||||
cursor += rcvCounts[i];
|
||||
}
|
||||
}
|
||||
|
||||
gatherArray(c, cnt, rcvCounts, displs, p, pg);
|
||||
gatherArray(c, cnt, rcvCounts, displs, u, ug);
|
||||
gatherArray(c, cnt, rcvCounts, displs, v, vg);
|
||||
#endif
|
||||
}
|
||||
|
||||
void commPartition(Comm* c, int jmax, int imax)
|
||||
{
|
||||
#ifdef _MPI
|
||||
c->imaxLocal = imax;
|
||||
c->jmaxLocal = sizeOfRank(c->coords[JDIM], c->size, jmax);
|
||||
|
||||
c->neighbours[BOTTOM] = c->rank == 0 ? -1 : c->rank - 1;
|
||||
c->neighbours[TOP] = c->rank == (c->size - 1) ? -1 : c->rank + 1;
|
||||
c->neighbours[LEFT] = -1;
|
||||
c->neighbours[RIGHT] = -1;
|
||||
|
||||
c->coords[IDIM] = 0;
|
||||
c->coords[JDIM] = c->rank;
|
||||
|
||||
c->dims[IDIM] = 1;
|
||||
c->dims[JDIM] = c->size;
|
||||
#else
|
||||
c->imaxLocal = imax;
|
||||
c->jmaxLocal = jmax;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
|
||||
{
|
||||
|
||||
#if defined _MPI
|
||||
newcomm->comm = MPI_COMM_NULL;
|
||||
int result = MPI_Comm_dup(MPI_COMM_WORLD, &newcomm->comm);
|
||||
|
||||
if (result == MPI_ERR_COMM) {
|
||||
printf("\nNull communicator. Duplication failed !!\n");
|
||||
}
|
||||
|
||||
newcomm->rank = oldcomm->rank;
|
||||
newcomm->size = oldcomm->size;
|
||||
|
||||
newcomm->imaxLocal = imaxLocal / 2;
|
||||
newcomm->jmaxLocal = jmaxLocal / 2;
|
||||
|
||||
newcomm->neighbours[BOTTOM] = newcomm->rank == 0 ? -1 : newcomm->rank - 1;
|
||||
newcomm->neighbours[TOP] = newcomm->rank == (newcomm->size - 1) ? -1 : newcomm->rank + 1;
|
||||
newcomm->neighbours[LEFT] = -1;
|
||||
newcomm->neighbours[RIGHT] = -1;
|
||||
|
||||
newcomm->coords[IDIM] = 0;
|
||||
newcomm->coords[JDIM] = newcomm->rank;
|
||||
|
||||
newcomm->dims[IDIM] = 1;
|
||||
newcomm->dims[JDIM] = newcomm->size;
|
||||
|
||||
|
||||
#endif
|
||||
newcomm->imaxLocal = imaxLocal;
|
||||
newcomm->jmaxLocal = jmaxLocal;
|
||||
}
|
||||
|
||||
void commFreeCommunicator(Comm* comm)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Comm_free(&comm->comm);
|
||||
#endif
|
||||
}
|
342
EnhancedSolver/2D-mpi/src/comm-v2.c
Normal file
342
EnhancedSolver/2D-mpi/src/comm-v2.c
Normal file
@ -0,0 +1,342 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include "comm.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifdef _MPI
|
||||
// subroutines local to this module
|
||||
static int sum(int* sizes, int init, int offset, int coord)
|
||||
{
|
||||
int sum = 0;
|
||||
|
||||
for (int i = init - offset; coord > 0; i -= offset, --coord) {
|
||||
sum += sizes[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax)
|
||||
{
|
||||
MPI_Request* requests;
|
||||
int numRequests = 1;
|
||||
|
||||
if (c->rank == 0) {
|
||||
numRequests = c->size + 1;
|
||||
} else {
|
||||
numRequests = 1;
|
||||
}
|
||||
|
||||
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
|
||||
|
||||
/* all ranks send their bulk array, including the external boundary layer */
|
||||
MPI_Datatype bulkType;
|
||||
int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
|
||||
int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
|
||||
int starts[NDIMS] = { 1, 1 };
|
||||
|
||||
if (commIsBoundary(c, L)) {
|
||||
newSizes[CIDIM] += 1;
|
||||
starts[CIDIM] = 0;
|
||||
}
|
||||
if (commIsBoundary(c, R)) {
|
||||
newSizes[CIDIM] += 1;
|
||||
}
|
||||
if (commIsBoundary(c, B)) {
|
||||
newSizes[CJDIM] += 1;
|
||||
starts[CJDIM] = 0;
|
||||
}
|
||||
if (commIsBoundary(c, T)) {
|
||||
newSizes[CJDIM] += 1;
|
||||
}
|
||||
|
||||
MPI_Type_create_subarray(NDIMS,
|
||||
oldSizes,
|
||||
newSizes,
|
||||
starts,
|
||||
MPI_ORDER_C,
|
||||
MPI_DOUBLE,
|
||||
&bulkType);
|
||||
MPI_Type_commit(&bulkType);
|
||||
MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
|
||||
|
||||
int newSizesI[c->size];
|
||||
int newSizesJ[c->size];
|
||||
MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD);
|
||||
MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD);
|
||||
|
||||
/* rank 0 assembles the subdomains */
|
||||
if (c->rank == 0) {
|
||||
for (int i = 0; i < c->size; i++) {
|
||||
MPI_Datatype domainType;
|
||||
int oldSizes[NDIMS] = { jmax + 2, imax + 2 };
|
||||
int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
|
||||
int coords[NDIMS];
|
||||
MPI_Cart_coords(c->comm, i, NDIMS, coords);
|
||||
int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]),
|
||||
sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) };
|
||||
printf(
|
||||
"Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
|
||||
"Starts(i,j): %d %d\n",
|
||||
i,
|
||||
coords[IDIM],
|
||||
coords[JDIM],
|
||||
oldSizes[CIDIM],
|
||||
oldSizes[CJDIM],
|
||||
newSizes[CIDIM],
|
||||
newSizes[CJDIM],
|
||||
starts[CIDIM],
|
||||
starts[CJDIM]);
|
||||
|
||||
MPI_Type_create_subarray(NDIMS,
|
||||
oldSizes,
|
||||
newSizes,
|
||||
starts,
|
||||
MPI_ORDER_C,
|
||||
MPI_DOUBLE,
|
||||
&domainType);
|
||||
MPI_Type_commit(&domainType);
|
||||
|
||||
MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]);
|
||||
MPI_Type_free(&domainType);
|
||||
}
|
||||
}
|
||||
|
||||
MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
|
||||
}
|
||||
#endif // defined _MPI
|
||||
|
||||
// exported subroutines
|
||||
int commIsBoundary(Comm* c, int direction)
|
||||
{
|
||||
#ifdef _MPI
|
||||
switch (direction) {
|
||||
case L:
|
||||
return c->coords[IDIM] == 0;
|
||||
break;
|
||||
case R:
|
||||
return c->coords[IDIM] == (c->dims[IDIM] - 1);
|
||||
break;
|
||||
case B:
|
||||
return c->coords[JDIM] == 0;
|
||||
break;
|
||||
case T:
|
||||
return c->coords[JDIM] == (c->dims[JDIM] - 1);
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
void commExchange(Comm* c, double* grid)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Request requests[8];
|
||||
for (int i = 0; i < 8; i++)
|
||||
requests[i] = MPI_REQUEST_NULL;
|
||||
|
||||
for (int i = 0; i < NDIRS; i++) {
|
||||
double* sbuf = grid + c->sdispls[i];
|
||||
double* rbuf = grid + c->rdispls[i];
|
||||
|
||||
int tag = 0;
|
||||
if (c->neighbours[i] != MPI_PROC_NULL) {
|
||||
// printf("DEBUG: Rank %d - SendRecv with %d\n", c->rank, c->neighbours[i]);
|
||||
tag = c->neighbours[i];
|
||||
}
|
||||
MPI_Irecv(rbuf,
|
||||
1,
|
||||
c->bufferTypes[i],
|
||||
c->neighbours[i],
|
||||
tag,
|
||||
c->comm,
|
||||
&requests[i * 2]);
|
||||
MPI_Isend(sbuf,
|
||||
1,
|
||||
c->bufferTypes[i],
|
||||
c->neighbours[i],
|
||||
c->rank,
|
||||
c->comm,
|
||||
&requests[i * 2 + 1]);
|
||||
}
|
||||
|
||||
MPI_Waitall(8, requests, MPI_STATUSES_IGNORE);
|
||||
#endif
|
||||
}
|
||||
|
||||
void commShift(Comm* c, double* f, double* g)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Request requests[4] = { MPI_REQUEST_NULL,
|
||||
MPI_REQUEST_NULL,
|
||||
MPI_REQUEST_NULL,
|
||||
MPI_REQUEST_NULL };
|
||||
|
||||
/* shift G */
|
||||
/* receive ghost cells from bottom neighbor */
|
||||
double* buf = g + 1;
|
||||
MPI_Irecv(buf,
|
||||
1,
|
||||
c->bufferTypes[B],
|
||||
c->neighbours[B],
|
||||
0,
|
||||
c->comm,
|
||||
&requests[0]);
|
||||
|
||||
/* send ghost cells to top neighbor */
|
||||
buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
|
||||
MPI_Isend(buf, 1, c->bufferTypes[T], c->neighbours[T], 0, c->comm, &requests[1]);
|
||||
|
||||
/* shift F */
|
||||
/* receive ghost cells from left neighbor */
|
||||
buf = f + (c->imaxLocal + 2);
|
||||
MPI_Irecv(buf,
|
||||
1,
|
||||
c->bufferTypes[L],
|
||||
c->neighbours[L],
|
||||
1,
|
||||
c->comm,
|
||||
&requests[2]);
|
||||
|
||||
/* send ghost cells to right neighbor */
|
||||
buf = f + (c->imaxLocal + 2) + (c->imaxLocal);
|
||||
MPI_Isend(buf,
|
||||
1,
|
||||
c->bufferTypes[R],
|
||||
c->neighbours[R],
|
||||
1,
|
||||
c->comm,
|
||||
&requests[3]);
|
||||
|
||||
MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
|
||||
#endif
|
||||
}
|
||||
|
||||
void commCollectResult(Comm* c,
|
||||
double* ug,
|
||||
double* vg,
|
||||
double* pg,
|
||||
double* u,
|
||||
double* v,
|
||||
double* p,
|
||||
int imax,
|
||||
int jmax)
|
||||
{
|
||||
#ifdef _MPI
|
||||
/* collect P */
|
||||
assembleResult(c, p, pg, imax, jmax);
|
||||
|
||||
/* collect U */
|
||||
assembleResult(c, u, ug, imax, jmax);
|
||||
|
||||
/* collect V */
|
||||
assembleResult(c, v, vg, imax, jmax);
|
||||
#endif
|
||||
}
|
||||
|
||||
void commPartition(Comm* c, int jmax, int imax)
|
||||
{
|
||||
#ifdef _MPI
|
||||
int dims[NDIMS] = { 0, 0 };
|
||||
int periods[NDIMS] = { 0, 0 };
|
||||
MPI_Dims_create(c->size, NDIMS, dims);
|
||||
MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm);
|
||||
MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[L], &c->neighbours[R]);
|
||||
MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[B], &c->neighbours[T]);
|
||||
MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
|
||||
|
||||
int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax);
|
||||
int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax);
|
||||
|
||||
c->imaxLocal = imaxLocal;
|
||||
c->jmaxLocal = jmaxLocal;
|
||||
|
||||
MPI_Datatype jBufferType;
|
||||
MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
|
||||
MPI_Type_commit(&jBufferType);
|
||||
|
||||
MPI_Datatype iBufferType;
|
||||
MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
|
||||
MPI_Type_commit(&iBufferType);
|
||||
|
||||
c->bufferTypes[L] = iBufferType;
|
||||
c->bufferTypes[R] = iBufferType;
|
||||
c->bufferTypes[B] = jBufferType;
|
||||
c->bufferTypes[T] = jBufferType;
|
||||
|
||||
c->sdispls[L] = (imaxLocal + 2) + 1;
|
||||
c->sdispls[R] = (imaxLocal + 2) + imaxLocal;
|
||||
c->sdispls[B] = (imaxLocal + 2) + 1;
|
||||
c->sdispls[T] = jmaxLocal * (imaxLocal + 2) + 1;
|
||||
|
||||
c->rdispls[L] = (imaxLocal + 2);
|
||||
c->rdispls[R] = (imaxLocal + 2) + (imaxLocal + 1);
|
||||
c->rdispls[B] = 1;
|
||||
c->rdispls[T] = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
|
||||
#else
|
||||
c->imaxLocal = imax;
|
||||
c->jmaxLocal = jmax;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
|
||||
{
|
||||
#if defined _MPI
|
||||
newcomm->comm = MPI_COMM_NULL;
|
||||
int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
|
||||
|
||||
if (result == MPI_ERR_COMM) {
|
||||
printf("\nNull communicator. Duplication failed !!\n");
|
||||
}
|
||||
|
||||
newcomm->rank = oldcomm->rank;
|
||||
newcomm->size = oldcomm->size;
|
||||
|
||||
memcpy(&newcomm->neighbours, &oldcomm->neighbours, sizeof(oldcomm->neighbours));
|
||||
memcpy(&newcomm->coords, &oldcomm->coords, sizeof(oldcomm->coords));
|
||||
memcpy(&newcomm->dims, &oldcomm->dims, sizeof(oldcomm->dims));
|
||||
|
||||
newcomm->imaxLocal = imaxLocal/2;
|
||||
newcomm->jmaxLocal = jmaxLocal/2;
|
||||
|
||||
MPI_Datatype jBufferType;
|
||||
MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
|
||||
MPI_Type_commit(&jBufferType);
|
||||
|
||||
MPI_Datatype iBufferType;
|
||||
MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
|
||||
MPI_Type_commit(&iBufferType);
|
||||
|
||||
newcomm->bufferTypes[L] = iBufferType;
|
||||
newcomm->bufferTypes[R] = iBufferType;
|
||||
newcomm->bufferTypes[B] = jBufferType;
|
||||
newcomm->bufferTypes[T] = jBufferType;
|
||||
|
||||
newcomm->sdispls[L] = (imaxLocal + 2) + 1;
|
||||
newcomm->sdispls[R] = (imaxLocal + 2) + imaxLocal;
|
||||
newcomm->sdispls[B] = (imaxLocal + 2) + 1;
|
||||
newcomm->sdispls[T] = jmaxLocal * (imaxLocal + 2) + 1;
|
||||
|
||||
newcomm->rdispls[L] = (imaxLocal + 2);
|
||||
newcomm->rdispls[R] = (imaxLocal + 2) + (imaxLocal + 1);
|
||||
newcomm->rdispls[B] = 1;
|
||||
newcomm->rdispls[T] = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
|
||||
#else
|
||||
newcomm->imaxLocal = imaxLocal;
|
||||
newcomm->jmaxLocal = jmaxLocal;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commFreeCommunicator(Comm* comm)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Comm_free(&comm->comm);
|
||||
#endif
|
||||
}
|
320
EnhancedSolver/2D-mpi/src/comm-v3.c
Normal file
320
EnhancedSolver/2D-mpi/src/comm-v3.c
Normal file
@ -0,0 +1,320 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "comm.h"
|
||||
|
||||
#ifdef _MPI
|
||||
// subroutines local to this module
|
||||
static int sum(int* sizes, int init, int offset, int coord)
|
||||
{
|
||||
int sum = 0;
|
||||
|
||||
for (int i = init - offset; coord > 0; i -= offset, --coord) {
|
||||
sum += sizes[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax)
|
||||
{
|
||||
MPI_Request* requests;
|
||||
int numRequests = 1;
|
||||
|
||||
if (c->rank == 0) {
|
||||
numRequests = c->size + 1;
|
||||
} else {
|
||||
numRequests = 1;
|
||||
}
|
||||
|
||||
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
|
||||
|
||||
/* all ranks send their bulk array, including the external boundary layer */
|
||||
MPI_Datatype bulkType;
|
||||
int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
|
||||
int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
|
||||
int starts[NDIMS] = { 1, 1 };
|
||||
|
||||
if (commIsBoundary(c, L)) {
|
||||
newSizes[CIDIM] += 1;
|
||||
starts[CIDIM] = 0;
|
||||
}
|
||||
if (commIsBoundary(c, R)) {
|
||||
newSizes[CIDIM] += 1;
|
||||
}
|
||||
if (commIsBoundary(c, B)) {
|
||||
newSizes[CJDIM] += 1;
|
||||
starts[CJDIM] = 0;
|
||||
}
|
||||
if (commIsBoundary(c, T)) {
|
||||
newSizes[CJDIM] += 1;
|
||||
}
|
||||
|
||||
MPI_Type_create_subarray(NDIMS,
|
||||
oldSizes,
|
||||
newSizes,
|
||||
starts,
|
||||
MPI_ORDER_C,
|
||||
MPI_DOUBLE,
|
||||
&bulkType);
|
||||
MPI_Type_commit(&bulkType);
|
||||
MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
|
||||
|
||||
int newSizesI[c->size];
|
||||
int newSizesJ[c->size];
|
||||
MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD);
|
||||
MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD);
|
||||
|
||||
/* rank 0 assembles the subdomains */
|
||||
if (c->rank == 0) {
|
||||
for (int i = 0; i < c->size; i++) {
|
||||
MPI_Datatype domainType;
|
||||
int oldSizes[NDIMS] = { jmax + 2, imax + 2 };
|
||||
int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
|
||||
int coords[NDIMS];
|
||||
MPI_Cart_coords(c->comm, i, NDIMS, coords);
|
||||
int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]),
|
||||
sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) };
|
||||
printf(
|
||||
"Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
|
||||
"Starts(i,j): %d %d\n",
|
||||
i,
|
||||
coords[IDIM],
|
||||
coords[JDIM],
|
||||
oldSizes[CIDIM],
|
||||
oldSizes[CJDIM],
|
||||
newSizes[CIDIM],
|
||||
newSizes[CJDIM],
|
||||
starts[CIDIM],
|
||||
starts[CJDIM]);
|
||||
|
||||
MPI_Type_create_subarray(NDIMS,
|
||||
oldSizes,
|
||||
newSizes,
|
||||
starts,
|
||||
MPI_ORDER_C,
|
||||
MPI_DOUBLE,
|
||||
&domainType);
|
||||
MPI_Type_commit(&domainType);
|
||||
|
||||
MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]);
|
||||
MPI_Type_free(&domainType);
|
||||
}
|
||||
}
|
||||
|
||||
MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
|
||||
}
|
||||
#endif // defined _MPI
|
||||
|
||||
// exported subroutines
|
||||
int commIsBoundary(Comm* c, int direction)
|
||||
{
|
||||
#ifdef _MPI
|
||||
switch (direction) {
|
||||
case L:
|
||||
return c->coords[IDIM] == 0;
|
||||
break;
|
||||
case R:
|
||||
return c->coords[IDIM] == (c->dims[IDIM] - 1);
|
||||
break;
|
||||
case B:
|
||||
return c->coords[JDIM] == 0;
|
||||
break;
|
||||
case T:
|
||||
return c->coords[JDIM] == (c->dims[JDIM] - 1);
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
void commExchange(Comm* c, double* grid)
|
||||
{
|
||||
#ifdef _MPI
|
||||
int counts[NDIRS] = { 1, 1, 1, 1 };
|
||||
MPI_Neighbor_alltoallw(grid,
|
||||
counts,
|
||||
c->sdispls,
|
||||
c->bufferTypes,
|
||||
grid,
|
||||
counts,
|
||||
c->rdispls,
|
||||
c->bufferTypes,
|
||||
c->comm);
|
||||
#endif
|
||||
}
|
||||
|
||||
void commShift(Comm* c, double* f, double* g)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Request requests[4] = { MPI_REQUEST_NULL,
|
||||
MPI_REQUEST_NULL,
|
||||
MPI_REQUEST_NULL,
|
||||
MPI_REQUEST_NULL };
|
||||
|
||||
/* shift G */
|
||||
/* receive ghost cells from bottom neighbor */
|
||||
double* buf = g + 1;
|
||||
MPI_Irecv(buf,
|
||||
1,
|
||||
c->bufferTypes[B],
|
||||
c->neighbours[B],
|
||||
0,
|
||||
c->comm,
|
||||
&requests[0]);
|
||||
|
||||
/* send ghost cells to top neighbor */
|
||||
buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
|
||||
MPI_Isend(buf, 1, c->bufferTypes[T], c->neighbours[T], 0, c->comm, &requests[1]);
|
||||
|
||||
/* shift F */
|
||||
/* receive ghost cells from left neighbor */
|
||||
buf = f + (c->imaxLocal + 2);
|
||||
MPI_Irecv(buf,
|
||||
1,
|
||||
c->bufferTypes[L],
|
||||
c->neighbours[L],
|
||||
1,
|
||||
c->comm,
|
||||
&requests[2]);
|
||||
|
||||
/* send ghost cells to right neighbor */
|
||||
buf = f + (c->imaxLocal + 2) + (c->imaxLocal);
|
||||
MPI_Isend(buf,
|
||||
1,
|
||||
c->bufferTypes[R],
|
||||
c->neighbours[R],
|
||||
1,
|
||||
c->comm,
|
||||
&requests[3]);
|
||||
|
||||
MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
|
||||
#endif
|
||||
}
|
||||
|
||||
void commCollectResult(Comm* c,
|
||||
double* ug,
|
||||
double* vg,
|
||||
double* pg,
|
||||
double* u,
|
||||
double* v,
|
||||
double* p,
|
||||
int imax,
|
||||
int jmax)
|
||||
{
|
||||
#ifdef _MPI
|
||||
/* collect P */
|
||||
assembleResult(c, p, pg, imax, jmax);
|
||||
|
||||
/* collect U */
|
||||
assembleResult(c, u, ug, imax, jmax);
|
||||
|
||||
/* collect V */
|
||||
assembleResult(c, v, vg, imax, jmax);
|
||||
#endif
|
||||
}
|
||||
|
||||
void commPartition(Comm* c, int jmax, int imax)
|
||||
{
|
||||
#ifdef _MPI
|
||||
int dims[NDIMS] = { 0, 0 };
|
||||
int periods[NDIMS] = { 0, 0 };
|
||||
MPI_Dims_create(c->size, NDIMS, dims);
|
||||
MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm);
|
||||
MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[L], &c->neighbours[R]);
|
||||
MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[B], &c->neighbours[T]);
|
||||
MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
|
||||
|
||||
int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax);
|
||||
int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax);
|
||||
|
||||
c->imaxLocal = imaxLocal;
|
||||
c->jmaxLocal = jmaxLocal;
|
||||
|
||||
MPI_Datatype jBufferType;
|
||||
MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
|
||||
MPI_Type_commit(&jBufferType);
|
||||
|
||||
MPI_Datatype iBufferType;
|
||||
MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
|
||||
MPI_Type_commit(&iBufferType);
|
||||
|
||||
c->bufferTypes[L] = iBufferType;
|
||||
c->bufferTypes[R] = iBufferType;
|
||||
c->bufferTypes[B] = jBufferType;
|
||||
c->bufferTypes[T] = jBufferType;
|
||||
|
||||
size_t dblsize = sizeof(double);
|
||||
c->sdispls[L] = ((imaxLocal + 2) + 1) * dblsize;
|
||||
c->sdispls[R] = ((imaxLocal + 2) + imaxLocal) * dblsize;
|
||||
c->sdispls[B] = ((imaxLocal + 2) + 1) * dblsize;
|
||||
c->sdispls[T] = (jmaxLocal * (imaxLocal + 2) + 1) * dblsize;
|
||||
|
||||
c->rdispls[L] = (imaxLocal + 2) * dblsize;
|
||||
c->rdispls[R] = ((imaxLocal + 2) + (imaxLocal + 1)) * dblsize;
|
||||
c->rdispls[B] = 1 * dblsize;
|
||||
c->rdispls[T] = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * dblsize;
|
||||
#else
|
||||
c->imaxLocal = imax;
|
||||
c->jmaxLocal = jmax;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
|
||||
{
|
||||
#if defined _MPI
|
||||
|
||||
int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
|
||||
|
||||
if (result == MPI_ERR_COMM) {
|
||||
printf("\nNull communicator. Duplication failed !!\n");
|
||||
}
|
||||
|
||||
newcomm->rank = oldcomm->rank;
|
||||
newcomm->size = oldcomm->size;
|
||||
|
||||
|
||||
newcomm->imaxLocal = imaxLocal / 2;
|
||||
newcomm->jmaxLocal = jmaxLocal / 2;
|
||||
|
||||
MPI_Datatype jBufferType;
|
||||
MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
|
||||
MPI_Type_commit(&jBufferType);
|
||||
|
||||
MPI_Datatype iBufferType;
|
||||
MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
|
||||
MPI_Type_commit(&iBufferType);
|
||||
|
||||
newcomm->bufferTypes[L] = iBufferType;
|
||||
newcomm->bufferTypes[R] = iBufferType;
|
||||
newcomm->bufferTypes[B] = jBufferType;
|
||||
newcomm->bufferTypes[T] = jBufferType;
|
||||
|
||||
newcomm->sdispls[L] = (imaxLocal + 2) + 1;
|
||||
newcomm->sdispls[R] = (imaxLocal + 2) + imaxLocal;
|
||||
newcomm->sdispls[B] = (imaxLocal + 2) + 1;
|
||||
newcomm->sdispls[T] = jmaxLocal * (imaxLocal + 2) + 1;
|
||||
|
||||
newcomm->rdispls[L] = (imaxLocal + 2);
|
||||
newcomm->rdispls[R] = (imaxLocal + 2) + (imaxLocal + 1);
|
||||
newcomm->rdispls[B] = 1;
|
||||
newcomm->rdispls[T] = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
|
||||
#else
|
||||
newcomm->imaxLocal = imaxLocal;
|
||||
newcomm->jmaxLocal = jmaxLocal;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commFreeCommunicator(Comm* comm)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Comm_free(&comm->comm);
|
||||
#endif
|
||||
}
|
129
EnhancedSolver/2D-mpi/src/comm.c
Normal file
129
EnhancedSolver/2D-mpi/src/comm.c
Normal file
@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "comm.h"
|
||||
|
||||
// subroutines local to this module
|
||||
int sizeOfRank(int rank, int size, int N)
|
||||
{
|
||||
return N / size + ((N % size > rank) ? 1 : 0);
|
||||
}
|
||||
|
||||
void commReduction(double* v, int op)
|
||||
{
|
||||
#ifdef _MPI
|
||||
if (op == MAX) {
|
||||
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
|
||||
} else if (op == SUM) {
|
||||
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void commPrintConfig(Comm* c)
|
||||
{
|
||||
#ifdef _MPI
|
||||
fflush(stdout);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
if (commIsMaster(c)) {
|
||||
printf("Communication setup:\n");
|
||||
}
|
||||
|
||||
for (int i = 0; i < c->size; i++) {
|
||||
if (i == c->rank) {
|
||||
printf("\tRank %d of %d\n", c->rank, c->size);
|
||||
printf("\tNeighbours (bottom, top, left, right): %d %d, %d, %d\n",
|
||||
c->neighbours[B],
|
||||
c->neighbours[T],
|
||||
c->neighbours[L],
|
||||
c->neighbours[R]);
|
||||
printf("\tIs boundary:\n");
|
||||
printf("\t\tLEFT: %d\n", commIsBoundary(c, L));
|
||||
printf("\t\tRIGHT: %d\n", commIsBoundary(c, R));
|
||||
printf("\t\tBOTTOM: %d\n", commIsBoundary(c, B));
|
||||
printf("\t\tTOP: %d\n", commIsBoundary(c, T));
|
||||
printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]);
|
||||
printf("\tDims (i,j) %d %d\n", c->dims[IDIM], c->dims[JDIM]);
|
||||
printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal);
|
||||
fflush(stdout);
|
||||
}
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void commInit(Comm* c, int argc, char** argv)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Init(&argc, &argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
|
||||
#else
|
||||
c->rank = 0;
|
||||
c->size = 1;
|
||||
#endif
|
||||
}
|
||||
|
||||
void commTestInit(Comm* c, double* p, double* f, double* g)
|
||||
{
|
||||
int imax = c->imaxLocal;
|
||||
int jmax = c->jmaxLocal;
|
||||
int rank = c->rank;
|
||||
|
||||
for (int j = 0; j < jmax + 2; j++) {
|
||||
for (int i = 0; i < imax + 2; i++) {
|
||||
p[j * (imax + 2) + i] = rank;
|
||||
f[j * (imax + 2) + i] = rank;
|
||||
g[j * (imax + 2) + i] = rank;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void testWriteFile(char* filename, double* grid, int imax, int jmax)
|
||||
{
|
||||
FILE* fp = fopen(filename, "w");
|
||||
|
||||
if (fp == NULL) {
|
||||
printf("Error!\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
for (int j = 0; j < jmax + 2; j++) {
|
||||
for (int i = 0; i < imax + 2; i++) {
|
||||
fprintf(fp, "%.2f ", grid[j * (imax + 2) + i]);
|
||||
}
|
||||
fprintf(fp, "\n");
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
void commTestWrite(Comm* c, double* p, double* f, double* g)
|
||||
{
|
||||
int imax = c->imaxLocal;
|
||||
int jmax = c->jmaxLocal;
|
||||
int rank = c->rank;
|
||||
|
||||
char filename[30];
|
||||
snprintf(filename, 30, "ptest-%d.dat", rank);
|
||||
testWriteFile(filename, p, imax, jmax);
|
||||
|
||||
snprintf(filename, 30, "ftest-%d.dat", rank);
|
||||
testWriteFile(filename, f, imax, jmax);
|
||||
|
||||
snprintf(filename, 30, "gtest-%d.dat", rank);
|
||||
testWriteFile(filename, g, imax, jmax);
|
||||
}
|
||||
|
||||
void commFinalize(Comm* c)
|
||||
{
|
||||
#ifdef _MPI
|
||||
MPI_Finalize();
|
||||
#endif
|
||||
}
|
57
EnhancedSolver/2D-mpi/src/comm.h
Normal file
57
EnhancedSolver/2D-mpi/src/comm.h
Normal file
@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#ifndef __COMM_H_
|
||||
#define __COMM_H_
|
||||
#if defined(_MPI)
|
||||
#include <mpi.h>
|
||||
#endif
|
||||
|
||||
enum direction { L = 0, R, B, T, NDIRS }; // L = Left, R = Right, B = Bottom, T =Top
|
||||
enum dimension { IDIM = 0, JDIM, NDIMS };
|
||||
enum cdimension { CJDIM = 0, CIDIM };
|
||||
enum layer { HALO = 0, BULK };
|
||||
enum op { MAX = 0, SUM };
|
||||
|
||||
typedef struct {
|
||||
int rank;
|
||||
int size;
|
||||
#if defined(_MPI)
|
||||
MPI_Comm comm;
|
||||
MPI_Datatype bufferTypes[NDIRS];
|
||||
MPI_Aint sdispls[NDIRS];
|
||||
MPI_Aint rdispls[NDIRS];
|
||||
#endif
|
||||
int neighbours[NDIRS];
|
||||
int coords[NDIMS], dims[NDIMS];
|
||||
int imaxLocal, jmaxLocal;
|
||||
} Comm;
|
||||
|
||||
extern int sizeOfRank(int rank, int size, int N);
|
||||
extern void commInit(Comm* c, int argc, char** argv);
|
||||
extern void commTestInit(Comm* c, double* p, double* f, double* g);
|
||||
extern void commTestWrite(Comm* c, double* p, double* f, double* g);
|
||||
extern void commFinalize(Comm* c);
|
||||
extern void commPartition(Comm* c, int jmax, int imax);
|
||||
extern void commPrintConfig(Comm*);
|
||||
extern void commExchange(Comm*, double*);
|
||||
extern void commShift(Comm* c, double* f, double* g);
|
||||
extern void commReduction(double* v, int op);
|
||||
extern int commIsBoundary(Comm* c, int direction);
|
||||
extern void commUpdateDatatypes(Comm*, Comm*, int, int);
|
||||
extern void commFreeCommunicator(Comm*);
|
||||
extern void commCollectResult(Comm* c,
|
||||
double* ug,
|
||||
double* vg,
|
||||
double* pg,
|
||||
double* u,
|
||||
double* v,
|
||||
double* p,
|
||||
int jmax,
|
||||
int imax);
|
||||
|
||||
static inline int commIsMaster(Comm* c) { return c->rank == 0; }
|
||||
#endif // __COMM_H_
|
724
EnhancedSolver/2D-mpi/src/discretization.c
Normal file
724
EnhancedSolver/2D-mpi/src/discretization.c
Normal file
@ -0,0 +1,724 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <float.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "allocate.h"
|
||||
#include "comm.h"
|
||||
#include "discretization.h"
|
||||
#include "parameter.h"
|
||||
#include "util.h"
|
||||
|
||||
static double distance(double i, double j, double iCenter, double jCenter)
|
||||
{
|
||||
return sqrt(pow(iCenter - i, 2) + pow(jCenter - j, 2) * 1.0);
|
||||
}
|
||||
|
||||
double sumOffset(double* sizes, int init, int offset, int coord)
|
||||
{
|
||||
double sum = 0;
|
||||
|
||||
for (int i = init - offset; coord > 0; i -= offset, --coord) {
|
||||
sum += sizes[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
void print(Discretization* d, double* grid)
|
||||
{
|
||||
int imaxLocal = d->comm.imaxLocal;
|
||||
|
||||
for (int i = 0; i < d->comm.size; i++) {
|
||||
if (i == d->comm.rank) {
|
||||
sleep(1 * d->comm.rank);
|
||||
printf("### RANK %d LVL "
|
||||
"###################################################### #\n ",
|
||||
d->comm.rank);
|
||||
for (int j = 0; j < d->comm.jmaxLocal + 2; j++) {
|
||||
printf("%02d: ", j);
|
||||
for (int i = 0; i < d->comm.imaxLocal + 2; i++) {
|
||||
printf("%2.2f ", grid[j * (imaxLocal + 2) + i]);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void printConfig(Discretization* d)
|
||||
{
|
||||
if (commIsMaster(&d->comm)) {
|
||||
printf("Parameters for #%s#\n", d->problem);
|
||||
printf("BC Left:%d Right:%d Bottom:%d Top:%d\n",
|
||||
d->bcLeft,
|
||||
d->bcRight,
|
||||
d->bcBottom,
|
||||
d->bcTop);
|
||||
printf("\tReynolds number: %.2f\n", d->re);
|
||||
printf("\tGx Gy: %.2f %.2f\n", d->gx, d->gy);
|
||||
printf("Geometry data:\n");
|
||||
printf("\tDomain box size (x, y): %.2f, %.2f\n",
|
||||
d->grid.xlength,
|
||||
d->grid.ylength);
|
||||
printf("\tCells (x, y): %d, %d\n", d->grid.imax, d->grid.jmax);
|
||||
printf("\tCell size (dx, dy): %f, %f\n", d->grid.dx, d->grid.dy);
|
||||
printf("Timestep parameters:\n");
|
||||
printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te);
|
||||
printf("\tdt bound: %.6f\n", d->dtBound);
|
||||
printf("\tTau factor: %.2f\n", d->tau);
|
||||
printf("Iterative s parameters:\n");
|
||||
printf("\tgamma factor: %f\n", d->gamma);
|
||||
}
|
||||
commPrintConfig(&d->comm);
|
||||
}
|
||||
|
||||
void initDiscretiztion(Discretization* d, Parameter* params)
|
||||
{
|
||||
d->problem = params->name;
|
||||
d->bcLeft = params->bcLeft;
|
||||
d->bcRight = params->bcRight;
|
||||
d->bcBottom = params->bcBottom;
|
||||
d->bcTop = params->bcTop;
|
||||
d->grid.imax = params->imax;
|
||||
d->grid.jmax = params->jmax;
|
||||
d->grid.xlength = params->xlength;
|
||||
d->grid.ylength = params->ylength;
|
||||
d->grid.dx = params->xlength / params->imax;
|
||||
d->grid.dy = params->ylength / params->jmax;
|
||||
d->re = params->re;
|
||||
d->gx = params->gx;
|
||||
d->gy = params->gy;
|
||||
d->dt = params->dt;
|
||||
d->te = params->te;
|
||||
d->tau = params->tau;
|
||||
d->gamma = params->gamma;
|
||||
|
||||
/* allocate arrays */
|
||||
int imaxLocal = d->comm.imaxLocal;
|
||||
int jmaxLocal = d->comm.jmaxLocal;
|
||||
size_t size = (imaxLocal + 2) * (jmaxLocal + 2);
|
||||
|
||||
d->u = allocate(64, size * sizeof(double));
|
||||
d->v = allocate(64, size * sizeof(double));
|
||||
d->p = allocate(64, size * sizeof(double));
|
||||
d->rhs = allocate(64, size * sizeof(double));
|
||||
d->f = allocate(64, size * sizeof(double));
|
||||
d->g = allocate(64, size * sizeof(double));
|
||||
d->grid.s = allocate(64, size * sizeof(double));
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
d->u[i] = params->u_init;
|
||||
d->v[i] = params->v_init;
|
||||
d->p[i] = params->p_init;
|
||||
d->rhs[i] = 0.0;
|
||||
d->f[i] = 0.0;
|
||||
d->g[i] = 0.0;
|
||||
d->grid.s[i] = FLUID;
|
||||
}
|
||||
|
||||
double dx = d->grid.dx;
|
||||
double dy = d->grid.dy;
|
||||
|
||||
double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy);
|
||||
d->dtBound = 0.5 * d->re * 1.0 / invSqrSum;
|
||||
|
||||
d->xLocal = d->comm.imaxLocal * d->grid.dx;
|
||||
d->yLocal = d->comm.jmaxLocal * d->grid.dy;
|
||||
|
||||
double xLocal[d->comm.size];
|
||||
double yLocal[d->comm.size];
|
||||
|
||||
#ifdef _MPI
|
||||
MPI_Allgather(&d->xLocal, 1, MPI_DOUBLE, xLocal, 1, MPI_DOUBLE, d->comm.comm);
|
||||
MPI_Allgather(&d->yLocal, 1, MPI_DOUBLE, yLocal, 1, MPI_DOUBLE, d->comm.comm);
|
||||
|
||||
d->xOffset = sumOffset(xLocal,
|
||||
d->comm.rank,
|
||||
d->comm.dims[JDIM],
|
||||
d->comm.coords[IDIM]);
|
||||
d->yOffset = sumOffset(yLocal, d->comm.rank, 1, d->comm.coords[JDIM]);
|
||||
d->xOffsetEnd = d->xOffset + d->xLocal;
|
||||
d->yOffsetEnd = d->yOffset + d->yLocal;
|
||||
#else
|
||||
|
||||
d->xOffset = 0;
|
||||
d->yOffset = 0;
|
||||
d->xOffsetEnd = d->xOffset + d->xLocal;
|
||||
d->yOffsetEnd = d->yOffset + d->yLocal;
|
||||
#endif
|
||||
|
||||
printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : "
|
||||
"%.2f\n",
|
||||
d->comm.rank,
|
||||
d->xOffset,
|
||||
d->yOffset,
|
||||
d->xOffsetEnd,
|
||||
d->yOffsetEnd);
|
||||
|
||||
double* s = d->grid.s;
|
||||
int iOffset = 0, jOffset = 0;
|
||||
|
||||
double xCenter = 0, yCenter = 0, radius = 0;
|
||||
double x1 = 0, x2 = 0, y1 = 0, y2 = 0;
|
||||
|
||||
switch (params->shape) {
|
||||
case NOSHAPE:
|
||||
break;
|
||||
case RECT:
|
||||
x1 = params->xCenter - params->xRectLength / 2;
|
||||
x2 = params->xCenter + params->xRectLength / 2;
|
||||
y1 = params->yCenter - params->yRectLength / 2;
|
||||
y2 = params->yCenter + params->yRectLength / 2;
|
||||
|
||||
iOffset = d->xOffset / dx;
|
||||
jOffset = d->yOffset / dy;
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; ++j) {
|
||||
for (int i = 1; i < imaxLocal + 1; ++i) {
|
||||
if ((x1 <= ((i + iOffset) * dx)) && (((i + iOffset) * dx) <= x2) &&
|
||||
(y1 <= ((j + jOffset) * dy)) && (((j + jOffset) * dy) <= y2)) {
|
||||
S(i, j) = OBSTACLE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
case CIRCLE:
|
||||
xCenter = params->xCenter;
|
||||
yCenter = params->yCenter;
|
||||
radius = params->circleRadius;
|
||||
|
||||
iOffset = d->xOffset / dx;
|
||||
jOffset = d->yOffset / dy;
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
if (distance(((i + iOffset) * dx),
|
||||
((j + jOffset) * dy),
|
||||
xCenter,
|
||||
yCenter) <= radius) {
|
||||
S(i, j) = OBSTACLE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
#ifdef _MPI
|
||||
commExchange(&d->comm, s);
|
||||
#endif
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
if (S(i, j - 1) == FLUID && S(i, j + 1) == OBSTACLE && S(i, j) == OBSTACLE)
|
||||
S(i, j) = BOTTOM; // BOTTOM
|
||||
if (S(i - 1, j) == FLUID && S(i + 1, j) == OBSTACLE && S(i, j) == OBSTACLE)
|
||||
S(i, j) = LEFT; // LEFT
|
||||
if (S(i + 1, j) == FLUID && S(i - 1, j) == OBSTACLE && S(i, j) == OBSTACLE)
|
||||
S(i, j) = RIGHT; // RIGHT
|
||||
if (S(i, j + 1) == FLUID && S(i, j - 1) == OBSTACLE && S(i, j) == OBSTACLE)
|
||||
S(i, j) = TOP; // TOP
|
||||
if (S(i - 1, j - 1) == FLUID && S(i, j - 1) == FLUID &&
|
||||
S(i - 1, j) == FLUID && S(i + 1, j + 1) == OBSTACLE &&
|
||||
(S(i, j) == OBSTACLE || S(i, j) == LEFT || S(i, j) == BOTTOM))
|
||||
S(i, j) = BOTTOMLEFT; // BOTTOMLEFT
|
||||
if (S(i + 1, j - 1) == FLUID && S(i, j - 1) == FLUID &&
|
||||
S(i + 1, j) == FLUID && S(i - 1, j + 1) == OBSTACLE &&
|
||||
(S(i, j) == OBSTACLE || S(i, j) == RIGHT || S(i, j) == BOTTOM))
|
||||
S(i, j) = BOTTOMRIGHT; // BOTTOMRIGHT
|
||||
if (S(i - 1, j + 1) == FLUID && S(i - 1, j) == FLUID &&
|
||||
S(i, j + 1) == FLUID && S(i + 1, j - 1) == OBSTACLE &&
|
||||
(S(i, j) == OBSTACLE || S(i, j) == LEFT || S(i, j) == TOP))
|
||||
S(i, j) = TOPLEFT; // TOPLEFT
|
||||
if (S(i + 1, j + 1) == FLUID && S(i + 1, j) == FLUID &&
|
||||
S(i, j + 1) == FLUID && S(i - 1, j - 1) == OBSTACLE &&
|
||||
(S(i, j) == OBSTACLE || S(i, j) == RIGHT || S(i, j) == TOP))
|
||||
S(i, j) = TOPRIGHT; // TOPRIGHT
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef VERBOSE
|
||||
printConfig(d);
|
||||
#endif
|
||||
}
|
||||
|
||||
void computeRHS(Discretization* d)
|
||||
{
|
||||
int imaxLocal = d->comm.imaxLocal;
|
||||
int jmaxLocal = d->comm.jmaxLocal;
|
||||
double idx = 1.0 / d->grid.dx;
|
||||
double idy = 1.0 / d->grid.dy;
|
||||
double idt = 1.0 / d->dt;
|
||||
double* rhs = d->rhs;
|
||||
double* f = d->f;
|
||||
double* g = d->g;
|
||||
|
||||
commShift(&d->comm, f, g);
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) *
|
||||
idt;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static double maxElement(Discretization* d, double* m)
|
||||
{
|
||||
int imaxLocal = d->comm.imaxLocal;
|
||||
int jmaxLocal = d->comm.jmaxLocal;
|
||||
int size = (imaxLocal + 2) * (jmaxLocal + 2);
|
||||
double maxval = DBL_MIN;
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
maxval = MAX(maxval, fabs(m[i]));
|
||||
}
|
||||
|
||||
commReduction(&maxval, MAX);
|
||||
return maxval;
|
||||
}
|
||||
|
||||
void computeTimestep(Discretization* d)
|
||||
{
|
||||
double dt = d->dtBound;
|
||||
double dx = d->grid.dx;
|
||||
double dy = d->grid.dy;
|
||||
double umax = maxElement(d, d->u);
|
||||
double vmax = maxElement(d, d->v);
|
||||
|
||||
if (umax > 0) {
|
||||
dt = (dt > dx / umax) ? dx / umax : dt;
|
||||
}
|
||||
if (vmax > 0) {
|
||||
dt = (dt > dy / vmax) ? dy / vmax : dt;
|
||||
}
|
||||
|
||||
d->dt = dt * d->tau;
|
||||
}
|
||||
|
||||
void setBoundaryConditions(Discretization* d)
|
||||
{
|
||||
int imaxLocal = d->comm.imaxLocal;
|
||||
int jmaxLocal = d->comm.jmaxLocal;
|
||||
double* u = d->u;
|
||||
double* v = d->v;
|
||||
|
||||
if (commIsBoundary(&d->comm, T)) {
|
||||
switch (d->bcTop) {
|
||||
case NOSLIP:
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
V(i, jmaxLocal) = 0.0;
|
||||
U(i, jmaxLocal + 1) = -U(i, jmaxLocal);
|
||||
}
|
||||
break;
|
||||
case SLIP:
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
V(i, jmaxLocal) = 0.0;
|
||||
U(i, jmaxLocal + 1) = U(i, jmaxLocal);
|
||||
}
|
||||
break;
|
||||
case OUTFLOW:
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, jmaxLocal + 1) = U(i, jmaxLocal);
|
||||
V(i, jmaxLocal) = V(i, jmaxLocal - 1);
|
||||
}
|
||||
break;
|
||||
case PERIODIC:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&d->comm, B)) {
|
||||
switch (d->bcBottom) {
|
||||
case NOSLIP:
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
V(i, 0) = 0.0;
|
||||
U(i, 0) = -U(i, 1);
|
||||
}
|
||||
break;
|
||||
case SLIP:
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
V(i, 0) = 0.0;
|
||||
U(i, 0) = U(i, 1);
|
||||
}
|
||||
break;
|
||||
case OUTFLOW:
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, 0) = U(i, 1);
|
||||
V(i, 0) = V(i, 1);
|
||||
}
|
||||
break;
|
||||
case PERIODIC:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&d->comm, R)) {
|
||||
switch (d->bcRight) {
|
||||
case NOSLIP:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(imaxLocal, j) = 0.0;
|
||||
V(imaxLocal + 1, j) = -V(imaxLocal, j);
|
||||
}
|
||||
break;
|
||||
case SLIP:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(imaxLocal, j) = 0.0;
|
||||
V(imaxLocal + 1, j) = V(imaxLocal, j);
|
||||
}
|
||||
break;
|
||||
case OUTFLOW:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(imaxLocal, j) = U(imaxLocal - 1, j);
|
||||
V(imaxLocal + 1, j) = V(imaxLocal, j);
|
||||
}
|
||||
break;
|
||||
case PERIODIC:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&d->comm, L)) {
|
||||
switch (d->bcLeft) {
|
||||
case NOSLIP:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(0, j) = 0.0;
|
||||
V(0, j) = -V(1, j);
|
||||
}
|
||||
break;
|
||||
case SLIP:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(0, j) = 0.0;
|
||||
V(0, j) = V(1, j);
|
||||
}
|
||||
break;
|
||||
case OUTFLOW:
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(0, j) = U(1, j);
|
||||
V(0, j) = V(1, j);
|
||||
}
|
||||
break;
|
||||
case PERIODIC:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void setSpecialBoundaryCondition(Discretization* d)
|
||||
{
|
||||
int imaxLocal = d->comm.imaxLocal;
|
||||
int jmaxLocal = d->comm.jmaxLocal;
|
||||
double* u = d->u;
|
||||
double* s = d->grid.s;
|
||||
|
||||
if (strcmp(d->problem, "dcavity") == 0) {
|
||||
if (commIsBoundary(&d->comm, T)) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal);
|
||||
}
|
||||
}
|
||||
} else if (strcmp(d->problem, "canal") == 0) {
|
||||
if (commIsBoundary(&d->comm, L)) {
|
||||
|
||||
double ylength = d->grid.ylength;
|
||||
double dy = d->grid.dy;
|
||||
int rest = d->grid.jmax % d->comm.dims[JDIM];
|
||||
int yc = d->comm.rank * (d->grid.jmax / d->comm.dims[JDIM]) +
|
||||
MIN(rest, d->comm.rank);
|
||||
double ys = dy * (yc + 0.5);
|
||||
double y;
|
||||
|
||||
// printf("RANK %d yc: %d ys: %f\n",d->comm.rank, yc, ys);
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
y = ys + dy * (j - 0.5);
|
||||
U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength);
|
||||
}
|
||||
}
|
||||
} else if (strcmp(d->problem, "backstep") == 0) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
if (S(0, j) == FLUID) U(0, j) = 1.0;
|
||||
}
|
||||
} else if (strcmp(d->problem, "karman") == 0) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
U(0, j) = 1.0;
|
||||
}
|
||||
}
|
||||
/* print(solver, solver->u); */
|
||||
}
|
||||
|
||||
void setObjectBoundaryCondition(Discretization* d)
|
||||
{
|
||||
int imaxLocal = d->comm.imaxLocal;
|
||||
int jmaxLocal = d->comm.jmaxLocal;
|
||||
double* u = d->u;
|
||||
double* v = d->v;
|
||||
double* s = d->grid.s;
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
switch ((int)S(i, j)) {
|
||||
case TOP:
|
||||
U(i, j) = -U(i, j + 1);
|
||||
U(i - 1, j) = -U(i - 1, j + 1);
|
||||
V(i, j) = 0.0;
|
||||
break;
|
||||
case BOTTOM:
|
||||
U(i, j) = -U(i, j - 1);
|
||||
U(i - 1, j) = -U(i - 1, j - 1);
|
||||
V(i, j) = 0.0;
|
||||
break;
|
||||
case LEFT:
|
||||
U(i - 1, j) = 0.0;
|
||||
V(i, j) = -V(i - 1, j);
|
||||
V(i, j - 1) = -V(i - 1, j - 1);
|
||||
break;
|
||||
case RIGHT:
|
||||
U(i, j) = 0.0;
|
||||
V(i, j) = -V(i + 1, j);
|
||||
V(i, j - 1) = -V(i + 1, j - 1);
|
||||
break;
|
||||
case TOPLEFT:
|
||||
U(i, j) = -U(i, j + 1);
|
||||
U(i - 1, j) = 0.0;
|
||||
V(i, j) = 0.0;
|
||||
V(i, j - 1) = -V(i - 1, j - 1);
|
||||
break;
|
||||
case TOPRIGHT:
|
||||
U(i, j) = 0.0;
|
||||
U(i - 1, j) = -U(i - 1, j + 1);
|
||||
V(i, j) = 0.0;
|
||||
V(i, j - 1) = -V(i + 1, j - 1);
|
||||
break;
|
||||
case BOTTOMLEFT:
|
||||
U(i, j) = -U(i, j - 1);
|
||||
U(i - 1, j) = 0.0;
|
||||
V(i, j) = -V(i - 1, j);
|
||||
V(i, j - 1) = 0.0;
|
||||
break;
|
||||
case BOTTOMRIGHT:
|
||||
U(i, j) = 0.0;
|
||||
U(i - 1, j) = -U(i - 1, j - 1);
|
||||
V(i, j) = -V(i, j + 1);
|
||||
V(i, j - 1) = 0.0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void computeFG(Discretization* d)
|
||||
{
|
||||
double* u = d->u;
|
||||
double* v = d->v;
|
||||
double* f = d->f;
|
||||
double* g = d->g;
|
||||
double* s = d->grid.s;
|
||||
|
||||
int imaxLocal = d->comm.imaxLocal;
|
||||
int jmaxLocal = d->comm.jmaxLocal;
|
||||
|
||||
double gx = d->gx;
|
||||
double gy = d->gy;
|
||||
double gamma = d->gamma;
|
||||
double dt = d->dt;
|
||||
double inverseRe = 1.0 / d->re;
|
||||
double inverseDx = 1.0 / d->grid.dx;
|
||||
double inverseDy = 1.0 / d->grid.dy;
|
||||
double du2dx, dv2dy, duvdx, duvdy;
|
||||
double du2dx2, du2dy2, dv2dx2, dv2dy2;
|
||||
|
||||
commExchange(&d->comm, u);
|
||||
commExchange(&d->comm, v);
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
if (S(i, j) == FLUID) {
|
||||
du2dx = inverseDx * 0.25 *
|
||||
((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) -
|
||||
(U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) +
|
||||
gamma * inverseDx * 0.25 *
|
||||
(fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) +
|
||||
fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j)));
|
||||
|
||||
duvdy = inverseDy * 0.25 *
|
||||
((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) -
|
||||
(V(i, j - 1) + V(i + 1, j - 1)) *
|
||||
(U(i, j) + U(i, j - 1))) +
|
||||
gamma * inverseDy * 0.25 *
|
||||
(fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) +
|
||||
fabs(V(i, j - 1) + V(i + 1, j - 1)) *
|
||||
(U(i, j) - U(i, j - 1)));
|
||||
|
||||
du2dx2 = inverseDx * inverseDx *
|
||||
(U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j));
|
||||
du2dy2 = inverseDy * inverseDy *
|
||||
(U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1));
|
||||
F(i, j) = U(i, j) +
|
||||
dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx);
|
||||
|
||||
duvdx = inverseDx * 0.25 *
|
||||
((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) -
|
||||
(U(i - 1, j) + U(i - 1, j + 1)) *
|
||||
(V(i, j) + V(i - 1, j))) +
|
||||
gamma * inverseDx * 0.25 *
|
||||
(fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) +
|
||||
fabs(U(i - 1, j) + U(i - 1, j + 1)) *
|
||||
(V(i, j) - V(i - 1, j)));
|
||||
|
||||
dv2dy = inverseDy * 0.25 *
|
||||
((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) -
|
||||
(V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) +
|
||||
gamma * inverseDy * 0.25 *
|
||||
(fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) +
|
||||
fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1)));
|
||||
|
||||
dv2dx2 = inverseDx * inverseDx *
|
||||
(V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j));
|
||||
dv2dy2 = inverseDy * inverseDy *
|
||||
(V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1));
|
||||
G(i, j) = V(i, j) +
|
||||
dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy);
|
||||
} else {
|
||||
switch ((int)S(i, j)) {
|
||||
case TOP:
|
||||
G(i, j) = V(i, j);
|
||||
break;
|
||||
case BOTTOM:
|
||||
G(i, j - 1) = V(i, j - 1);
|
||||
break;
|
||||
case LEFT:
|
||||
F(i - 1, j) = U(i - 1, j);
|
||||
break;
|
||||
case RIGHT:
|
||||
F(i, j) = U(i, j);
|
||||
break;
|
||||
case TOPLEFT:
|
||||
F(i - 1, j) = U(i - 1, j);
|
||||
G(i, j) = V(i, j);
|
||||
break;
|
||||
case TOPRIGHT:
|
||||
F(i, j) = U(i, j);
|
||||
G(i, j) = V(i, j);
|
||||
break;
|
||||
case BOTTOMLEFT:
|
||||
F(i - 1, j) = U(i - 1, j);
|
||||
G(i, j - 1) = V(i, j - 1);
|
||||
break;
|
||||
case BOTTOMRIGHT:
|
||||
F(i, j) = U(i, j);
|
||||
G(i, j - 1) = V(i, j - 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ----------------------------- boundary of F --------------------------- */
|
||||
if (commIsBoundary(&d->comm, L)) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
F(0, j) = U(0, j);
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&d->comm, R)) {
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
F(imaxLocal, j) = U(imaxLocal, j);
|
||||
}
|
||||
}
|
||||
|
||||
/* ----------------------------- boundary of G --------------------------- */
|
||||
if (commIsBoundary(&d->comm, B)) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
G(i, 0) = V(i, 0);
|
||||
}
|
||||
}
|
||||
|
||||
if (commIsBoundary(&d->comm, T)) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
G(i, jmaxLocal) = V(i, jmaxLocal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void adaptUV(Discretization* d)
|
||||
{
|
||||
int imaxLocal = d->comm.imaxLocal;
|
||||
int jmaxLocal = d->comm.jmaxLocal;
|
||||
|
||||
double* p = d->p;
|
||||
double* u = d->u;
|
||||
double* v = d->v;
|
||||
double* f = d->f;
|
||||
double* g = d->g;
|
||||
|
||||
double factorX = d->dt / d->grid.dx;
|
||||
double factorY = d->dt / d->grid.dy;
|
||||
|
||||
for (int j = 1; j < jmaxLocal + 1; j++) {
|
||||
for (int i = 1; i < imaxLocal + 1; i++) {
|
||||
U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX;
|
||||
V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void writeResult(Discretization* d, double* u, double* v, double* p)
|
||||
{
|
||||
int imax = d->grid.imax;
|
||||
int jmax = d->grid.jmax;
|
||||
double dx = d->grid.dx;
|
||||
double dy = d->grid.dy;
|
||||
double x = 0.0, y = 0.0;
|
||||
|
||||
FILE* fp;
|
||||
fp = fopen("pressure.dat", "w");
|
||||
|
||||
if (fp == NULL) {
|
||||
printf("Error!\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
for (int j = 1; j <= jmax; j++) {
|
||||
y = (double)(j - 0.5) * dy;
|
||||
for (int i = 1; i <= imax; i++) {
|
||||
x = (double)(i - 0.5) * dx;
|
||||
fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax + 2) + i]);
|
||||
}
|
||||
fprintf(fp, "\n");
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
|
||||
fp = fopen("velocity.dat", "w");
|
||||
|
||||
if (fp == NULL) {
|
||||
printf("Error!\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
for (int j = 1; j <= jmax; j++) {
|
||||
y = dy * (j - 0.5);
|
||||
for (int i = 1; i <= imax; i++) {
|
||||
x = dx * (i - 0.5);
|
||||
double velU = (u[j * (imax + 2) + i] + u[j * (imax + 2) + (i - 1)]) / 2.0;
|
||||
double velV = (v[j * (imax + 2) + i] + v[(j - 1) * (imax + 2) + i]) / 2.0;
|
||||
double len = sqrt((velU * velU) + (velV * velV));
|
||||
fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, velU, velV, len);
|
||||
}
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
}
|
66
EnhancedSolver/2D-mpi/src/discretization.h
Normal file
66
EnhancedSolver/2D-mpi/src/discretization.h
Normal file
@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#ifndef __DISCRETIZATION_H_
|
||||
#define __DISCRETIZATION_H_
|
||||
#include "comm.h"
|
||||
#include "grid.h"
|
||||
#include "parameter.h"
|
||||
#include<unistd.h>
|
||||
|
||||
enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
|
||||
|
||||
enum OBJECTBOUNDARY {
|
||||
FLUID = 0,
|
||||
TOP,
|
||||
BOTTOM,
|
||||
LEFT,
|
||||
RIGHT,
|
||||
TOPLEFT,
|
||||
BOTTOMLEFT,
|
||||
TOPRIGHT,
|
||||
BOTTOMRIGHT,
|
||||
OBSTACLE
|
||||
};
|
||||
|
||||
enum SHAPE { NOSHAPE = 0, RECT, CIRCLE };
|
||||
|
||||
typedef struct {
|
||||
/* geometry and grid information */
|
||||
Grid grid;
|
||||
/* arrays */
|
||||
double *p, *rhs;
|
||||
double *f, *g;
|
||||
double *u, *v;
|
||||
/* parameters */
|
||||
double re, tau, gamma;
|
||||
double gx, gy;
|
||||
/* time stepping */
|
||||
double dt, te;
|
||||
double dtBound;
|
||||
char* problem;
|
||||
|
||||
double xLocal, yLocal, xOffset, yOffset, xOffsetEnd, yOffsetEnd;
|
||||
|
||||
|
||||
int bcLeft, bcRight, bcBottom, bcTop;
|
||||
/* communication */
|
||||
Comm comm;
|
||||
} Discretization;
|
||||
|
||||
extern void initDiscretiztion(Discretization*, Parameter*);
|
||||
extern void computeRHS(Discretization*);
|
||||
extern void normalizePressure(Discretization*);
|
||||
extern void computeTimestep(Discretization*);
|
||||
extern void setBoundaryConditions(Discretization*);
|
||||
extern void setSpecialBoundaryCondition(Discretization*);
|
||||
extern void setObjectBoundaryCondition(Discretization*);
|
||||
extern void computeFG(Discretization*);
|
||||
extern void adaptUV(Discretization*);
|
||||
extern void writeResult(Discretization* s, double* u, double* v, double* p);
|
||||
extern double sumOffset(double* , int , int , int );
|
||||
extern void print(Discretization* , double* );
|
||||
#endif
|
17
EnhancedSolver/2D-mpi/src/grid.h
Normal file
17
EnhancedSolver/2D-mpi/src/grid.h
Normal file
@ -0,0 +1,17 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#ifndef __GRID_H_
|
||||
#define __GRID_H_
|
||||
|
||||
typedef struct {
|
||||
double dx, dy;
|
||||
int imax, jmax;
|
||||
double xlength, ylength;
|
||||
double* s;
|
||||
} Grid;
|
||||
|
||||
#endif // __GRID_H_
|
54
EnhancedSolver/2D-mpi/src/likwid-marker.h
Normal file
54
EnhancedSolver/2D-mpi/src/likwid-marker.h
Normal file
@ -0,0 +1,54 @@
|
||||
/*
|
||||
* =======================================================================================
|
||||
*
|
||||
* Author: Jan Eitzinger (je), jan.eitzinger@fau.de
|
||||
* Copyright (c) 2020 RRZE, University Erlangen-Nuremberg
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*
|
||||
* =======================================================================================
|
||||
*/
|
||||
#ifndef LIKWID_MARKERS_H
|
||||
#define LIKWID_MARKERS_H
|
||||
|
||||
#ifdef LIKWID_PERFMON
|
||||
#include <likwid.h>
|
||||
#define LIKWID_MARKER_INIT likwid_markerInit()
|
||||
#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit()
|
||||
#define LIKWID_MARKER_SWITCH likwid_markerNextGroup()
|
||||
#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag)
|
||||
#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag)
|
||||
#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag)
|
||||
#define LIKWID_MARKER_CLOSE likwid_markerClose()
|
||||
#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag)
|
||||
#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \
|
||||
likwid_markerGetRegion(regionTag, nevents, events, time, count)
|
||||
#else /* LIKWID_PERFMON */
|
||||
#define LIKWID_MARKER_INIT
|
||||
#define LIKWID_MARKER_THREADINIT
|
||||
#define LIKWID_MARKER_SWITCH
|
||||
#define LIKWID_MARKER_REGISTER(regionTag)
|
||||
#define LIKWID_MARKER_START(regionTag)
|
||||
#define LIKWID_MARKER_STOP(regionTag)
|
||||
#define LIKWID_MARKER_CLOSE
|
||||
#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count)
|
||||
#define LIKWID_MARKER_RESET(regionTag)
|
||||
#endif /* LIKWID_PERFMON */
|
||||
|
||||
#endif /*LIKWID_MARKERS_H*/
|
125
EnhancedSolver/2D-mpi/src/main.c
Normal file
125
EnhancedSolver/2D-mpi/src/main.c
Normal file
@ -0,0 +1,125 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved.
|
||||
* Use of this source code is governed by a MIT-style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "allocate.h"
|
||||
#include "comm.h"
|
||||
#include "discretization.h"
|
||||
#include "grid.h"
|
||||
#include "parameter.h"
|
||||
#include "particletracing.h"
|
||||
#include "progress.h"
|
||||
#include "timing.h"
|
||||
|
||||
static void writeResults(Discretization* s)
|
||||
{
|
||||
#ifdef _MPI
|
||||
size_t bytesize = (s->grid.imax + 2) * (s->grid.jmax + 2) * sizeof(double);
|
||||
|
||||
double* ug = allocate(64, bytesize);
|
||||
double* vg = allocate(64, bytesize);
|
||||
double* pg = allocate(64, bytesize);
|
||||
|
||||
commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->grid.imax, s->grid.jmax);
|
||||
if (commIsMaster(&s->comm)) {
|
||||
writeResult(s, ug, vg, pg);
|
||||
}
|
||||
|
||||
free(ug);
|
||||
free(vg);
|
||||
free(pg);
|
||||
#else
|
||||
writeResult(s, s->u, s->v, s->p);
|
||||
#endif
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
int rank;
|
||||
double timeStart, timeStop;
|
||||
Parameter p;
|
||||
Discretization d;
|
||||
Solver s;
|
||||
ParticleTracer particletracer;
|
||||
|
||||
commInit(&d.comm, argc, argv);
|
||||
initParameter(&p);
|
||||
FILE* fp;
|
||||
fp = initResidualWriter();
|
||||
|
||||
if (argc != 2) {
|
||||
printf("Usage: %s <configFile>\n", argv[0]);
|
||||
exit(EXIT_SUCCESS);
|
||||
}
|
||||
|
||||
readParameter(&p, argv[1]);
|
||||
commPartition(&d.comm, p.jmax, p.imax);
|
||||
if (commIsMaster(&d.comm)) {
|
||||
printParameter(&p);
|
||||
}
|
||||
|
||||
initDiscretiztion(&d, &p);
|
||||
initSolver(&s, &d, &p);
|
||||
initParticleTracer(&particletracer, &d, &p);
|
||||
|
||||
#ifdef TEST
|
||||
commPrintConfig(&d.comm);
|
||||
commTestInit(&d.comm, d.p, d.f, d.g);
|
||||
commExchange(&d.comm, d.p);
|
||||
commShift(&d.comm, d.f, d.g);
|
||||
commTestWrite(&d.comm, d.p, d.f, d.g);
|
||||
writeResults(&d);
|
||||
commFinalize(&d.comm);
|
||||
exit(EXIT_SUCCESS);
|
||||
#endif
|
||||
#ifndef VERBOSE
|
||||
initProgress(d.te);
|
||||
#endif
|
||||
|
||||
double tau = d.tau;
|
||||
double te = d.te;
|
||||
double t = 0.0;
|
||||
double res = 0.0;
|
||||
timeStart = getTimeStamp();
|
||||
while (t <= te) {
|
||||
if (tau > 0.0) computeTimestep(&d);
|
||||
setBoundaryConditions(&d);
|
||||
setSpecialBoundaryCondition(&d);
|
||||
setObjectBoundaryCondition(&d);
|
||||
computeFG(&d);
|
||||
computeRHS(&d);
|
||||
res = solve(&s, d.p, d.rhs);
|
||||
adaptUV(&d);
|
||||
trace(&particletracer, &d, t);
|
||||
|
||||
writeResidual(fp, t, res);
|
||||
|
||||
t += d.dt;
|
||||
|
||||
#ifdef VERBOSE
|
||||
if (commIsMaster(s.comm)) {
|
||||
printf("TIME %f , TIMESTEP %f\n", t, d.dt);
|
||||
}
|
||||
#else
|
||||
printProgress(t);
|
||||
#endif
|
||||
}
|
||||
timeStop = getTimeStamp();
|
||||
#ifndef VERBOSE
|
||||
stopProgress();
|
||||
#endif
|
||||
if (commIsMaster(s.comm)) {
|
||||
printf("Solution took %.2fs\n", timeStop - timeStart);
|
||||
}
|
||||
fclose(fp);
|
||||
freeParticles(&particletracer);
|
||||
writeResults(&d);
|
||||
commFinalize(s.comm);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
143
EnhancedSolver/2D-mpi/src/parameter.c
Normal file
143
EnhancedSolver/2D-mpi/src/parameter.c
Normal file
@ -0,0 +1,143 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "parameter.h"
|
||||
#include "util.h"
|
||||
#define MAXLINE 4096
|
||||
|
||||
void initParameter(Parameter* param)
|
||||
{
|
||||
param->xlength = 1.0;
|
||||
param->ylength = 1.0;
|
||||
param->imax = 100;
|
||||
param->jmax = 100;
|
||||
param->itermax = 1000;
|
||||
param->eps = 0.0001;
|
||||
param->omg = 1.8;
|
||||
param->levels = 5;
|
||||
param->presmooth = 5;
|
||||
param->postsmooth = 5;
|
||||
}
|
||||
|
||||
void readParameter(Parameter* param, const char* filename)
|
||||
{
|
||||
FILE* fp = fopen(filename, "r");
|
||||
char line[MAXLINE];
|
||||
int i;
|
||||
|
||||
if (!fp) {
|
||||
fprintf(stderr, "Could not open parameter file: %s\n", filename);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
while (!feof(fp)) {
|
||||
line[0] = '\0';
|
||||
fgets(line, MAXLINE, fp);
|
||||
for (i = 0; line[i] != '\0' && line[i] != '#'; i++)
|
||||
;
|
||||
line[i] = '\0';
|
||||
|
||||
char* tok = strtok(line, " ");
|
||||
char* val = strtok(NULL, " ");
|
||||
|
||||
#define PARSE_PARAM(p, f) \
|
||||
if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \
|
||||
param->p = f(val); \
|
||||
}
|
||||
#define PARSE_STRING(p) PARSE_PARAM(p, strdup)
|
||||
#define PARSE_INT(p) PARSE_PARAM(p, atoi)
|
||||
#define PARSE_REAL(p) PARSE_PARAM(p, atof)
|
||||
|
||||
if (tok != NULL && val != NULL) {
|
||||
PARSE_REAL(xlength);
|
||||
PARSE_REAL(ylength);
|
||||
PARSE_INT(imax);
|
||||
PARSE_INT(jmax);
|
||||
PARSE_INT(itermax);
|
||||
PARSE_REAL(eps);
|
||||
PARSE_REAL(omg);
|
||||
PARSE_REAL(re);
|
||||
PARSE_REAL(tau);
|
||||
PARSE_REAL(gamma);
|
||||
PARSE_REAL(dt);
|
||||
PARSE_REAL(te);
|
||||
PARSE_REAL(gx);
|
||||
PARSE_REAL(gy);
|
||||
PARSE_STRING(name);
|
||||
PARSE_INT(bcLeft);
|
||||
PARSE_INT(bcRight);
|
||||
PARSE_INT(bcBottom);
|
||||
PARSE_INT(bcTop);
|
||||
PARSE_INT(levels);
|
||||
PARSE_INT(presmooth);
|
||||
PARSE_INT(postsmooth);
|
||||
PARSE_REAL(u_init);
|
||||
PARSE_REAL(v_init);
|
||||
PARSE_REAL(p_init);
|
||||
|
||||
/* Added new particle tracing parameters */
|
||||
PARSE_INT(numberOfParticles);
|
||||
PARSE_REAL(startTime);
|
||||
PARSE_REAL(injectTimePeriod);
|
||||
PARSE_REAL(writeTimePeriod);
|
||||
PARSE_REAL(x1);
|
||||
PARSE_REAL(y1);
|
||||
PARSE_REAL(x2);
|
||||
PARSE_REAL(y2);
|
||||
|
||||
/* Added obstacle geometry parameters */
|
||||
PARSE_INT(shape);
|
||||
PARSE_REAL(xCenter);
|
||||
PARSE_REAL(yCenter);
|
||||
PARSE_REAL(xRectLength);
|
||||
PARSE_REAL(yRectLength);
|
||||
PARSE_REAL(circleRadius);
|
||||
}
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
void printParameter(Parameter* param)
|
||||
{
|
||||
printf("Parameters for %s\n", param->name);
|
||||
printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n",
|
||||
param->bcLeft,
|
||||
param->bcRight,
|
||||
param->bcBottom,
|
||||
param->bcTop);
|
||||
printf("\tReynolds number: %.2f\n", param->re);
|
||||
printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n",
|
||||
param->u_init,
|
||||
param->v_init,
|
||||
param->p_init);
|
||||
printf("Geometry data:\n");
|
||||
printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength);
|
||||
printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax);
|
||||
printf("Timestep parameters:\n");
|
||||
printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te);
|
||||
printf("\tTau factor: %.2f\n", param->tau);
|
||||
printf("Iterative solver parameters:\n");
|
||||
printf("\tMax iterations: %d\n", param->itermax);
|
||||
printf("\tepsilon (stopping tolerance) : %f\n", param->eps);
|
||||
printf("\tgamma (stopping tolerance) : %f\n", param->gamma);
|
||||
printf("\tomega (SOR relaxation): %f\n", param->omg);
|
||||
printf("Particle tracer parameters:\n");
|
||||
printf("\tNumber of particles: %d\n", param->numberOfParticles);
|
||||
printf("\tstartTime : %f\n", param->startTime);
|
||||
printf("\tinjecTimePeriod : %f\n", param->injectTimePeriod);
|
||||
printf("\twriteTimePeriod: %f\n", param->writeTimePeriod);
|
||||
printf("\tx1 : %f, x2 : %f, y1 : %f, y2 : %f\n",
|
||||
param->x1,
|
||||
param->x2,
|
||||
param->y1,
|
||||
param->y2);
|
||||
printf("\tShape : %d\n", param->shape);
|
||||
}
|
38
EnhancedSolver/2D-mpi/src/parameter.h
Normal file
38
EnhancedSolver/2D-mpi/src/parameter.h
Normal file
@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#ifndef __PARAMETER_H_
|
||||
#define __PARAMETER_H_
|
||||
|
||||
typedef struct {
|
||||
double xlength, ylength;
|
||||
int imax, jmax;
|
||||
int itermax;
|
||||
double eps, omg;
|
||||
double re, tau, gamma;
|
||||
double te, dt;
|
||||
double gx, gy;
|
||||
char* name;
|
||||
int bcLeft, bcRight, bcBottom, bcTop;
|
||||
double u_init, v_init, p_init;
|
||||
|
||||
int levels, presmooth, postsmooth;
|
||||
|
||||
int numberOfParticles;
|
||||
double startTime, injectTimePeriod, writeTimePeriod;
|
||||
|
||||
double x1, y1, x2, y2;
|
||||
|
||||
int shape;
|
||||
double xCenter, yCenter, xRectLength, yRectLength, circleRadius;
|
||||
|
||||
|
||||
} Parameter;
|
||||
|
||||
void initParameter(Parameter*);
|
||||
void readParameter(Parameter*, const char*);
|
||||
void printParameter(Parameter*);
|
||||
#endif
|
675
EnhancedSolver/2D-mpi/src/particletracing.c
Normal file
675
EnhancedSolver/2D-mpi/src/particletracing.c
Normal file
@ -0,0 +1,675 @@
|
||||
/*
|
||||
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <float.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "particletracing.h"
|
||||
|
||||
#define U(i, j) u[(j) * (imaxLocal + 2) + (i)]
|
||||
#define V(i, j) v[(j) * (imaxLocal + 2) + (i)]
|
||||
#define S(i, j) s[(j) * (imaxLocal + 2) + (i)]
|
||||
|
||||
static int ts = 0;
|
||||
|
||||
#define IDIM 0
|
||||
#define JDIM 1
|
||||
|
||||
#define XOFFSET 0
|
||||
#define YOFFSET 1
|
||||
#define XOFFSETEND 2
|
||||
#define YOFFSETEND 3
|
||||
|
||||
static double sum(int* sizes, int size)
|
||||
{
|
||||
double sum = 0;
|
||||
|
||||
for (int i = 0; i < size; ++i) {
|
||||
sum += sizes[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
void printUV(ParticleTracer* particletracer, double* u, double* v)
|
||||
{
|
||||
int imaxLocal = particletracer->imaxLocal;
|
||||
|
||||
for (int i = 0; i < particletracer->size; i++) {
|
||||
if (i == particletracer->rank) {
|
||||
printf(
|
||||
"\n### RANK %d #######################################################\n",
|
||||
particletracer->rank);
|
||||
printf("\nGrid U : \n");
|
||||
for (int j = 0; j < particletracer->jmaxLocal + 2; j++) {
|
||||
printf("%02d: ", j);
|
||||
for (int i = 0; i < particletracer->imaxLocal + 2; i++) {
|
||||
printf("%4.2f ", u[j * (imaxLocal + 2) + i]);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
fflush(stdout);
|
||||
printf("\nGrid V : \n");
|
||||
for (int j = 0; j < particletracer->jmaxLocal + 2; j++) {
|
||||
printf("%02d: ", j);
|
||||
for (int i = 0; i < particletracer->imaxLocal + 2; i++) {
|
||||
printf("%4.2f ", v[j * (imaxLocal + 2) + i]);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
#ifdef _MPI
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
void initParticleTracer(
|
||||
ParticleTracer* particletracer, Discretization* d, Parameter* params)
|
||||
{
|
||||
int dims[NDIMS] = { 0, 0 };
|
||||
int periods[NDIMS] = { 0, 0 };
|
||||
|
||||
/* initializing local properties from params */
|
||||
particletracer->rank = d->comm.rank;
|
||||
particletracer->size = d->comm.size;
|
||||
|
||||
particletracer->numberOfParticles = params->numberOfParticles;
|
||||
particletracer->startTime = params->startTime;
|
||||
particletracer->injectTimePeriod = params->injectTimePeriod;
|
||||
particletracer->writeTimePeriod = params->writeTimePeriod;
|
||||
|
||||
particletracer->dt = params->dt;
|
||||
particletracer->dx = params->xlength / params->imax;
|
||||
particletracer->dy = params->ylength / params->jmax;
|
||||
|
||||
particletracer->xlength = params->xlength;
|
||||
particletracer->ylength = params->ylength;
|
||||
|
||||
particletracer->x1 = params->x1;
|
||||
particletracer->y1 = params->y1;
|
||||
particletracer->x2 = params->x2;
|
||||
particletracer->y2 = params->y2;
|
||||
|
||||
particletracer->lastInjectTime = params->startTime;
|
||||
particletracer->lastUpdateTime = params->startTime;
|
||||
particletracer->lastWriteTime = params->startTime;
|
||||
|
||||
particletracer->pointer = 0;
|
||||
particletracer->totalParticles = 0;
|
||||
particletracer->removedParticles = 0;
|
||||
|
||||
particletracer->imax = params->imax;
|
||||
particletracer->jmax = params->jmax;
|
||||
|
||||
particletracer->imaxLocal = d->comm.imaxLocal;
|
||||
particletracer->jmaxLocal = d->comm.jmaxLocal;
|
||||
particletracer->removedParticles = 0;
|
||||
|
||||
// Estimating the number of particles over the number of timesteps that could be
|
||||
// required.
|
||||
particletracer->estimatedNumParticles = (particletracer->imaxLocal *
|
||||
particletracer->jmaxLocal);
|
||||
|
||||
// Allocating memory for the estimated particles over the timesteps.
|
||||
particletracer->particlePool = malloc(
|
||||
sizeof(Particle) * particletracer->estimatedNumParticles);
|
||||
|
||||
// Initializing the number of particles to 0 and turning OFF all of the particles.
|
||||
// Contain information in x and y length metric, not i and j discretization metrics.
|
||||
for (int i = 0; i < particletracer->estimatedNumParticles; ++i) {
|
||||
particletracer->particlePool[i].x = 0.0;
|
||||
particletracer->particlePool[i].y = 0.0;
|
||||
particletracer->particlePool[i].flag = false;
|
||||
}
|
||||
|
||||
// Creating a linearly spaced particle line.
|
||||
particletracer->linSpaceLine = malloc(
|
||||
sizeof(Particle) * particletracer->numberOfParticles);
|
||||
|
||||
// Creating an array for each rank that will hold information about
|
||||
// offsets from other ranks. Holds each ranks x and y length metrics.
|
||||
particletracer->offset = (double*)malloc(sizeof(double) * 4 * particletracer->size);
|
||||
|
||||
// Calculating each ranks local x and y length metrics.
|
||||
|
||||
double offset[4][particletracer->size];
|
||||
|
||||
// Calculating each ranks x and y local lengths.
|
||||
particletracer->xLocal = d->xLocal;
|
||||
particletracer->yLocal = d->yLocal;
|
||||
|
||||
double xLocal[particletracer->size];
|
||||
double yLocal[particletracer->size];
|
||||
|
||||
// Calculate own x and y length metric offset based on other ranks offset data.
|
||||
particletracer->xOffset = d->xOffset;
|
||||
particletracer->yOffset = d->yOffset;
|
||||
particletracer->xOffsetEnd = d->xOffsetEnd;
|
||||
particletracer->yOffsetEnd = d->yOffsetEnd;
|
||||
|
||||
printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : "
|
||||
"%.2f\n",
|
||||
particletracer->rank,
|
||||
particletracer->xOffset,
|
||||
particletracer->yOffset,
|
||||
particletracer->xOffsetEnd,
|
||||
particletracer->yOffsetEnd);
|
||||
|
||||
#ifdef _MPI
|
||||
// Gather each ranks x and y length metric that marks each ranks own territory.
|
||||
// Once the boundary leaves local domain, then it needs to know which ranks to send.
|
||||
// And to know whos boundary it is, we need to know the rank.
|
||||
MPI_Allgather(&particletracer->xOffset,
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
offset[0],
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
d->comm.comm);
|
||||
MPI_Allgather(&particletracer->yOffset,
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
offset[1],
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
d->comm.comm);
|
||||
MPI_Allgather(&particletracer->xOffsetEnd,
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
offset[2],
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
d->comm.comm);
|
||||
MPI_Allgather(&particletracer->yOffsetEnd,
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
offset[3],
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
d->comm.comm);
|
||||
#endif
|
||||
|
||||
memcpy(particletracer->offset, offset, sizeof(offset));
|
||||
|
||||
particleRandomizer(particletracer);
|
||||
|
||||
#ifdef _MPI
|
||||
// Create the mpi_particle datatype
|
||||
MPI_Datatype mpi_particle;
|
||||
int lengths[3] = { 1, 1, 1 };
|
||||
|
||||
MPI_Aint displacements[3];
|
||||
Particle dummy_particle;
|
||||
MPI_Aint base_address;
|
||||
MPI_Get_address(&dummy_particle, &base_address);
|
||||
MPI_Get_address(&dummy_particle.x, &displacements[0]);
|
||||
MPI_Get_address(&dummy_particle.y, &displacements[1]);
|
||||
MPI_Get_address(&dummy_particle.flag, &displacements[2]);
|
||||
displacements[0] = MPI_Aint_diff(displacements[0], base_address);
|
||||
displacements[1] = MPI_Aint_diff(displacements[1], base_address);
|
||||
displacements[2] = MPI_Aint_diff(displacements[2], base_address);
|
||||
|
||||
MPI_Datatype types[3] = { MPI_DOUBLE, MPI_DOUBLE, MPI_C_BOOL };
|
||||
MPI_Type_create_struct(3,
|
||||
lengths,
|
||||
displacements,
|
||||
types,
|
||||
&particletracer->mpi_particle);
|
||||
MPI_Type_commit(&particletracer->mpi_particle);
|
||||
#endif
|
||||
}
|
||||
|
||||
void printParticles(ParticleTracer* particletracer)
|
||||
{
|
||||
for (int i = 0; i < particletracer->totalParticles; ++i) {
|
||||
printf("Rank : %d Particle position X : %.2f, Y : %.2f, flag : %d, total pt : "
|
||||
"%d, pointer : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, "
|
||||
"yOffsetEnd : %.2f\n",
|
||||
particletracer->rank,
|
||||
particletracer->particlePool[i].x,
|
||||
particletracer->particlePool[i].y,
|
||||
particletracer->particlePool[i].flag,
|
||||
particletracer->totalParticles,
|
||||
particletracer->pointer,
|
||||
particletracer->xOffset,
|
||||
particletracer->yOffset,
|
||||
particletracer->xOffsetEnd,
|
||||
particletracer->yOffsetEnd);
|
||||
}
|
||||
}
|
||||
void injectParticles(ParticleTracer* particletracer, double* s)
|
||||
{
|
||||
double x, y;
|
||||
compress(particletracer);
|
||||
particleRandomizer(particletracer);
|
||||
|
||||
int imaxLocal = particletracer->imaxLocal;
|
||||
int jmaxLocal = particletracer->jmaxLocal;
|
||||
|
||||
for (int i = 0; i < particletracer->numberOfParticles; ++i) {
|
||||
x = particletracer->linSpaceLine[i].x;
|
||||
y = particletracer->linSpaceLine[i].y;
|
||||
if (x >= particletracer->xOffset && y >= particletracer->yOffset &&
|
||||
x <= particletracer->xOffsetEnd && y <= particletracer->yOffsetEnd) {
|
||||
|
||||
particletracer->particlePool[particletracer->pointer].x = x;
|
||||
particletracer->particlePool[particletracer->pointer].y = y;
|
||||
|
||||
int i = particletracer->particlePool[particletracer->pointer].x /
|
||||
particletracer->dx;
|
||||
int j = particletracer->particlePool[particletracer->pointer].y /
|
||||
particletracer->dy;
|
||||
|
||||
int iOffset = particletracer->xOffset / particletracer->dx,
|
||||
jOffset = particletracer->yOffset / particletracer->dy;
|
||||
|
||||
if (S(i - iOffset, j - jOffset) == FLUID) {
|
||||
particletracer->particlePool[particletracer->pointer].flag = true;
|
||||
++(particletracer->pointer);
|
||||
++(particletracer->totalParticles);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void advanceParticles(ParticleTracer* particletracer,
|
||||
double* restrict u,
|
||||
double* restrict v,
|
||||
double* restrict s,
|
||||
Comm* comm,
|
||||
double time)
|
||||
{
|
||||
|
||||
int imax = particletracer->imax;
|
||||
int jmax = particletracer->jmax;
|
||||
int imaxLocal = particletracer->imaxLocal;
|
||||
int jmaxLocal = particletracer->jmaxLocal;
|
||||
|
||||
double dx = particletracer->dx;
|
||||
double dy = particletracer->dy;
|
||||
|
||||
double xlength = particletracer->xlength;
|
||||
double ylength = particletracer->ylength;
|
||||
|
||||
Particle buff[particletracer->size][(particletracer->estimatedNumParticles)];
|
||||
memset(buff, 0, sizeof(buff));
|
||||
Particle recvbuff[particletracer->size][(particletracer->estimatedNumParticles)];
|
||||
memset(buff, 0, sizeof(recvbuff));
|
||||
|
||||
int particleBufIndex[particletracer->size],
|
||||
recvparticleBufIndex[particletracer->size];
|
||||
|
||||
memset(particleBufIndex, 0, sizeof(particleBufIndex));
|
||||
memset(recvparticleBufIndex, 0, sizeof(recvparticleBufIndex));
|
||||
|
||||
for (int i = 0; i < particletracer->totalParticles; ++i) {
|
||||
if (particletracer->particlePool[i].flag == true) {
|
||||
double xTemp = particletracer->particlePool[i].x;
|
||||
double yTemp = particletracer->particlePool[i].y;
|
||||
|
||||
double x = xTemp - particletracer->xOffset;
|
||||
double y = yTemp - particletracer->yOffset;
|
||||
|
||||
int iCoord = (int)(x / dx) + 1;
|
||||
int jCoord = (int)((y + 0.5 * dy) / dy) + 1;
|
||||
|
||||
double x1 = (double)(iCoord - 1) * dx;
|
||||
double y1 = ((double)(jCoord - 1) - 0.5) * dy;
|
||||
double x2 = (double)iCoord * dx;
|
||||
double y2 = ((double)jCoord - 0.5) * dy;
|
||||
|
||||
double u_n = (1.0 / (dx * dy)) *
|
||||
((x2 - x) * (y2 - y) * U(iCoord - 1, jCoord - 1) +
|
||||
(x - x1) * (y2 - y) * U(iCoord, jCoord - 1) +
|
||||
(x2 - x) * (y - y1) * U(iCoord - 1, jCoord) +
|
||||
(x - x1) * (y - y1) * U(iCoord, jCoord));
|
||||
|
||||
double new_x = (x + particletracer->xOffset) + particletracer->dt * u_n;
|
||||
particletracer->particlePool[i].x = new_x;
|
||||
|
||||
iCoord = (int)((x + 0.5 * dx) / dx) + 1;
|
||||
jCoord = (int)(y / dy) + 1;
|
||||
|
||||
x1 = ((double)(iCoord - 1) - 0.5) * dx;
|
||||
y1 = (double)(jCoord - 1) * dy;
|
||||
x2 = ((double)iCoord - 0.5) * dx;
|
||||
y2 = (double)jCoord * dy;
|
||||
|
||||
double v_n = (1.0 / (dx * dy)) *
|
||||
((x2 - x) * (y2 - y) * V(iCoord - 1, jCoord - 1) +
|
||||
(x - x1) * (y2 - y) * V(iCoord, jCoord - 1) +
|
||||
(x2 - x) * (y - y1) * V(iCoord - 1, jCoord) +
|
||||
(x - x1) * (y - y1) * V(iCoord, jCoord));
|
||||
|
||||
double new_y = (y + particletracer->yOffset) + particletracer->dt * v_n;
|
||||
particletracer->particlePool[i].y = new_y;
|
||||
|
||||
if (((new_x < particletracer->xOffset) ||
|
||||
(new_x > particletracer->xOffsetEnd) ||
|
||||
(new_y < particletracer->yOffset) ||
|
||||
(new_y > particletracer->yOffsetEnd))) {
|
||||
// New logic to transfer particles to neighbouring ranks or discard the
|
||||
// particle.
|
||||
#ifdef _MPI
|
||||
for (int i = 0; i < particletracer->size; ++i) {
|
||||
if ((new_x >=
|
||||
particletracer->offset[i + particletracer->size * XOFFSET]) &&
|
||||
(new_x <= particletracer
|
||||
->offset[i + particletracer->size * XOFFSETEND]) &&
|
||||
(new_y >=
|
||||
particletracer->offset[i + particletracer->size * YOFFSET]) &&
|
||||
(new_y <= particletracer
|
||||
->offset[i + particletracer->size * YOFFSETEND]) &&
|
||||
i != particletracer->rank) {
|
||||
buff[i][particleBufIndex[i]].x = new_x;
|
||||
buff[i][particleBufIndex[i]].y = new_y;
|
||||
buff[i][particleBufIndex[i]].flag = true;
|
||||
++particleBufIndex[i];
|
||||
}
|
||||
}
|
||||
#endif
|
||||
particletracer->particlePool[i].flag = false;
|
||||
particletracer->removedParticles++;
|
||||
}
|
||||
|
||||
int i_new = new_x / dx, j_new = new_y / dy;
|
||||
int iOffset = particletracer->xOffset / dx,
|
||||
jOffset = particletracer->yOffset / dy;
|
||||
|
||||
if (S(i_new - iOffset, j_new - jOffset) != FLUID) {
|
||||
particletracer->particlePool[i].flag = false;
|
||||
particletracer->removedParticles++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef _MPI
|
||||
for (int i = 0; i < particletracer->size; ++i) {
|
||||
if (i != comm->rank) {
|
||||
MPI_Send(&particleBufIndex[i], 1, MPI_INT, i, 0, comm->comm);
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < particletracer->size; ++i) {
|
||||
if (i != particletracer->rank) {
|
||||
MPI_Recv(&recvparticleBufIndex[i],
|
||||
1,
|
||||
MPI_INT,
|
||||
i,
|
||||
0,
|
||||
comm->comm,
|
||||
MPI_STATUS_IGNORE);
|
||||
|
||||
// if (0 !=recvparticleBufIndex[i]) {
|
||||
// printf("Rank %d will receive %d particles from rank %d\n",
|
||||
// particletracer->rank,
|
||||
// recvparticleBufIndex[i],
|
||||
// i);
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < particletracer->size; ++i) {
|
||||
if (i != particletracer->rank) {
|
||||
MPI_Send(buff[i],
|
||||
particleBufIndex[i],
|
||||
particletracer->mpi_particle,
|
||||
i,
|
||||
0,
|
||||
comm->comm);
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < particletracer->size; ++i) {
|
||||
if (i != particletracer->rank) {
|
||||
MPI_Recv(recvbuff[i],
|
||||
recvparticleBufIndex[i],
|
||||
particletracer->mpi_particle,
|
||||
i,
|
||||
0,
|
||||
comm->comm,
|
||||
MPI_STATUS_IGNORE);
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < particletracer->size; ++i) {
|
||||
if (i != particletracer->rank) {
|
||||
for (int j = 0; j < recvparticleBufIndex[i]; ++j) {
|
||||
particletracer->particlePool[particletracer->pointer].x = recvbuff[i][j]
|
||||
.x;
|
||||
particletracer->particlePool[particletracer->pointer].y = recvbuff[i][j]
|
||||
.y;
|
||||
particletracer->particlePool[particletracer->pointer].flag = true;
|
||||
++(particletracer->pointer);
|
||||
++(particletracer->totalParticles);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void freeParticles(ParticleTracer* particletracer)
|
||||
{
|
||||
free(particletracer->particlePool);
|
||||
free(particletracer->linSpaceLine);
|
||||
free(particletracer->offset);
|
||||
}
|
||||
|
||||
void writeParticles(ParticleTracer* particletracer, Comm* comm)
|
||||
{
|
||||
int collectedBuffIndex[particletracer->size];
|
||||
compress(particletracer);
|
||||
#ifdef _MPI
|
||||
|
||||
MPI_Gather(&particletracer->totalParticles,
|
||||
1,
|
||||
MPI_INT,
|
||||
collectedBuffIndex,
|
||||
1,
|
||||
MPI_INT,
|
||||
0,
|
||||
comm->comm);
|
||||
|
||||
if (particletracer->rank != 0) {
|
||||
Particle buff[particletracer->totalParticles];
|
||||
for (int i = 0; i < particletracer->totalParticles; ++i) {
|
||||
buff[i].x = particletracer->particlePool[i].x;
|
||||
buff[i].y = particletracer->particlePool[i].y;
|
||||
buff[i].flag = particletracer->particlePool[i].flag;
|
||||
// printf("Rank : %d sending to rank 0 X : %.2f, Y : %.2f with totalpt :
|
||||
// %d\n", particletracer->rank, buff[i].x, buff[i].y,
|
||||
// particletracer->totalParticles);
|
||||
}
|
||||
MPI_Send(buff,
|
||||
particletracer->totalParticles,
|
||||
particletracer->mpi_particle,
|
||||
0,
|
||||
1,
|
||||
comm->comm);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (particletracer->rank == 0) {
|
||||
char filename[50];
|
||||
FILE* fp;
|
||||
|
||||
snprintf(filename, 50, "vis_files/particles_%d.dat", ts);
|
||||
fp = fopen(filename, "w");
|
||||
|
||||
if (fp == NULL) {
|
||||
printf("Error!\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
// fprintf(fp, "# vtk DataFile Version 3.0\n");
|
||||
// fprintf(fp, "PAMPI cfd solver particle tracing file\n");
|
||||
// fprintf(fp, "ASCII\n");
|
||||
|
||||
// fprintf(fp, "DATASET UNSTRUCTURED_GRID\n");
|
||||
// fprintf(fp, "FIELD FieldData 2\n");
|
||||
// fprintf(fp, "TIME 1 1 double\n");
|
||||
// fprintf(fp, "%d\n", ts);
|
||||
// fprintf(fp, "CYCLE 1 1 int\n");
|
||||
// fprintf(fp, "1\n");
|
||||
|
||||
#ifdef _MPI
|
||||
int overallTotalParticles = sum(collectedBuffIndex, particletracer->size);
|
||||
|
||||
// fprintf(fp, "POINTS %d float\n", overallTotalParticles);
|
||||
|
||||
// printf("Total particles : %d\n", overallTotalParticles);
|
||||
for (int i = 1; i < particletracer->size; ++i) {
|
||||
Particle recvBuff[collectedBuffIndex[i]];
|
||||
MPI_Recv(&recvBuff,
|
||||
collectedBuffIndex[i],
|
||||
particletracer->mpi_particle,
|
||||
i,
|
||||
1,
|
||||
comm->comm,
|
||||
MPI_STATUS_IGNORE);
|
||||
|
||||
for (int j = 0; j < collectedBuffIndex[i]; ++j) {
|
||||
double x = recvBuff[j].x;
|
||||
double y = recvBuff[j].y;
|
||||
fprintf(fp, "%f %f\n", x, y);
|
||||
|
||||
// printf("Rank : 0 receiving from rank %d X : %.2f, Y : %.2f with totalpt
|
||||
// : %d\n", i, x, y, particletracer->totalParticles);
|
||||
}
|
||||
}
|
||||
#else
|
||||
int overallTotalParticles = particletracer->totalParticles;
|
||||
|
||||
// fprintf(fp, "POINTS %d float\n", overallTotalParticles);
|
||||
|
||||
// printf("Total particles : %d\n", overallTotalParticles);
|
||||
#endif
|
||||
for (int i = 0; i < particletracer->totalParticles; ++i) {
|
||||
double x = particletracer->particlePool[i].x;
|
||||
double y = particletracer->particlePool[i].y;
|
||||
fprintf(fp, "%f %f\n", x, y);
|
||||
}
|
||||
|
||||
// fprintf(fp, "CELLS %d %d\n", overallTotalParticles, 2 * overallTotalParticles);
|
||||
|
||||
// for (int i = 0; i < overallTotalParticles; ++i)
|
||||
// {
|
||||
// fprintf(fp, "1 %d\n", i);
|
||||
// }
|
||||
|
||||
// fprintf(fp, "CELL_TYPES %d\n", overallTotalParticles);
|
||||
|
||||
// for (int i = 0; i < overallTotalParticles; ++i)
|
||||
// {
|
||||
// fprintf(fp, "1\n");
|
||||
// }
|
||||
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
++ts;
|
||||
}
|
||||
|
||||
void printParticleTracerParameters(ParticleTracer* particletracer)
|
||||
{
|
||||
printf("Particle Tracing data:\n");
|
||||
printf("Rank : %d\n", particletracer->rank);
|
||||
printf("\tNumber of particles : %d being injected for every period of %.2f\n",
|
||||
particletracer->numberOfParticles,
|
||||
particletracer->injectTimePeriod);
|
||||
printf("\tstartTime : %.2f\n", particletracer->startTime);
|
||||
printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : "
|
||||
"%.2f, x2 : %.2f, y2 : %.2f\n",
|
||||
particletracer->x1,
|
||||
particletracer->y1,
|
||||
particletracer->x2,
|
||||
particletracer->y2);
|
||||
printf("\tPointer : %d, TotalParticles : %d\n",
|
||||
particletracer->pointer,
|
||||
particletracer->totalParticles);
|
||||
printf("\tdt : %.2f, dx : %.2f, dy : %.2f\n",
|
||||
particletracer->dt,
|
||||
particletracer->dx,
|
||||
particletracer->dy);
|
||||
printf("\txOffset : %.2f, yOffset : %.2f\n",
|
||||
particletracer->xOffset,
|
||||
particletracer->yOffset);
|
||||
printf("\txOffsetEnd : %.2f, yOffsetEnd : %.2f\n",
|
||||
particletracer->xOffsetEnd,
|
||||
particletracer->yOffsetEnd);
|
||||
printf("\txLocal : %.2f, yLocal : %.2f\n",
|
||||
particletracer->xLocal,
|
||||
particletracer->yLocal);
|
||||
}
|
||||
|
||||
void trace(ParticleTracer* particletracer, Discretization* d, double time)
|
||||
{
|
||||
if (time >= particletracer->startTime) {
|
||||
if ((time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod) {
|
||||
injectParticles(particletracer, d->grid.s);
|
||||
particletracer->lastInjectTime = time;
|
||||
}
|
||||
if ((time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod) {
|
||||
writeParticles(particletracer, &d->comm);
|
||||
particletracer->lastWriteTime = time;
|
||||
}
|
||||
advanceParticles(particletracer, d->u, d->v, d->grid.s, &d->comm, time);
|
||||
|
||||
if (particletracer->removedParticles > (particletracer->totalParticles * 0.2)) {
|
||||
compress(particletracer);
|
||||
}
|
||||
|
||||
particletracer->lastUpdateTime = time;
|
||||
}
|
||||
}
|
||||
|
||||
void compress(ParticleTracer* particletracer)
|
||||
{
|
||||
Particle* memPool = particletracer->particlePool;
|
||||
Particle tempPool[particletracer->totalParticles];
|
||||
int totalParticles = 0;
|
||||
|
||||
// printf("\nPerforming compression ...");
|
||||
|
||||
for (int i = 0; i < particletracer->totalParticles; ++i) {
|
||||
if (memPool[i].flag == true) {
|
||||
tempPool[totalParticles].x = memPool[i].x;
|
||||
tempPool[totalParticles].y = memPool[i].y;
|
||||
tempPool[totalParticles].flag = memPool[i].flag;
|
||||
++totalParticles;
|
||||
}
|
||||
}
|
||||
|
||||
// printf(" remove %d particles\n", particletracer->totalParticles - totalParticles);
|
||||
|
||||
particletracer->totalParticles = totalParticles;
|
||||
particletracer->removedParticles = 0;
|
||||
particletracer->pointer = totalParticles;
|
||||
|
||||
memcpy(particletracer->particlePool, tempPool, totalParticles * sizeof(Particle));
|
||||
}
|
||||
|
||||
void particleRandomizer(ParticleTracer* particletracer)
|
||||
{
|
||||
memset(particletracer->linSpaceLine,
|
||||
0,
|
||||
sizeof(Particle) * particletracer->numberOfParticles);
|
||||
|
||||
for (int i = 0; i < particletracer->numberOfParticles; ++i) {
|
||||
|
||||
particletracer->linSpaceLine[i].x = (((double)rand() / RAND_MAX) *
|
||||
(particletracer->x2 -
|
||||
particletracer->x1)) +
|
||||
particletracer->x1;
|
||||
particletracer->linSpaceLine[i].y = (((double)rand() / RAND_MAX) *
|
||||
(particletracer->y2 -
|
||||
particletracer->y1)) +
|
||||
particletracer->y1;
|
||||
particletracer->linSpaceLine[i].flag = true;
|
||||
}
|
||||
}
|
65
EnhancedSolver/2D-mpi/src/particletracing.h
Normal file
65
EnhancedSolver/2D-mpi/src/particletracing.h
Normal file
@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#ifndef __PARTICLETRACING_H_
|
||||
#define __PARTICLETRACING_H_
|
||||
#include "allocate.h"
|
||||
#include "parameter.h"
|
||||
|
||||
#include "solver.h"
|
||||
#include <mpi.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#define NDIMS 2
|
||||
|
||||
typedef enum COORD { X = 0, Y, NCOORD } COORD;
|
||||
|
||||
typedef struct {
|
||||
double x, y;
|
||||
bool flag;
|
||||
} Particle;
|
||||
|
||||
typedef struct {
|
||||
int numberOfParticles, removedParticles, totalParticles;
|
||||
double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime,
|
||||
lastWriteTime;
|
||||
|
||||
int estimatedNumParticles;
|
||||
|
||||
double dx, dy, dt;
|
||||
Particle* linSpaceLine;
|
||||
Particle* particlePool;
|
||||
|
||||
int pointer;
|
||||
|
||||
double imax, jmax, xlength, ylength, imaxLocal, jmaxLocal;
|
||||
|
||||
double x1, y1, x2, y2;
|
||||
|
||||
int size, rank;
|
||||
|
||||
#ifdef _MPI
|
||||
MPI_Datatype mpi_particle;
|
||||
#endif
|
||||
|
||||
double xLocal, yLocal, xOffset, yOffset, xOffsetEnd, yOffsetEnd;
|
||||
|
||||
double* offset;
|
||||
|
||||
} ParticleTracer;
|
||||
|
||||
extern void initParticleTracer(ParticleTracer*, Discretization*, Parameter*);
|
||||
extern void injectParticles(ParticleTracer*, double*);
|
||||
extern void advanceParticles(ParticleTracer*, double*, double*, double*, Comm*, double);
|
||||
extern void freeParticles(ParticleTracer*);
|
||||
extern void writeParticles(ParticleTracer*, Comm*);
|
||||
extern void printParticleTracerParameters(ParticleTracer*);
|
||||
extern void printParticles(ParticleTracer*);
|
||||
extern void trace(ParticleTracer*, Discretization*, double);
|
||||
extern void compress(ParticleTracer*);
|
||||
extern void particleRandomizer(ParticleTracer*);
|
||||
|
||||
#endif
|
70
EnhancedSolver/2D-mpi/src/progress.c
Normal file
70
EnhancedSolver/2D-mpi/src/progress.c
Normal file
@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||
* All rights reserved. This file is part of nusif-solver.
|
||||
* Use of this source code is governed by a MIT style
|
||||
* license that can be found in the LICENSE file.
|
||||
*/
|
||||
#include <math.h>
|
||||
#include <mpi.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "progress.h"
|
||||
|
||||
static double _end;
|
||||
static int _current;
|
||||
|
||||
void initProgress(double end)
|
||||
{
|
||||
_end = end;
|
||||
_current = 0;
|
||||
|
||||
printf("[ ]");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
void printProgress(double current)
|
||||
{
|
||||
int new = (int)rint((current / _end) * 10.0);
|
||||
|
||||
if (new > _current) {
|
||||
char progress[11];
|
||||
_current = new;
|
||||
progress[0] = 0;
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (i < _current) {
|
||||
sprintf(progress + strlen(progress), "#");
|
||||
} else {
|
||||
sprintf(progress + strlen(progress), " ");
|
||||
}
|
||||
}
|
||||
printf("\r[%s]", progress);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
void stopProgress()
|
||||
{
|
||||
printf("\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
FILE* initResidualWriter()
|
||||
{
|
||||
FILE* fp;
|
||||
fp = fopen("residual.dat", "w");
|
||||
|
||||
if (fp == NULL) {
|
||||
printf("Error!\n");
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
return fp;
|
||||
|
||||
}
|
||||
|
||||
void writeResidual(FILE* fp, double ts, double res)
|
||||
{
|
||||
fprintf(fp, "%f, %f\n", ts, res);
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user