moebiusband/NuSiF-Solver
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP: Pull Request for a complete Solver package #2

Closed
AdityaUjeniya wants to merge 51 commits from (deleted):main into main
pull from: (deleted):main
merge into: moebiusband:main
Conversation 0 Commits 51 Files Changed 331 +1011 -370
30 changed files with 1011 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit f48927ce6f - Show all commits

1
BasicSolver/3D-mpi/Makefile
View File

@ -22,6 +22,7 @@ SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)
ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC))
OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC))
OBJ += $(BUILD_DIR)/vtkWriter-$(VTK_OUTPUT_FMT).o OBJ += $(BUILD_DIR)/vtkWriter-$(VTK_OUTPUT_FMT).o
OBJ += $(BUILD_DIR)/solver-$(SOLVER).o
SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h)
ifeq ($(VTK_OUTPUT_FMT),mpi) ifeq ($(VTK_OUTPUT_FMT),mpi)
DEFINES += -D_VTK_WRITER_MPI DEFINES += -D_VTK_WRITER_MPI

11
BasicSolver/3D-mpi/canal.par
View File

@ -38,15 +38,22 @@ kmax 50 # number of interior cells in z-direction
# Time Data: # Time Data:
# --------- # ---------
te 100.0 # final time te 60.0 # final time
dt 0.02 # time stepsize dt 0.02 # time stepsize
tau 0.5 # safety factor for time stepsize control (<0 constant delt) tau 0.5 # safety factor for time stepsize control (<0 constant delt)
# Multigrid data:
# ---------
levels 3 # Multigrid levels
presmooth 5 # Pre-smoothning iterations
postsmooth 5 # Post-smoothning iterations
# Pressure Iteration Data: # Pressure Iteration Data:
# ----------------------- # -----------------------
itermax 500 # maximal number of pressure iteration in one time step itermax 500 # maximal number of pressure iteration in one time step
eps 0.0001 # stopping tolerance for pressure iteration eps 0.0001 # stopping tolerance for pressure iteration
omg 1.3 # relaxation parameter for SOR iteration omg 1.7 # relaxation parameter for SOR iteration
gamma 0.9 # upwind differencing factor gamma gamma 0.9 # upwind differencing factor gamma
#=============================================================================== #===============================================================================

4
BasicSolver/3D-mpi/config.mk
View File

@ -1,7 +1,9 @@
# Supported: GCC, CLANG, ICC # Supported: GCC, CLANG, ICC
TAG ?= CLANG TAG ?= ICC
ENABLE_MPI ?= true ENABLE_MPI ?= true
ENABLE_OPENMP ?= false ENABLE_OPENMP ?= false
# Supported: rb, mg
SOLVER ?= mg
# Supported: seq, mpi # Supported: seq, mpi
VTK_OUTPUT_FMT ?= seq VTK_OUTPUT_FMT ?= seq

7
BasicSolver/3D-mpi/dcavity.par
View File

@ -42,6 +42,13 @@ te 10.0 # final time
dt 0.02 # time stepsize dt 0.02 # time stepsize
tau 0.5 # safety factor for time stepsize control (<0 constant delt) tau 0.5 # safety factor for time stepsize control (<0 constant delt)
# Multigrid data:
# ---------
levels 3 # Multigrid levels
presmooth 10 # Pre-smoothning iterations
postsmooth 5 # Post-smoothning iterations
# Pressure Iteration Data: # Pressure Iteration Data:
# ----------------------- # -----------------------

48
BasicSolver/3D-mpi/src/comm.c
View File

@ -336,7 +336,6 @@ void commCollectResult(Comm* c,
int imaxLocal = c->imaxLocal; int imaxLocal = c->imaxLocal;
int jmaxLocal = c->jmaxLocal; int jmaxLocal = c->jmaxLocal;
int kmaxLocal = c->kmaxLocal; int kmaxLocal = c->kmaxLocal;
#if defined(_MPI) #if defined(_MPI)
int offset[c->size * NDIMS]; int offset[c->size * NDIMS];
int imaxLocalAll[c->size]; int imaxLocalAll[c->size];
@ -562,9 +561,9 @@ void commPartition(Comm* c, int kmax, int jmax, int imax)
MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]); MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]);
MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords); MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords);
c->imaxLocal = sizeOfRank(c->rank, dims[ICORD], imax); c->imaxLocal = sizeOfRank(c->coords[IDIM], dims[ICORD], imax);
c->jmaxLocal = sizeOfRank(c->rank, dims[JCORD], jmax); c->jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JCORD], jmax);
c->kmaxLocal = sizeOfRank(c->rank, dims[KCORD], kmax); c->kmaxLocal = sizeOfRank(c->coords[KDIM], dims[KCORD], kmax);
// setup buffer types for communication // setup buffer types for communication
setupCommunication(c, LEFT, BULK); setupCommunication(c, LEFT, BULK);
@ -597,3 +596,44 @@ void commFinalize(Comm* c)
MPI_Finalize(); MPI_Finalize();
#endif #endif
} }
void commUpdateDatatypes(
Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal)
{
#if defined _MPI
int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
if (result == MPI_ERR_COMM) {
printf("\nNull communicator. Duplication failed !!\n");
}
newcomm->imaxLocal = imaxLocal;
newcomm->jmaxLocal = jmaxLocal;
newcomm->kmaxLocal = kmaxLocal;
setupCommunication(newcomm, LEFT, BULK);
setupCommunication(newcomm, LEFT, HALO);
setupCommunication(newcomm, RIGHT, BULK);
setupCommunication(newcomm, RIGHT, HALO);
setupCommunication(newcomm, BOTTOM, BULK);
setupCommunication(newcomm, BOTTOM, HALO);
setupCommunication(newcomm, TOP, BULK);
setupCommunication(newcomm, TOP, HALO);
setupCommunication(newcomm, FRONT, BULK);
setupCommunication(newcomm, FRONT, HALO);
setupCommunication(newcomm, BACK, BULK);
setupCommunication(newcomm, BACK, HALO);
#else
newcomm->imaxLocal = imaxLocal;
newcomm->jmaxLocal = jmaxLocal;
newcomm->kmaxLocal = kmaxLocal;
#endif
}
void commFreeCommunicator(Comm* comm)
{
#ifdef _MPI
MPI_Comm_free(&comm->comm);
#endif
}

4
BasicSolver/3D-mpi/src/comm.h
View File

@ -23,6 +23,7 @@ typedef enum dimension { KDIM = 0, JDIM, IDIM, NDIMS } Dimension;
enum layer { HALO = 0, BULK }; enum layer { HALO = 0, BULK };
enum op { MAX = 0, SUM }; enum op { MAX = 0, SUM };
typedef struct { typedef struct {
int rank; int rank;
int size; int size;
@ -45,6 +46,9 @@ extern void commShift(Comm* c, double* f, double* g, double* h);
extern void commReduction(double* v, int op); extern void commReduction(double* v, int op);
extern int commIsBoundary(Comm* c, Direction direction); extern int commIsBoundary(Comm* c, Direction direction);
extern void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax); extern void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax);
extern void commFreeCommunicator(Comm* comm);
extern void commUpdateDatatypes(
Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal);
extern void commCollectResult(Comm* c, extern void commCollectResult(Comm* c,
double* ug, double* ug,
double* vg, double* vg,

332
BasicSolver/3D-mpi/src/solver.c → BasicSolver/3D-mpi/src/discretization.c
View File

@ -7,13 +7,11 @@
#include <float.h> #include <float.h>
#include <math.h> #include <math.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h>
#include <string.h> #include <string.h>
#include "allocate.h" #include "allocate.h"
#include "comm.h" #include "discretization.h"
#include "parameter.h" #include "parameter.h"
#include "solver.h"
#include "util.h" #include "util.h"
#define P(i, j, k) \ #define P(i, j, k) \
@ -33,7 +31,7 @@
#define RHS(i, j, k) \ #define RHS(i, j, k) \
rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
static void printConfig(Solver* s) static void printConfig(Discretization* s)
{ {
if (commIsMaster(&s->comm)) { if (commIsMaster(&s->comm)) {
printf("Parameters for #%s#\n", s->problem); printf("Parameters for #%s#\n", s->problem);
@ -72,7 +70,7 @@ static void printConfig(Solver* s)
commPrintConfig(&s->comm); commPrintConfig(&s->comm);
} }
void initSolver(Solver* s, Parameter* params) void initDiscretization(Discretization* s, Parameter* params)
{ {
s->problem = params->name; s->problem = params->name;
s->bcLeft = params->bcLeft; s->bcLeft = params->bcLeft;
@ -142,226 +140,7 @@ void initSolver(Solver* s, Parameter* params)
#endif /* VERBOSE */ #endif /* VERBOSE */
} }
void computeRHS(Solver* s) void setBoundaryConditions(Discretization* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
int kmaxLocal = s->comm.kmaxLocal;
double idx = 1.0 / s->grid.dx;
double idy = 1.0 / s->grid.dy;
double idz = 1.0 / s->grid.dz;
double idt = 1.0 / s->dt;
double* rhs = s->rhs;
double* f = s->f;
double* g = s->g;
double* h = s->h;
commShift(&s->comm, f, g, h);
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx +
(G(i, j, k) - G(i, j - 1, k)) * idy +
(H(i, j, k) - H(i, j, k - 1)) * idz) *
idt;
}
}
}
}
void solve(Solver* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
int kmaxLocal = s->comm.kmaxLocal;
int imax = s->grid.imax;
int jmax = s->grid.jmax;
int kmax = s->grid.kmax;
double eps = s->eps;
int itermax = s->itermax;
double dx2 = s->grid.dx * s->grid.dx;
double dy2 = s->grid.dy * s->grid.dy;
double dz2 = s->grid.dz * s->grid.dz;
double idx2 = 1.0 / dx2;
double idy2 = 1.0 / dy2;
double idz2 = 1.0 / dz2;
double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
(dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
double* p = s->p;
double* rhs = s->rhs;
double epssq = eps * eps;
int it = 0;
double res = 1.0;
int pass, ksw, jsw, isw;
while ((res >= epssq) && (it < itermax)) {
ksw = 1;
for (pass = 0; pass < 2; pass++) {
jsw = ksw;
commExchange(&s->comm, p);
for (int k = 1; k < kmaxLocal + 1; k++) {
isw = jsw;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = isw; i < imaxLocal + 1; i += 2) {
double r =
RHS(i, j, k) -
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
idy2 +
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
idz2);
P(i, j, k) -= (factor * r);
res += (r * r);
}
isw = 3 - isw;
}
jsw = 3 - jsw;
}
ksw = 3 - ksw;
}
if (commIsBoundary(&s->comm, FRONT)) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, j, 0) = P(i, j, 1);
}
}
}
if (commIsBoundary(&s->comm, BACK)) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal);
}
}
}
if (commIsBoundary(&s->comm, BOTTOM)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, 0, k) = P(i, 1, k);
}
}
}
if (commIsBoundary(&s->comm, TOP)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k);
}
}
}
if (commIsBoundary(&s->comm, LEFT)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
P(0, j, k) = P(1, j, k);
}
}
}
if (commIsBoundary(&s->comm, RIGHT)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
P(imaxLocal + 1, j, k) = P(imaxLocal, j, k);
}
}
}
commReduction(&res, SUM);
res = res / (double)(imax * jmax * kmax);
#ifdef DEBUG
if (commIsMaster(&s->comm)) {
printf("%d Residuum: %e\n", it, res);
}
#endif
commExchange(&s->comm, p);
it++;
}
#ifdef VERBOSE
if (commIsMaster(&s->comm)) {
printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
}
#endif
}
static double maxElement(Solver* s, double* m)
{
int size = (s->comm.imaxLocal + 2) * (s->comm.jmaxLocal + 2) *
(s->comm.kmaxLocal + 2);
double maxval = DBL_MIN;
for (int i = 0; i < size; i++) {
maxval = MAX(maxval, fabs(m[i]));
}
commReduction(&maxval, MAX);
return maxval;
}
void normalizePressure(Solver* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
int kmaxLocal = s->comm.kmaxLocal;
double* p = s->p;
double avgP = 0.0;
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
avgP += P(i, j, k);
}
}
}
commReduction(&avgP, SUM);
avgP /= (s->grid.imax * s->grid.jmax * s->grid.kmax);
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, j, k) = P(i, j, k) - avgP;
}
}
}
}
void computeTimestep(Solver* s)
{
double dt = s->dtBound;
double dx = s->grid.dx;
double dy = s->grid.dy;
double dz = s->grid.dz;
double umax = maxElement(s, s->u);
double vmax = maxElement(s, s->v);
double wmax = maxElement(s, s->w);
if (umax > 0) {
dt = (dt > dx / umax) ? dx / umax : dt;
}
if (vmax > 0) {
dt = (dt > dy / vmax) ? dy / vmax : dt;
}
if (wmax > 0) {
dt = (dt > dz / wmax) ? dz / wmax : dt;
}
s->dt = dt * s->tau;
}
void setBoundaryConditions(Solver* s)
{ {
int imaxLocal = s->comm.imaxLocal; int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal; int jmaxLocal = s->comm.jmaxLocal;
@ -576,7 +355,37 @@ void setBoundaryConditions(Solver* s)
} }
} }
void setSpecialBoundaryCondition(Solver* s) void computeRHS(Discretization* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
int kmaxLocal = s->comm.kmaxLocal;
double idx = 1.0 / s->grid.dx;
double idy = 1.0 / s->grid.dy;
double idz = 1.0 / s->grid.dz;
double idt = 1.0 / s->dt;
double* rhs = s->rhs;
double* f = s->f;
double* g = s->g;
double* h = s->h;
commShift(&s->comm, f, g, h);
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx +
(G(i, j, k) - G(i, j - 1, k)) * idy +
(H(i, j, k) - H(i, j, k - 1)) * idz) *
idt;
}
}
}
}
void setSpecialBoundaryCondition(Discretization* s)
{ {
int imaxLocal = s->comm.imaxLocal; int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal; int jmaxLocal = s->comm.jmaxLocal;
@ -603,7 +412,74 @@ void setSpecialBoundaryCondition(Solver* s)
} }
} }
void computeFG(Solver* s) static double maxElement(Discretization* s, double* m)
{
int size = (s->comm.imaxLocal + 2) * (s->comm.jmaxLocal + 2) *
(s->comm.kmaxLocal + 2);
double maxval = DBL_MIN;
for (int i = 0; i < size; i++) {
maxval = MAX(maxval, fabs(m[i]));
}
commReduction(&maxval, MAX);
return maxval;
}
void normalizePressure(Discretization* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
int kmaxLocal = s->comm.kmaxLocal;
double* p = s->p;
double avgP = 0.0;
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
avgP += P(i, j, k);
}
}
}
commReduction(&avgP, SUM);
avgP /= (s->grid.imax * s->grid.jmax * s->grid.kmax);
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, j, k) = P(i, j, k) - avgP;
}
}
}
}
void computeTimestep(Discretization* s)
{
double dt = s->dtBound;
double dx = s->grid.dx;
double dy = s->grid.dy;
double dz = s->grid.dz;
double umax = maxElement(s, s->u);
double vmax = maxElement(s, s->v);
double wmax = maxElement(s, s->w);
if (umax > 0) {
dt = (dt > dx / umax) ? dx / umax : dt;
}
if (vmax > 0) {
dt = (dt > dy / vmax) ? dy / vmax : dt;
}
if (wmax > 0) {
dt = (dt > dz / wmax) ? dz / wmax : dt;
}
s->dt = dt * s->tau;
}
void computeFG(Discretization* s)
{ {
int imaxLocal = s->comm.imaxLocal; int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal; int jmaxLocal = s->comm.jmaxLocal;
@ -823,7 +699,7 @@ void computeFG(Solver* s)
} }
} }
void adaptUV(Solver* s) void adaptUV(Discretization* s)
{ {
int imaxLocal = s->comm.imaxLocal; int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal; int jmaxLocal = s->comm.jmaxLocal;

44
BasicSolver/3D-mpi/src/discretization.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#ifndef __DISCRETIZATION_H_
#define __DISCRETIZATION_H_
#include "grid.h"
#include "parameter.h"
#include "comm.h"
enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
typedef struct {
/* geometry and grid information */
Grid grid;
/* arrays */
double *p, *rhs;
double *f, *g, *h;
double *u, *v, *w;
/* parameters */
double eps, omega;
double re, tau, gamma;
double gx, gy, gz;
/* time stepping */
int itermax;
double dt, te;
double dtBound;
char* problem;
int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
Comm comm;
} Discretization;
extern void initDiscretization(Discretization*, Parameter*);
extern void computeRHS(Discretization*);
extern void normalizePressure(Discretization*);
extern void computeTimestep(Discretization*);
extern void setBoundaryConditions(Discretization*);
extern void setSpecialBoundaryCondition(Discretization*);
extern void computeFG(Discretization*);
extern void adaptUV(Discretization*);
#endif

77
BasicSolver/3D-mpi/src/main.c
View File

@ -9,7 +9,7 @@
#include <unistd.h> #include <unistd.h>
#include "allocate.h" #include "allocate.h"
#include "comm.h" #include "discretization.h"
#include "parameter.h" #include "parameter.h"
#include "progress.h" #include "progress.h"
#include "solver.h" #include "solver.h"
@ -21,8 +21,9 @@ int main(int argc, char** argv)
double timeStart, timeStop; double timeStart, timeStop;
Parameter p; Parameter p;
Solver s; Solver s;
Discretization d;
commInit(&s.comm, argc, argv); commInit(&d.comm, argc, argv);
initParameter(&p); initParameter(&p);
if (argc != 2) { if (argc != 2) {
@ -31,33 +32,40 @@ int main(int argc, char** argv)
} }
readParameter(&p, argv[1]); readParameter(&p, argv[1]);
commPartition(&s.comm, p.kmax, p.jmax, p.imax); commPartition(&d.comm, p.kmax, p.jmax, p.imax);
if (commIsMaster(&s.comm)) {
if (commIsMaster(&d.comm)) {
printParameter(&p); printParameter(&p);
} }
initSolver(&s, &p); initDiscretization(&d, &p);
initSolver(&s, &d, &p);
#ifndef VERBOSE #ifndef VERBOSE
initProgress(s.te); initProgress(d.te);
#endif #endif
double tau = s.tau; double tau = d.tau;
double te = s.te; double te = d.te;
double t = 0.0; double t = 0.0;
int nt = 0;
timeStart = getTimeStamp(); timeStart = getTimeStamp();
while (t <= te) { while (t <= te) {
if (tau > 0.0) computeTimestep(&s); if (tau > 0.0) computeTimestep(&d);
setBoundaryConditions(&s); setBoundaryConditions(&d);
setSpecialBoundaryCondition(&s); setSpecialBoundaryCondition(&d);
computeFG(&s); computeFG(&d);
computeRHS(&s); computeRHS(&d);
solve(&s); if (nt % 100 == 0) normalizePressure(&d);
adaptUV(&s); solve(&s, d.p, d.rhs);
t += s.dt; adaptUV(&d);
t += d.dt;
nt++;
#ifdef VERBOSE #ifdef VERBOSE
if (commIsMaster(&s.comm)) { if (commIsMaster(s.comm)) {
printf("TIME %f , TIMESTEP %f\n", t, s.dt); printf("TIME %f , TIMESTEP %f\n", t, d.dt);
} }
#else #else
printProgress(t); printProgress(t);
@ -67,7 +75,7 @@ int main(int argc, char** argv)
#ifndef VERBOSE #ifndef VERBOSE
stopProgress(); stopProgress();
#endif #endif
if (commIsMaster(&s.comm)) { if (commIsMaster(s.comm)) {
printf("Solution took %.2fs\n", timeStop - timeStart); printf("Solution took %.2fs\n", timeStop - timeStart);
} }
@ -75,14 +83,14 @@ int main(int argc, char** argv)
#ifdef _VTK_WRITER_MPI #ifdef _VTK_WRITER_MPI
VtkOptions opts = { .grid = s.grid, .comm = s.comm }; VtkOptions opts = { .grid = s.grid, .comm = s.comm };
vtkOpen(&opts, s.problem); vtkOpen(&opts, s.problem);
vtkScalar(&opts, "pressure", s.p); vtkScalar(&opts, "pressure", d.p);
vtkVector(&opts, "velocity", (VtkVector) { s.u, s.v, s.w }); vtkVector(&opts, "velocity", (VtkVector) { d.u, d.v, d.w });
vtkClose(&opts); vtkClose(&opts);
#else #else
double *pg, *ug, *vg, *wg; double *pg, *ug, *vg, *wg;
if (commIsMaster(&s.comm)) { if (commIsMaster(s.comm)) {
size_t bytesize = s.grid.imax * s.grid.jmax * s.grid.kmax * sizeof(double); size_t bytesize = s.grid->imax * s.grid->jmax * s.grid->kmax * sizeof(double);
pg = allocate(64, bytesize); pg = allocate(64, bytesize);
ug = allocate(64, bytesize); ug = allocate(64, bytesize);
@ -90,34 +98,35 @@ int main(int argc, char** argv)
wg = allocate(64, bytesize); wg = allocate(64, bytesize);
} }
commCollectResult(&s.comm, commCollectResult(s.comm,
ug, ug,
vg, vg,
wg, wg,
pg, pg,
s.u, d.u,
s.v, d.v,
s.w, d.w,
s.p, d.p,
s.grid.kmax, s.grid->kmax,
s.grid.jmax, s.grid->jmax,
s.grid.imax); s.grid->imax);
if (commIsMaster(&s.comm)) { if (commIsMaster(s.comm)) {
VtkOptions opts = { .grid = s.grid }; VtkOptions opts = { .grid = s.grid };
vtkOpen(&opts, s.problem); vtkOpen(&opts, s.problem);
vtkScalar(&opts, "pressure", pg); vtkScalar(&opts, "pressure", pg);
vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg });
vtkClose(&opts); vtkClose(&opts);
} }
#endif #endif
timeStop = getTimeStamp(); timeStop = getTimeStamp();
if (commIsMaster(&s.comm)) { if (commIsMaster(s.comm)) {
printf("Result output took %.2fs\n", timeStop - timeStart); printf("Result output took %.2fs\n", timeStop - timeStart);
} }
commFinalize(&s.comm); commFinalize(s.comm);
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }

4
BasicSolver/3D-mpi/src/parameter.c
View File

@ -26,6 +26,10 @@ void initParameter(Parameter* param)
param->re = 100.0; param->re = 100.0;
param->gamma = 0.9; param->gamma = 0.9;
param->tau = 0.5; param->tau = 0.5;
param->levels = 5;
param->presmooth = 5;
param->postsmooth = 5;
} }
void readParameter(Parameter* param, const char* filename) void readParameter(Parameter* param, const char* filename)

1
BasicSolver/3D-mpi/src/parameter.h
View File

@ -18,6 +18,7 @@ typedef struct {
char* name; char* name;
int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
double u_init, v_init, w_init, p_init; double u_init, v_init, w_init, p_init;
int levels, presmooth, postsmooth;
} Parameter; } Parameter;
void initParameter(Parameter*); void initParameter(Parameter*);

401
BasicSolver/3D-mpi/src/solver-mg.c Normal file
View File

@ -0,0 +1,401 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <stdio.h>
#include <stdlib.h>
#include "allocate.h"
#include "solver.h"
#include "util.h"
#define FINEST_LEVEL 0
#define COARSEST_LEVEL (s->levels - 1)
#define S(i, j, k) \
s[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define E(i, j, k) \
e[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define R(i, j, k) \
r[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define OLD(i, j, k) \
old[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
static void restrictMG(Solver* s, int level, Comm* comm)
{
double* r = s->r[level + 1];
double* old = s->r[level];
int imaxLocal = comm->imaxLocal;
int jmaxLocal = comm->jmaxLocal;
int kmaxLocal = comm->kmaxLocal;
commExchange(comm, old);
for (int k = 1; k < comm->kmaxLocal + 1; k++) {
for (int j = 1; j < comm->jmaxLocal + 1; j++) {
for (int i = 1; i < comm->imaxLocal + 1; ++i) {
R(i, j, k) = (OLD(2 * i - 1, 2 * j - 1, 2 * k) +
OLD(2 * i, 2 * j - 1, 2 * k) * 2 +
OLD(2 * i + 1, 2 * j - 1, 2 * k) +
OLD(2 * i - 1, 2 * j, 2 * k) * 2 +
OLD(2 * i, 2 * j, 2 * k) * 8 +
OLD(2 * i + 1, 2 * j, 2 * k) * 2 +
OLD(2 * i - 1, 2 * j + 1, 2 * k) +
OLD(2 * i, 2 * j + 1, 2 * k) * 2 +
OLD(2 * i + 1, 2 * j + 1, 2 * k) +
OLD(2 * i - 1, 2 * j - 1, 2 * k - 1) +
OLD(2 * i, 2 * j - 1, 2 * k - 1) * 2 +
OLD(2 * i + 1, 2 * j - 1, 2 * k - 1) +
OLD(2 * i - 1, 2 * j, 2 * k - 1) * 2 +
OLD(2 * i, 2 * j, 2 * k - 1) * 4 +
OLD(2 * i + 1, 2 * j, 2 * k - 1) * 2 +
OLD(2 * i - 1, 2 * j + 1, 2 * k - 1) +
OLD(2 * i, 2 * j + 1, 2 * k - 1) * 2 +
OLD(2 * i + 1, 2 * j + 1, 2 * k - 1) +
OLD(2 * i - 1, 2 * j - 1, 2 * k + 1) +
OLD(2 * i, 2 * j - 1, 2 * k + 1) * 2 +
OLD(2 * i + 1, 2 * j - 1, 2 * k + 1) +
OLD(2 * i - 1, 2 * j, 2 * k + 1) * 2 +
OLD(2 * i, 2 * j, 2 * k + 1) * 4 +
OLD(2 * i + 1, 2 * j, 2 * k + 1) * 2 +
OLD(2 * i - 1, 2 * j + 1, 2 * k + 1) +
OLD(2 * i, 2 * j + 1, 2 * k + 1) * 2 +
OLD(2 * i + 1, 2 * j + 1, 2 * k + 1)) /
64.0;
}
}
}
}
static void prolongate(Solver* s, int level, Comm* comm)
{
double* old = s->r[level + 1];
double* e = s->r[level];
int imaxLocal = comm->imaxLocal;
int jmaxLocal = comm->jmaxLocal;
int kmaxLocal = comm->kmaxLocal;
for (int k = 2; k < kmaxLocal + 1; k += 2) {
for (int j = 2; j < jmaxLocal + 1; j += 2) {
for (int i = 2; i < imaxLocal + 1; i += 2) {
E(i, j, k) = OLD(i / 2, j / 2, k / 2);
}
}
}
}
static void correct(Solver* s, double* p, int level, Comm* comm)
{
double* e = s->e[level];
int imaxLocal = comm->imaxLocal;
int jmaxLocal = comm->jmaxLocal;
int kmaxLocal = comm->kmaxLocal;
for (int k = 1; k < kmaxLocal + 1; ++k) {
for (int j = 1; j < jmaxLocal + 1; ++j) {
for (int i = 1; i < imaxLocal + 1; ++i) {
P(i, j, k) += E(i, j, k);
}
}
}
}
static void setBoundaryCondition(
Solver* s, double* p, int imaxLocal, int jmaxLocal, int kmaxLocal)
{
#ifdef _MPI
if (commIsBoundary(s->comm, FRONT)) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, j, 0) = P(i, j, 1);
}
}
}
if (commIsBoundary(s->comm, BACK)) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal);
}
}
}
if (commIsBoundary(s->comm, BOTTOM)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, 0, k) = P(i, 1, k);
}
}
}
if (commIsBoundary(s->comm, TOP)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k);
}
}
}
if (commIsBoundary(s->comm, LEFT)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
P(0, j, k) = P(1, j, k);
}
}
}
if (commIsBoundary(s->comm, RIGHT)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
P(imaxLocal + 1, j, k) = P(imaxLocal, j, k);
}
}
}
#else
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, j, 0) = P(i, j, 1);
P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal);
}
}
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, 0, k) = P(i, 1, k);
P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k);
}
}
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
P(0, j, k) = P(1, j, k);
P(imaxLocal + 1, j, k) = P(imaxLocal, j, k);
}
}
#endif
}
static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm)
{
int imaxLocal = comm->imaxLocal;
int jmaxLocal = comm->jmaxLocal;
int kmaxLocal = comm->kmaxLocal;
double eps = s->eps;
int itermax = s->itermax;
double dx2 = s->grid->dx * s->grid->dx;
double dy2 = s->grid->dy * s->grid->dy;
double dz2 = s->grid->dz * s->grid->dz;
double idx2 = 1.0 / dx2;
double idy2 = 1.0 / dy2;
double idz2 = 1.0 / dz2;
double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
(dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
double* r = s->r[level];
double epssq = eps * eps;
int it = 0;
int pass, ksw, jsw, isw;
double res = 1.0;
ksw = 1;
for (pass = 0; pass < 2; pass++) {
jsw = ksw;
commExchange(comm, p);
for (int k = 1; k < kmaxLocal + 1; k++) {
isw = jsw;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = isw; i < imaxLocal + 1; i += 2) {
P(i, j, k) -=
factor *
(RHS(i, j, k) -
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
idy2 +
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
idz2));
}
isw = 3 - isw;
}
jsw = 3 - jsw;
}
ksw = 3 - ksw;
}
}
static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm)
{
int imaxLocal = comm->imaxLocal;
int jmaxLocal = comm->jmaxLocal;
int kmaxLocal = comm->kmaxLocal;
double eps = s->eps;
int itermax = s->itermax;
double dx2 = s->grid->dx * s->grid->dx;
double dy2 = s->grid->dy * s->grid->dy;
double dz2 = s->grid->dz * s->grid->dz;
double idx2 = 1.0 / dx2;
double idy2 = 1.0 / dy2;
double idz2 = 1.0 / dz2;
double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
(dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
double* r = s->r[level];
double epssq = eps * eps;
int it = 0;
int pass, ksw, jsw, isw;
double res = 1.0;
ksw = 1;
for (pass = 0; pass < 2; pass++) {
jsw = ksw;
commExchange(comm, p);
for (int k = 1; k < kmaxLocal + 1; k++) {
isw = jsw;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = isw; i < imaxLocal + 1; i += 2) {
R(i,
j,
k) = (RHS(i, j, k) -
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) *
idx2 +
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
idy2 +
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
idz2));
res += (R(i, j, k) * R(i, j, k));
}
isw = 3 - isw;
}
jsw = 3 - jsw;
}
ksw = 3 - ksw;
}
commReduction(&res, SUM);
res = res / (double)(imaxLocal * jmaxLocal * kmaxLocal);
#ifdef DEBUG
if (commIsMaster(s->comm)) {
printf("%d Residuum: %e\n", it, res);
}
#endif
return res;
}
static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm)
{
int imaxLocal = comm->imaxLocal;
int jmaxLocal = comm->jmaxLocal;
int kmaxLocal = comm->kmaxLocal;
double res = 0.0;
// coarsest level
if (level == COARSEST_LEVEL) {
for (int i = 0; i < 5; i++) {
smooth(s, p, rhs, level, comm);
}
return res;
}
// pre-smoothing
for (int i = 0; i < s->presmooth; i++) {
smooth(s, p, rhs, level, comm);
if (level == FINEST_LEVEL)
setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal);
}
res = calculateResidual(s, p, rhs, level, comm);
// restrict
restrictMG(s, level, comm);
Comm newcomm;
commUpdateDatatypes(s->comm,
&newcomm,
comm->imaxLocal / 2,
comm->jmaxLocal / 2,
comm->kmaxLocal / 2);
// MGSolver on residual and error.
multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm);
commFreeCommunicator(&newcomm);
// prolongate
prolongate(s, level, comm);
// correct p on finer level using residual
correct(s, p, level, comm);
if (level == FINEST_LEVEL)
setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal);
// post-smoothing
for (int i = 0; i < s->postsmooth; i++) {
smooth(s, p, rhs, level, comm);
if (level == FINEST_LEVEL)
setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal);
}
return res;
}
void initSolver(Solver* s, Discretization* d, Parameter* p)
{
s->eps = p->eps;
s->omega = p->omg;
s->itermax = p->itermax;
s->levels = p->levels;
s->grid = &d->grid;
s->presmooth = p->presmooth;
s->postsmooth = p->postsmooth;
s->comm = &d->comm;
s->problem = p->name;
int imax = s->grid->imax;
int jmax = s->grid->jmax;
int kmax = s->grid->kmax;
int levels = s->levels;
printf("Using Multigrid solver with %d levels\n", levels);
s->r = malloc(levels * sizeof(double*));
s->e = malloc(levels * sizeof(double*));
size_t size = (imax + 2) * (jmax + 2) * (kmax + 2);
for (int j = 0; j < levels; j++) {
s->r[j] = allocate(64, size * sizeof(double));
s->e[j] = allocate(64, size * sizeof(double));
for (size_t i = 0; i < size; i++) {
s->r[j][i] = 0.0;
s->e[j][i] = 0.0;
}
}
}
void solve(Solver* s, double* p, double* rhs)
{
double res = multiGrid(s, p, rhs, 0, s->comm);
#ifdef VERBOSE
if (commIsMaster(s->comm)) {
printf("Residuum: %.6f\n", res);
}
#endif
}

173
BasicSolver/3D-mpi/src/solver-rb.c Normal file
View File

@ -0,0 +1,173 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <float.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "allocate.h"
#include "comm.h"
#include "parameter.h"
#include "solver.h"
#include "util.h"
void solve(Solver* s, double* p, double* rhs)
{
int imaxLocal = s->comm->imaxLocal;
int jmaxLocal = s->comm->jmaxLocal;
int kmaxLocal = s->comm->kmaxLocal;
int imax = s->grid->imax;
int jmax = s->grid->jmax;
int kmax = s->grid->kmax;
double eps = s->eps;
int itermax = s->itermax;
double dx2 = s->grid->dx * s->grid->dx;
double dy2 = s->grid->dy * s->grid->dy;
double dz2 = s->grid->dz * s->grid->dz;
double idx2 = 1.0 / dx2;
double idy2 = 1.0 / dy2;
double idz2 = 1.0 / dz2;
double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
(dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
double epssq = eps * eps;
int it = 0;
double res = 1.0;
int pass, ksw, jsw, isw;
while ((res >= epssq) && (it < itermax)) {
ksw = 1;
for (pass = 0; pass < 2; pass++) {
jsw = ksw;
commExchange(s->comm, p);
for (int k = 1; k < kmaxLocal + 1; k++) {
isw = jsw;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = isw; i < imaxLocal + 1; i += 2) {
double r =
RHS(i, j, k) -
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
idy2 +
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
idz2);
P(i, j, k) -= (factor * r);
res += (r * r);
}
isw = 3 - isw;
}
jsw = 3 - jsw;
}
ksw = 3 - ksw;
}
#ifdef _MPI
if (commIsBoundary(s->comm, FRONT)) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, j, 0) = P(i, j, 1);
}
}
}
if (commIsBoundary(s->comm, BACK)) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal);
}
}
}
if (commIsBoundary(s->comm, BOTTOM)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, 0, k) = P(i, 1, k);
}
}
}
if (commIsBoundary(s->comm, TOP)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k);
}
}
}
if (commIsBoundary(s->comm, LEFT)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
P(0, j, k) = P(1, j, k);
}
}
}
if (commIsBoundary(s->comm, RIGHT)) {
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
P(imaxLocal + 1, j, k) = P(imaxLocal, j, k);
}
}
}
#else
for (int j = 1; j < jmax + 1; j++) {
for (int i = 1; i < imax + 1; i++) {
P(i, j, 0) = P(i, j, 1);
P(i, j, kmax + 1) = P(i, j, kmax);
}
}
for (int k = 1; k < kmax + 1; k++) {
for (int i = 1; i < imax + 1; i++) {
P(i, 0, k) = P(i, 1, k);
P(i, jmax + 1, k) = P(i, jmax, k);
}
}
for (int k = 1; k < kmax + 1; k++) {
for (int j = 1; j < jmax + 1; j++) {
P(0, j, k) = P(1, j, k);
P(imax + 1, j, k) = P(imax, j, k);
}
}
#endif
commReduction(&res, SUM);
res = res / (double)(imax * jmax * kmax);
#ifdef DEBUG
if (commIsMaster(&s->comm)) {
printf("%d Residuum: %e\n", it, res);
}
#endif
commExchange(s->comm, p);
it++;
}
#ifdef VERBOSE
if (commIsMaster(s->comm)) {
printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
}
#endif
}
void initSolver(Solver* s, Discretization* d, Parameter* p)
{
s->eps = p->eps;
s->omega = p->omg;
s->itermax = p->itermax;
s->levels = p->levels;
s->grid = &d->grid;
s->presmooth = p->presmooth;
s->postsmooth = p->postsmooth;
s->comm = &d->comm;
s->problem = p->name;
}

20
BasicSolver/3D-mpi/src/solver.h
View File

@ -9,12 +9,11 @@
#include "comm.h" #include "comm.h"
#include "grid.h" #include "grid.h"
#include "parameter.h" #include "parameter.h"
#include "discretization.h"
enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
typedef struct { typedef struct {
/* geometry and grid information */ /* geometry and grid information */
Grid grid; Grid* grid;
/* arrays */ /* arrays */
double *p, *rhs; double *p, *rhs;
double *f, *g, *h; double *f, *g, *h;
@ -30,16 +29,11 @@ typedef struct {
char* problem; char* problem;
int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
/* communication */ /* communication */
Comm comm; double **r, **e;
int levels, presmooth, postsmooth;
Comm* comm;
} Solver; } Solver;
extern void initSolver(Solver*, Parameter*); extern void solve(Solver* , double* , double* );
extern void computeRHS(Solver*); extern void initSolver(Solver*, Discretization*, Parameter*);
extern void solve(Solver*);
extern void normalizePressure(Solver*);
extern void computeTimestep(Solver*);
extern void setBoundaryConditions(Solver*);
extern void setSpecialBoundaryCondition(Solver*);
extern void computeFG(Solver*);
extern void adaptUV(Solver*);
#endif #endif

22
BasicSolver/3D-mpi/src/test.c
View File

@ -14,10 +14,10 @@
void testInit(Solver* s) void testInit(Solver* s)
{ {
int imaxLocal = s->comm.imaxLocal; int imaxLocal = s->comm->imaxLocal;
int jmaxLocal = s->comm.jmaxLocal; int jmaxLocal = s->comm->jmaxLocal;
int kmaxLocal = s->comm.kmaxLocal; int kmaxLocal = s->comm->kmaxLocal;
int myrank = s->comm.rank; int myrank = s->comm->rank;
double* p = s->p; double* p = s->p;
double* f = s->f; double* f = s->f;
double* g = s->g; double* g = s->g;
@ -76,11 +76,11 @@ static char* direction2String(Direction dir)
static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir) static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir)
{ {
int imaxLocal = s->comm.imaxLocal; int imaxLocal = s->comm->imaxLocal;
int jmaxLocal = s->comm.jmaxLocal; int jmaxLocal = s->comm->jmaxLocal;
int kmaxLocal = s->comm.kmaxLocal; int kmaxLocal = s->comm->kmaxLocal;
char filename[50]; char filename[50];
snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm.rank); snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm->rank);
FILE* fh = fopen(filename, "w"); FILE* fh = fopen(filename, "w");
for (int y = 0; y < ymax; y++) { for (int y = 0; y < ymax; y++) {
@ -116,9 +116,9 @@ static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir)
void testPrintHalo(Solver* s, double* a) void testPrintHalo(Solver* s, double* a)
{ {
int imaxLocal = s->comm.imaxLocal; int imaxLocal = s->comm->imaxLocal;
int jmaxLocal = s->comm.jmaxLocal; int jmaxLocal = s->comm->jmaxLocal;
int kmaxLocal = s->comm.kmaxLocal; int kmaxLocal = s->comm->kmaxLocal;
printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, BOTTOM); printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, BOTTOM);
printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, TOP); printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, TOP);

9
BasicSolver/3D-mpi/src/util.h
View File

@ -19,4 +19,13 @@
#define ABS(a) ((a) >= 0 ? (a) : -(a)) #define ABS(a) ((a) >= 0 ? (a) : -(a))
#endif #endif
#define P(i, j, k) p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define F(i, j, k) f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define G(i, j, k) g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define H(i, j, k) h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define U(i, j, k) u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define V(i, j, k) v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define W(i, j, k) w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define RHS(i, j, k) rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#endif // __UTIL_H_ #endif // __UTIL_H_

25
BasicSolver/3D-mpi/src/vtkWriter-mpi.c
View File

@ -46,18 +46,18 @@ static void writeHeader(VtkOptions* o)
cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n"); cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n");
cursor += sprintf(cursor, cursor += sprintf(cursor,
"DIMENSIONS %d %d %d\n", "DIMENSIONS %d %d %d\n",
o->grid.imax, o->grid->imax,
o->grid.jmax, o->grid->jmax,
o->grid.kmax); o->grid->kmax);
cursor += sprintf(cursor, cursor += sprintf(cursor,
"ORIGIN %f %f %f\n", "ORIGIN %f %f %f\n",
o->grid.dx * 0.5, o->grid->dx * 0.5,
o->grid.dy * 0.5, o->grid->dy * 0.5,
o->grid.dz * 0.5); o->grid->dz * 0.5);
cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz); cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz);
cursor += sprintf(cursor, cursor += sprintf(cursor,
"POINT_DATA %d\n", "POINT_DATA %d\n",
o->grid.imax * o->grid.jmax * o->grid.kmax); o->grid->imax * o->grid->jmax * o->grid->kmax);
if (commIsMaster(&o->comm)) { if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
@ -67,7 +67,6 @@ static void writeHeader(VtkOptions* o)
void vtkOpen(VtkOptions* o, char* problem) void vtkOpen(VtkOptions* o, char* problem)
{ {
char filename[50]; char filename[50];
snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size); snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size);
MPI_File_open(o->comm.comm, MPI_File_open(o->comm.comm,
filename, filename,
@ -98,7 +97,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
} }
int offsets[NDIMS]; int offsets[NDIMS];
commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax); commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax);
// set global view in file // set global view in file
MPI_Offset disp; MPI_Offset disp;
@ -108,7 +107,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
MPI_File_get_size(o->fh, &disp); MPI_File_get_size(o->fh, &disp);
MPI_Type_create_subarray(NDIMS, MPI_Type_create_subarray(NDIMS,
(int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax }, (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax },
(int[NDIMS]) { o->comm.kmaxLocal, o->comm.jmaxLocal, o->comm.imaxLocal }, (int[NDIMS]) { o->comm.kmaxLocal, o->comm.jmaxLocal, o->comm.imaxLocal },
offsets, offsets,
MPI_ORDER_C, MPI_ORDER_C,
@ -177,7 +176,7 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec)
} }
int offsets[NDIMS]; int offsets[NDIMS];
commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax); commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax);
// set global view in file // set global view in file
MPI_Offset disp; MPI_Offset disp;
@ -190,7 +189,7 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec)
MPI_Type_commit(&vectorType); MPI_Type_commit(&vectorType);
MPI_Type_create_subarray(NDIMS, MPI_Type_create_subarray(NDIMS,
(int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax }, (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax },
(int[NDIMS]) { kmaxLocal, jmaxLocal, imaxLocal }, (int[NDIMS]) { kmaxLocal, jmaxLocal, imaxLocal },
offsets, offsets,
MPI_ORDER_C, MPI_ORDER_C,

24
BasicSolver/3D-mpi/src/vtkWriter-seq.c
View File

@ -41,14 +41,14 @@ static void writeHeader(VtkOptions* o)
} }
fprintf(o->fh, "DATASET STRUCTURED_POINTS\n"); fprintf(o->fh, "DATASET STRUCTURED_POINTS\n");
fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid.imax, o->grid.jmax, o->grid.kmax); fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid->imax, o->grid->jmax, o->grid->kmax);
fprintf(o->fh, fprintf(o->fh,
"ORIGIN %f %f %f\n", "ORIGIN %f %f %f\n",
o->grid.dx * 0.5, o->grid->dx * 0.5,
o->grid.dy * 0.5, o->grid->dy * 0.5,
o->grid.dz * 0.5); o->grid->dz * 0.5);
fprintf(o->fh, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz); fprintf(o->fh, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz);
fprintf(o->fh, "POINT_DATA %d\n", o->grid.imax * o->grid.jmax * o->grid.kmax); fprintf(o->fh, "POINT_DATA %d\n", o->grid->imax * o->grid->jmax * o->grid->kmax);
} }
void vtkOpen(VtkOptions* o, char* problem) void vtkOpen(VtkOptions* o, char* problem)
@ -64,9 +64,9 @@ void vtkOpen(VtkOptions* o, char* problem)
static void writeScalar(VtkOptions* o, double* s) static void writeScalar(VtkOptions* o, double* s)
{ {
int imax = o->grid.imax; int imax = o->grid->imax;
int jmax = o->grid.jmax; int jmax = o->grid->jmax;
int kmax = o->grid.kmax; int kmax = o->grid->kmax;
for (int k = 0; k < kmax; k++) { for (int k = 0; k < kmax; k++) {
for (int j = 0; j < jmax; j++) { for (int j = 0; j < jmax; j++) {
@ -105,9 +105,9 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
static void writeVector(VtkOptions* o, VtkVector vec) static void writeVector(VtkOptions* o, VtkVector vec)
{ {
int imax = o->grid.imax; int imax = o->grid->imax;
int jmax = o->grid.jmax; int jmax = o->grid->jmax;
int kmax = o->grid.kmax; int kmax = o->grid->kmax;
for (int k = 0; k < kmax; k++) { for (int k = 0; k < kmax; k++) {
for (int j = 0; j < jmax; j++) { for (int j = 0; j < jmax; j++) {

2
BasicSolver/3D-mpi/src/vtkWriter.h
View File

@ -14,7 +14,7 @@
typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat; typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat;
typedef struct VtkOptions { typedef struct VtkOptions {
Grid grid; Grid* grid;
#ifdef _VTK_WRITER_MPI #ifdef _VTK_WRITER_MPI
MPI_File fh; MPI_File fh;
#else #else

7
BasicSolver/3D-seq/canal.par
View File

@ -42,6 +42,13 @@ te 100.0 # final time
dt 0.02 # time stepsize dt 0.02 # time stepsize
tau 0.5 # safety factor for time stepsize control (<0 constant delt) tau 0.5 # safety factor for time stepsize control (<0 constant delt)
# Multigrid data:
# ---------
levels 3 # Multigrid levels
presmooth 5 # Pre-smoothning iterations
postsmooth 5 # Post-smoothning iterations
# Pressure Iteration Data: # Pressure Iteration Data:
# ----------------------- # -----------------------

6
BasicSolver/3D-seq/config.mk
View File

@ -1,12 +1,12 @@
# Supported: GCC, CLANG, ICC # Supported: GCC, CLANG, ICC
TAG ?= CLANG TAG ?= ICC
ENABLE_OPENMP ?= false ENABLE_OPENMP ?= false
# Supported: sor, mg # Supported: rb, mg
SOLVER ?= mg SOLVER ?= mg
# Run in debug settings # Run in debug settings
DEBUG ?= false DEBUG ?= false
#Feature options #Feature options
OPTIONS += -DARRAY_ALIGNMENT=64 OPTIONS += -DARRAY_ALIGNMENT=64
#OPTIONS += -DVERBOSE OPTIONS += -DVERBOSE
#OPTIONS += -DDEBUG #OPTIONS += -DDEBUG

8
BasicSolver/3D-seq/dcavity.par
View File

@ -42,6 +42,13 @@ te 2.0 # final time
dt 0.02 # time stepsize dt 0.02 # time stepsize
tau 0.5 # safety factor for time stepsize control (<0 constant delt) tau 0.5 # safety factor for time stepsize control (<0 constant delt)
# Multigrid data:
# ---------
levels 3 # Multigrid levels
presmooth 10 # Pre-smoothning iterations
postsmooth 5 # Post-smoothning iterations
# Pressure Iteration Data: # Pressure Iteration Data:
# ----------------------- # -----------------------
@ -50,5 +57,4 @@ eps 0.001 # stopping tolerance for pressure iteration
rho 0.5 rho 0.5
omg 1.7 # relaxation parameter for SOR iteration omg 1.7 # relaxation parameter for SOR iteration
gamma 0.9 # upwind differencing factor gamma gamma 0.9 # upwind differencing factor gamma
levels 5 # Multigrid levels
#=============================================================================== #===============================================================================

2
BasicSolver/3D-seq/src/discretization.c
View File

@ -106,7 +106,7 @@ void initDiscretization(Discretization* d, Parameter* p)
d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; d->dtBound = 0.5 * d->re * 1.0 / invSqrSum;
#ifdef VERBOSE #ifdef VERBOSE
printConfig(s); printConfig(d);
#endif /* VERBOSE */ #endif /* VERBOSE */
} }

2
BasicSolver/3D-seq/src/main.c
View File

@ -105,7 +105,7 @@ int main(int argc, char** argv)
nt++; nt++;
#ifdef VERBOSE #ifdef VERBOSE
printf("TIME %f , TIMESTEP %f\n", t, solver.dt); printf("TIME %f , TIMESTEP %f\n", t, d.dt);
#else #else
printProgress(t); printProgress(t);
#endif #endif

2
BasicSolver/3D-seq/src/parameter.c
View File

@ -27,6 +27,8 @@ void initParameter(Parameter* param)
param->gamma = 0.9; param->gamma = 0.9;
param->tau = 0.5; param->tau = 0.5;
param->levels = 5; param->levels = 5;
param->presmooth = 5;
param->postsmooth = 5;
} }
void readParameter(Parameter* param, const char* filename) void readParameter(Parameter* param, const char* filename)

1
BasicSolver/3D-seq/src/parameter.h
View File

@ -18,6 +18,7 @@ typedef struct {
char* name; char* name;
int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
double u_init, v_init, w_init, p_init; double u_init, v_init, w_init, p_init;
int presmooth, postsmooth;
} Parameter; } Parameter;
void initParameter(Parameter*); void initParameter(Parameter*);

74
BasicSolver/3D-seq/src/solver-mg.c
View File

@ -141,13 +141,62 @@ static double smooth(
for (int j = 1; j < jmax + 1; j++) { for (int j = 1; j < jmax + 1; j++) {
for (int i = isw; i < imax + 1; i += 2) { for (int i = isw; i < imax + 1; i += 2) {
R(i, j, k) = P(i, j, k) -=
RHS(i, j, k) - factor *
(RHS(i, j, k) -
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * idy2 + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * idz2); idy2 +
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
idz2));
}
isw = 3 - isw;
}
jsw = 3 - jsw;
}
ksw = 3 - ksw;
}
}
static double calculateResidual(
Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax)
{
double eps = s->eps;
int itermax = s->itermax;
double dx2 = s->grid->dx * s->grid->dx;
double dy2 = s->grid->dy * s->grid->dy;
double dz2 = s->grid->dz * s->grid->dz;
double idx2 = 1.0 / dx2;
double idy2 = 1.0 / dy2;
double idz2 = 1.0 / dz2;
double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
(dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
double* r = s->r[level];
double epssq = eps * eps;
int it = 0;
int pass, ksw, jsw, isw;
double res = 1.0;
ksw = 1;
for (pass = 0; pass < 2; pass++) {
jsw = ksw;
for (int k = 1; k < kmax + 1; k++) {
isw = jsw;
for (int j = 1; j < jmax + 1; j++) {
for (int i = isw; i < imax + 1; i += 2) {
R(i,
j,
k) = (RHS(i, j, k) -
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) *
idx2 +
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
idy2 +
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
idz2));
P(i, j, k) -= (factor * R(i, j, k));
res += (R(i, j, k) * R(i, j, k)); res += (R(i, j, k) * R(i, j, k));
} }
isw = 3 - isw; isw = 3 - isw;
@ -167,7 +216,7 @@ static double multiGrid(
{ {
double res = 0.0; double res = 0.0;
// coarsest level TODO: Use direct solver? // coarsest level
if (level == COARSEST_LEVEL) { if (level == COARSEST_LEVEL) {
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {
smooth(s, p, rhs, level, imax, jmax, kmax); smooth(s, p, rhs, level, imax, jmax, kmax);
@ -175,17 +224,18 @@ static double multiGrid(
return res; return res;
} }
// pre-smoothing TODO: Make smoothing steps configurable? // pre-smoothing
for (int i = 0; i < 5; i++) { for (int i = 0; i < s->presmooth; i++) {
smooth(s, p, rhs, level, imax, jmax, kmax); smooth(s, p, rhs, level, imax, jmax, kmax);
if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax);
} }
res = calculateResidual(s, p, rhs, level, imax, jmax, kmax);
// restrict // restrict
restrictMG(s, level, imax, jmax, kmax); restrictMG(s, level, imax, jmax, kmax);
// MGSolver on residual and error. // MGSolver on residual and error.
// TODO: What if there is a rest?
multiGrid(s, multiGrid(s,
s->e[level + 1], s->e[level + 1],
s->r[level + 1], s->r[level + 1],
@ -202,8 +252,8 @@ static double multiGrid(
if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax);
// post-smoothing // post-smoothing
for (int i = 0; i < 5; i++) { for (int i = 0; i < s->postsmooth; i++) {
res = smooth(s, p, rhs, level, imax, jmax, kmax); smooth(s, p, rhs, level, imax, jmax, kmax);
if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax);
} }
@ -217,6 +267,8 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
s->itermax = p->itermax; s->itermax = p->itermax;
s->levels = p->levels; s->levels = p->levels;
s->grid = &d->grid; s->grid = &d->grid;
s->presmooth = p->presmooth;
s->postsmooth = p->postsmooth;
int imax = s->grid->imax; int imax = s->grid->imax;
int jmax = s->grid->jmax; int jmax = s->grid->jmax;

0
BasicSolver/3D-seq/src/solver-sor.c → BasicSolver/3D-seq/src/solver-rb.c
View File

1
BasicSolver/3D-seq/src/solver.h
View File

@ -18,6 +18,7 @@ typedef struct {
int itermax; int itermax;
int levels; int levels;
double **r, **e; double **r, **e;
int presmooth, postsmooth;
} Solver; } Solver;
extern void initSolver(Solver*, Discretization*, Parameter*); extern void initSolver(Solver*, Discretization*, Parameter*);

1
BasicSolver/3D-seq/src/vtkWriter.c
View File

@ -70,6 +70,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
fprintf(o->fh, "SCALARS %s float\n", name); fprintf(o->fh, "SCALARS %s float\n", name);
fprintf(o->fh, "LOOKUP_TABLE default\n");
for (int k = 0; k < kmax; k++) { for (int k = 0; k < kmax; k++) {
for (int j = 0; j < jmax; j++) { for (int j = 0; j < jmax; j++) {