Merge pull request 'EnhancedSolver port complete' (#3) from AdityaUjeniya/NuSiF-Solver:main into main

Reviewed-on: #3

BasicSolver/2D-mpi/Makefile

@@ -22,6 +22,7 @@ SRC       = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)
 ASM       = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC))
 OBJ       = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC))
 OBJ      += $(BUILD_DIR)/comm-$(COMM_TYPE).o
+OBJ      += $(BUILD_DIR)/solver-$(SOLVER).o
 SOURCES   = $(SRC) $(wildcard $(SRC_DIR)/*.h)
 CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES)
 
@@ -38,9 +39,20 @@ $(BUILD_DIR)/%.s:  %.c
 	$(info ===>  GENERATE ASM  $@)
 	$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
 
-.PHONY: clean distclean tags info asm format
+.PHONY: clean distclean vis vis_clean tags info asm format
 
-clean:
+vis:
+	$(info ===>  GENERATE VISUALIZATION)
+	@gnuplot -e "filename='pressure.dat'" ./surface.plot
+	@gnuplot -e "filename='velocity.dat'" ./vector.plot
+	@gnuplot -e "filename='residual.dat'" ./residual.plot
+
+vis_clean:
+	$(info ===>  CLEAN VISUALIZATION)
+	@rm -f *.dat
+	@rm -f *.png
+
+clean: vis_clean
 	$(info ===>  CLEAN)
 	@rm -rf $(BUILD_DIR)
 	@rm -f tags

BasicSolver/2D-mpi/canal.par

@@ -7,10 +7,10 @@
 
 name canal             # name of flow setup
 
-bcN     1              #  flags for boundary conditions
-bcE     3              #  1 = no-slip      3 = outflow
-bcS     1              #  2 = free-slip    4 = periodic
-bcW     3              #
+bcTop      1			#  flags for boundary conditions
+bcBottom   1			#  1 = no-slip      3 = outflow
+bcLeft     3			#  2 = free-slip    4 = periodic
+bcRight    3			#
 
 gx     0.0      # Body forces (e.g. gravity)
 gy     0.0      #
@@ -27,15 +27,22 @@ p_init        0.0      # initial value for pressure
 xlength       30.0     # domain size in x-direction
 ylength       4.0	   # domain size in y-direction
 imax          200      # number of interior cells in x-direction
-jmax          50	   # number of interior cells in y-direction
+jmax          40	   # number of interior cells in y-direction
 
 # Time Data:
 # ---------
 
-te       100.0   # final time
+te       60.0   # final time
 dt       0.02    # time stepsize
 tau      0.5     # safety factor for time stepsize control (<0 constant delt)
 
+# Multigrid data:
+# ---------
+
+levels        2         # Multigrid levels
+presmooth     5         # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
 # Pressure Iteration Data:
 # -----------------------
 

BasicSolver/2D-mpi/config.mk

@@ -1,7 +1,11 @@
-# Supported: GCC, CLANG, ICC
-TAG ?= CLANG
+# Supported: GCC, CLANG, ICX
+TAG ?= ICX
+# Supported: true, false
 ENABLE_MPI ?= true
 ENABLE_OPENMP ?= false
+# Supported: rb, mg
+SOLVER ?= mg
+# Supported: v1, v2, v3
 COMM_TYPE ?= v3
 
 #Feature options

BasicSolver/2D-mpi/dcavity.par

@@ -26,8 +26,8 @@ p_init    0.0		# initial value for pressure
 
 xlength    1.0		# domain size in x-direction
 ylength    1.0		# domain size in y-direction
-imax       80		# number of interior cells in x-direction
-jmax       80		# number of interior cells in y-direction
+imax       128		# number of interior cells in x-direction
+jmax       128		# number of interior cells in y-direction
 
 # Time Data:
 # ---------
@@ -36,11 +36,18 @@ te      10.0		# final time
 dt      0.02	    # time stepsize
 tau     0.5		    # safety factor for time stepsize control (<0 constant delt)
 
+# Multigrid data:
+# ---------
+
+levels        2         # Multigrid levels
+presmooth     20        # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
 # Pressure Iteration Data:
 # -----------------------
 
 itermax  1000		# maximal number of pressure iteration in one time step
 eps      0.001		# stopping tolerance for pressure iteration
-omg      1.9		# relaxation parameter for SOR iteration
+omg      1.7		# relaxation parameter for SOR iteration
 gamma    0.9		# upwind differencing factor gamma
 #===============================================================================

BasicSolver/2D-mpi/include_ICX.mk

@@ -0,0 +1,20 @@
+ifeq ($(ENABLE_MPI),true)
+CC   = mpiicx
+DEFINES  = -D_MPI
+else
+CC = icx
+endif
+
+GCC  = gcc
+LINKER = $(CC)
+
+ifeq ($(ENABLE_OPENMP),true)
+OPENMP   = -qopenmp
+endif
+
+VERSION  = --version
+CFLAGS   =  -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) -Wno-unused-command-line-argument
+LFLAGS   = $(OPENMP)
+DEFINES  += -D_GNU_SOURCE# -DDEBUG
+INCLUDES =
+LIBS     =

BasicSolver/2D-mpi/residual.plot

@@ -0,0 +1,9 @@
+set terminal png size 1800,768 enhanced font ,12
+set output 'residual.png'
+set datafile separator whitespace
+set xlabel "Timestep"
+set ylabel "Residual"
+
+set logscale y 2
+
+plot 'residual.dat' using 1:2 title "Residual"

BasicSolver/2D-mpi/src/comm-v1.c

@@ -4,23 +4,12 @@
  * Use of this source code is governed by a MIT style
  * license that can be found in the LICENSE file.
  */
+#include <stdio.h>
 #include <stdlib.h>
 
 #include "comm.h"
 
 #ifdef _MPI
-// subroutines local to this module
-static int sum(int* sizes, int position)
-{
-    int sum = 0;
-
-    for (int i = 0; i < position; i++) {
-        sum += sizes[i];
-    }
-
-    return sum;
-}
-
 static void gatherArray(
     Comm* c, int cnt, int* rcvCounts, int* displs, double* src, double* dst)
 {
@@ -41,6 +30,52 @@ static void gatherArray(
         MPI_COMM_WORLD);
 }
 #endif // defined _MPI
+void commCollectResult(Comm* c,
+    double* ug,
+    double* vg,
+    double* pg,
+    double* u,
+    double* v,
+    double* p,
+    int imax,
+    int jmax)
+{
+#ifdef _MPI
+    int *rcvCounts, *displs;
+    int cnt = c->jmaxLocal * (imax + 2);
+
+    if (c->rank == 0) {
+        rcvCounts = (int*)malloc(c->size * sizeof(int));
+        displs    = (int*)malloc(c->size * sizeof(int));
+        for (int i = 0; i < c->size; ++i) {
+            rcvCounts[i] = 0;
+            displs[i]    = 0;
+        }
+    }
+
+    if (c->rank == 0 && c->size == 1) {
+        cnt = (c->jmaxLocal + 2) * (imax + 2);
+    } else if (c->rank == 0 || c->rank == (c->size - 1)) {
+        cnt = (c->jmaxLocal + 1) * (imax + 2);
+    }
+
+    MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD);
+
+    if (c->rank == 0) {
+        displs[0]  = 0;
+        int cursor = rcvCounts[0];
+
+        for (int i = 1; i < c->size; i++) {
+            displs[i] = cursor;
+            cursor += rcvCounts[i];
+        }
+    }
+
+    gatherArray(c, cnt, rcvCounts, displs, p, pg);
+    gatherArray(c, cnt, rcvCounts, displs, u, ug);
+    gatherArray(c, cnt, rcvCounts, displs, v, vg);
+#endif
+}
 
 // exported subroutines
 int commIsBoundary(Comm* c, int direction)
@@ -67,6 +102,7 @@ int commIsBoundary(Comm* c, int direction)
 
 void commExchange(Comm* c, double* grid)
 {
+    // printf("Rank : %d In exchange \n", c->rank);
 #ifdef _MPI
     MPI_Request requests[4] = { MPI_REQUEST_NULL,
         MPI_REQUEST_NULL,
@@ -126,56 +162,65 @@ void commShift(Comm* c, double* f, double* g)
 #endif
 }
 
-void commCollectResult(Comm* c,
-    double* ug,
-    double* vg,
-    double* pg,
-    double* u,
-    double* v,
-    double* p,
-    int jmax,
-    int imax)
-{
-#ifdef _MPI
-    int *rcvCounts, *displs;
-    int cnt = c->jmaxLocal * (imax + 2);
-
-    if (c->rank == 0) {
-        rcvCounts = (int*)malloc(c->size * sizeof(int));
-        displs    = (int*)malloc(c->size * sizeof(int));
-    }
-
-    if (c->rank == 0 && c->size == 1) {
-        cnt = (c->jmaxLocal + 2) * (imax + 2);
-    } else if (c->rank == 0 || c->rank == (c->size - 1)) {
-        cnt = (c->jmaxLocal + 1) * (imax + 2);
-    }
-
-    MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD);
-
-    if (c->rank == 0) {
-        displs[0]  = 0;
-        int cursor = rcvCounts[0];
-
-        for (int i = 1; i < c->size; i++) {
-            displs[i] = cursor;
-            cursor += rcvCounts[i];
-        }
-    }
-
-    gatherArray(c, cnt, rcvCounts, displs, p, pg);
-    gatherArray(c, cnt, rcvCounts, displs, u, ug);
-    gatherArray(c, cnt, rcvCounts, displs, v, vg);
-#endif
-}
-
 void commPartition(Comm* c, int jmax, int imax)
 {
 #ifdef _MPI
     c->imaxLocal = imax;
-    c->jmaxLocal = sizeOfRank(c->rank, c->size, jmax);
+    c->jmaxLocal = sizeOfRank(c->coords[JDIM], c->size, jmax);
+
+    c->neighbours[BOTTOM] = c->rank == 0 ? -1 : c->rank - 1;
+    c->neighbours[TOP]    = c->rank == (c->size - 1) ? -1 : c->rank + 1;
+    c->neighbours[LEFT]   = -1;
+    c->neighbours[RIGHT]  = -1;
+
+    c->coords[IDIM] = 0;
+    c->coords[JDIM] = c->rank;
+
+    c->dims[IDIM] = 1;
+    c->dims[JDIM] = c->size;
 #else
     c->imaxLocal = imax;
     c->jmaxLocal = jmax;
 #endif
 }
+
+void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
+{
+
+#if defined _MPI
+    newcomm->comm = MPI_COMM_NULL;
+    int result    = MPI_Comm_dup(MPI_COMM_WORLD, &newcomm->comm);
+
+    if (result == MPI_ERR_COMM) {
+        printf("\nNull communicator. Duplication failed !!\n");
+    }
+
+    newcomm->rank = oldcomm->rank;
+    newcomm->size = oldcomm->size;
+
+    newcomm->imaxLocal = imaxLocal / 2;
+    newcomm->jmaxLocal = jmaxLocal / 2;
+
+    newcomm->neighbours[BOTTOM] = newcomm->rank == 0 ? -1 : newcomm->rank - 1;
+    newcomm->neighbours[TOP]    = newcomm->rank == (newcomm->size - 1) ? -1
+                                                                       : newcomm->rank + 1;
+    newcomm->neighbours[LEFT]   = -1;
+    newcomm->neighbours[RIGHT]  = -1;
+
+    newcomm->coords[IDIM] = 0;
+    newcomm->coords[JDIM] = newcomm->rank;
+
+    newcomm->dims[IDIM] = 1;
+    newcomm->dims[JDIM] = newcomm->size;
+
+#endif
+    newcomm->imaxLocal = imaxLocal;
+    newcomm->jmaxLocal = jmaxLocal;
+}
+
+void commFreeCommunicator(Comm* comm)
+{
+#ifdef _MPI
+    MPI_Comm_free(&comm->comm);
+#endif
+}

BasicSolver/2D-mpi/src/comm-v2.c

@@ -4,18 +4,18 @@
  * Use of this source code is governed by a MIT style
  * license that can be found in the LICENSE file.
  */
+#include "comm.h"
 #include <stdio.h>
 #include <stdlib.h>
-
-#include "comm.h"
+#include <string.h>
 
 #ifdef _MPI
 // subroutines local to this module
-static int sum(int* sizes, int position)
+static int sum(int* sizes, int init, int offset, int coord)
 {
     int sum = 0;
 
-    for (int i = 0; i < position; i++) {
+    for (int i = init - offset; coord > 0; i -= offset, --coord) {
         sum += sizes[i];
     }
 
@@ -79,8 +79,8 @@ static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax
             int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
             int coords[NDIMS];
             MPI_Cart_coords(c->comm, i, NDIMS, coords);
-            int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
-                sum(newSizesI, coords[IDIM]) };
+            int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]),
+                sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) };
             printf(
                 "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
                 "Starts(i,j): %d %d\n",
@@ -252,8 +252,8 @@ void commPartition(Comm* c, int jmax, int imax)
     MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]);
     MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
 
-    int imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
-    int jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
+    int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax);
+    int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax);
 
     c->imaxLocal = imaxLocal;
     c->jmaxLocal = jmaxLocal;
@@ -285,3 +285,58 @@ void commPartition(Comm* c, int jmax, int imax)
     c->jmaxLocal = jmax;
 #endif
 }
+
+void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
+{
+#if defined _MPI
+    newcomm->comm = MPI_COMM_NULL;
+    int result    = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
+
+    if (result == MPI_ERR_COMM) {
+        printf("\nNull communicator. Duplication failed !!\n");
+    }
+
+    newcomm->rank = oldcomm->rank;
+    newcomm->size = oldcomm->size;
+
+    memcpy(&newcomm->neighbours, &oldcomm->neighbours, sizeof(oldcomm->neighbours));
+    memcpy(&newcomm->coords, &oldcomm->coords, sizeof(oldcomm->coords));
+    memcpy(&newcomm->dims, &oldcomm->dims, sizeof(oldcomm->dims));
+
+    newcomm->imaxLocal = imaxLocal/2;
+    newcomm->jmaxLocal = jmaxLocal/2;
+
+    MPI_Datatype jBufferType;
+    MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
+    MPI_Type_commit(&jBufferType);
+
+    MPI_Datatype iBufferType;
+    MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
+    MPI_Type_commit(&iBufferType);
+
+    newcomm->bufferTypes[LEFT]   = iBufferType;
+    newcomm->bufferTypes[RIGHT]  = iBufferType;
+    newcomm->bufferTypes[BOTTOM] = jBufferType;
+    newcomm->bufferTypes[TOP]    = jBufferType;
+
+    newcomm->sdispls[LEFT]   = (imaxLocal + 2) + 1;
+    newcomm->sdispls[RIGHT]  = (imaxLocal + 2) + imaxLocal;
+    newcomm->sdispls[BOTTOM] = (imaxLocal + 2) + 1;
+    newcomm->sdispls[TOP]    = jmaxLocal * (imaxLocal + 2) + 1;
+
+    newcomm->rdispls[LEFT]   = (imaxLocal + 2);
+    newcomm->rdispls[RIGHT]  = (imaxLocal + 2) + (imaxLocal + 1);
+    newcomm->rdispls[BOTTOM] = 1;
+    newcomm->rdispls[TOP]    = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
+#else
+    newcomm->imaxLocal = imaxLocal;
+    newcomm->jmaxLocal = jmaxLocal;
+#endif
+}
+
+void commFreeCommunicator(Comm* comm)
+{
+#ifdef _MPI
+    MPI_Comm_free(&comm->comm);
+#endif
+}

BasicSolver/2D-mpi/src/comm-v3.c

@@ -11,11 +11,11 @@
 
 #ifdef _MPI
 // subroutines local to this module
-static int sum(int* sizes, int position)
+static int sum(int* sizes, int init, int offset, int coord)
 {
     int sum = 0;
 
-    for (int i = 0; i < position; i++) {
+    for (int i = init - offset; coord > 0; i -= offset, --coord) {
         sum += sizes[i];
     }
 
@@ -79,8 +79,8 @@ static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax
             int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
             int coords[NDIMS];
             MPI_Cart_coords(c->comm, i, NDIMS, coords);
-            int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
-                sum(newSizesI, coords[IDIM]) };
+            int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]),
+                sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) };
             printf(
                 "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
                 "Starts(i,j): %d %d\n",
@@ -139,7 +139,6 @@ void commExchange(Comm* c, double* grid)
 {
 #ifdef _MPI
     int counts[NDIRS] = { 1, 1, 1, 1 };
-
     MPI_Neighbor_alltoallw(grid,
         counts,
         c->sdispls,
@@ -233,8 +232,8 @@ void commPartition(Comm* c, int jmax, int imax)
     MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]);
     MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
 
-    int imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
-    int jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
+    int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax);
+    int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax);
 
     c->imaxLocal = imaxLocal;
     c->jmaxLocal = jmaxLocal;
@@ -267,3 +266,55 @@ void commPartition(Comm* c, int jmax, int imax)
     c->jmaxLocal = jmax;
 #endif
 }
+
+void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
+{
+#if defined _MPI
+
+    int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
+
+    if (result == MPI_ERR_COMM) {
+        printf("\nNull communicator. Duplication failed !!\n");
+    }
+
+    newcomm->rank = oldcomm->rank;
+    newcomm->size = oldcomm->size;
+    
+
+    newcomm->imaxLocal = imaxLocal / 2;
+    newcomm->jmaxLocal = jmaxLocal / 2;
+
+    MPI_Datatype jBufferType;
+    MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
+    MPI_Type_commit(&jBufferType);
+
+    MPI_Datatype iBufferType;
+    MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
+    MPI_Type_commit(&iBufferType);
+
+    newcomm->bufferTypes[LEFT]   = iBufferType;
+    newcomm->bufferTypes[RIGHT]  = iBufferType;
+    newcomm->bufferTypes[BOTTOM] = jBufferType;
+    newcomm->bufferTypes[TOP]    = jBufferType;
+
+    newcomm->sdispls[LEFT]   = (imaxLocal + 2) + 1;
+    newcomm->sdispls[RIGHT]  = (imaxLocal + 2) + imaxLocal;
+    newcomm->sdispls[BOTTOM] = (imaxLocal + 2) + 1;
+    newcomm->sdispls[TOP]    = jmaxLocal * (imaxLocal + 2) + 1;
+
+    newcomm->rdispls[LEFT]   = (imaxLocal + 2);
+    newcomm->rdispls[RIGHT]  = (imaxLocal + 2) + (imaxLocal + 1);
+    newcomm->rdispls[BOTTOM] = 1;
+    newcomm->rdispls[TOP]    = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
+#else
+    newcomm->imaxLocal = imaxLocal;
+    newcomm->jmaxLocal = jmaxLocal;
+#endif
+}
+
+void commFreeCommunicator(Comm* comm)
+{
+#ifdef _MPI
+    MPI_Comm_free(&comm->comm);
+#endif
+}

BasicSolver/2D-mpi/src/comm.h

@@ -41,6 +41,8 @@ extern void commExchange(Comm*, double*);
 extern void commShift(Comm* c, double* f, double* g);
 extern void commReduction(double* v, int op);
 extern int commIsBoundary(Comm* c, int direction);
+extern void commUpdateDatatypes(Comm*, Comm*, int, int);
+extern void commFreeCommunicator(Comm*);
 extern void commCollectResult(Comm* c,
     double* ug,
     double* vg,

BasicSolver/2D-mpi/src/discretization.c

@@ -273,17 +273,18 @@ void setSpecialBoundaryCondition(Discretization* s)
         if (commIsBoundary(&s->comm, LEFT)) {
             double ylength = s->grid.ylength;
             double dy      = s->grid.dy;
-            int rest       = s->grid.jmax % s->comm.size;
-            int yc         = s->comm.rank * (s->grid.jmax / s->comm.size) +
+            int rest       = s->grid.jmax % s->comm.dims[JDIM];
+            int yc         = s->comm.rank * (s->grid.jmax / s->comm.dims[JDIM]) +
                      MIN(rest, s->comm.rank);
             double ys = dy * (yc + 0.5);
             double y;
 
-            /* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */
+            // printf("RANK %d yc: %d ys: %f\n", s->comm.rank, yc, ys);
 
             for (int j = 1; j < jmaxLocal + 1; j++) {
                 y       = ys + dy * (j - 0.5);
                 U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength);
+
             }
         }
     }

BasicSolver/2D-mpi/src/main.c

@@ -16,6 +16,24 @@
 #include "solver.h"
 #include "timing.h"
 
+static FILE* initResidualWriter()
+{
+    FILE* fp;
+    fp = fopen("residual.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return fp;
+}
+
+static void writeResidual(FILE* fp, double ts, double res)
+{
+    fprintf(fp, "%f, %f\n", ts, res);
+}
+
 static void writeResults(Discretization* s)
 {
 #ifdef _MPI
@@ -26,7 +44,9 @@ static void writeResults(Discretization* s)
     double* pg = allocate(64, bytesize);
 
     commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->grid.imax, s->grid.jmax);
+    if (commIsMaster(&s->comm)) {
         writeResult(s, ug, vg, pg);
+    }
 
     free(ug);
     free(vg);
@@ -47,6 +67,9 @@ int main(int argc, char** argv)
     commInit(&d.comm, argc, argv);
     initParameter(&p);
 
+    FILE* fp;
+    if (commIsMaster(&d.comm)) fp = initResidualWriter();
+
     if (argc != 2) {
         printf("Usage: %s <configFile>\n", argv[0]);
         exit(EXIT_SUCCESS);
@@ -77,16 +100,21 @@ int main(int argc, char** argv)
     double tau = d.tau;
     double te  = d.te;
     double t   = 0.0;
+    double res = 0.0;
 
     timeStart = getTimeStamp();
     while (t <= te) {
+
         if (tau > 0.0) computeTimestep(&d);
         setBoundaryConditions(&d);
         setSpecialBoundaryCondition(&d);
         computeFG(&d);
         computeRHS(&d);
-        solve(&s, d.p, d.rhs);
+        res = solve(&s, d.p, d.rhs);
         adaptUV(&d);
+
+        if (commIsMaster(&d.comm)) writeResidual(fp, t, res);
+
         t += d.dt;
 
 #ifdef VERBOSE
@@ -104,7 +132,7 @@ int main(int argc, char** argv)
     if (commIsMaster(s.comm)) {
         printf("Solution took %.2fs\n", timeStop - timeStart);
     }
-
+    if (commIsMaster(&d.comm)) fclose(fp);
     writeResults(&d);
     commFinalize(s.comm);
     return EXIT_SUCCESS;

BasicSolver/2D-mpi/src/parameter.c

@@ -21,6 +21,9 @@ void initParameter(Parameter* param)
     param->itermax    = 1000;
     param->eps        = 0.0001;
     param->omg        = 1.8;
+    param->levels     = 5;
+    param->presmooth  = 5;
+    param->postsmooth = 5;
 }
 
 void readParameter(Parameter* param, const char* filename)
@@ -72,6 +75,9 @@ void readParameter(Parameter* param, const char* filename)
             PARSE_INT(bcRight);
             PARSE_INT(bcBottom);
             PARSE_INT(bcTop);
+            PARSE_INT(levels);
+            PARSE_INT(presmooth);
+            PARSE_INT(postsmooth);
             PARSE_REAL(u_init);
             PARSE_REAL(v_init);
             PARSE_REAL(p_init);

BasicSolver/2D-mpi/src/parameter.h

@@ -18,6 +18,7 @@ typedef struct {
     char* name;
     int bcLeft, bcRight, bcBottom, bcTop;
     double u_init, v_init, p_init;
+    int levels, presmooth, postsmooth;
 } Parameter;
 
 void initParameter(Parameter*);

BasicSolver/2D-mpi/src/progress.c

@@ -8,7 +8,7 @@
 #include <mpi.h>
 #include <stdio.h>
 #include <string.h>
-
+#include <stdlib.h>
 #include "progress.h"
 
 static double _end;

BasicSolver/2D-mpi/src/progress.h

@@ -10,5 +10,4 @@
 extern void initProgress(double);
 extern void printProgress(double);
 extern void stopProgress();
-
 #endif

BasicSolver/2D-mpi/src/solver-mg.c

@@ -0,0 +1,303 @@
+/*
+ * Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "allocate.h"
+#include "solver.h"
+#include "util.h"
+
+#define FINEST_LEVEL   0
+#define COARSEST_LEVEL (s->levels - 1)
+// #define S(i, j)        s[(j) * (imaxLocal + 2) + (i)]
+#define E(i, j)   e[(j) * (imaxLocal + 2) + (i)]
+#define R(i, j)   r[(j) * (imaxLocal + 2) + (i)]
+#define OLD(i, j) old[(j) * (imaxLocal + 2) + (i)]
+
+static void restrictMG(Solver* s, int level, Comm* comm)
+{
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+
+    double* r   = s->r[level + 1];
+    double* old = s->r[level];
+
+#ifdef _MPI
+    commExchange(comm, old);
+#endif
+
+    for (int j = 1; j < (jmaxLocal / 2) + 1; j++) {
+        for (int i = 1; i < (imaxLocal / 2) + 1; i++) {
+            R(i, j) = (OLD(2 * i - 1, 2 * j - 1) + OLD(2 * i, 2 * j - 1) * 2 +
+                          OLD(2 * i + 1, 2 * j - 1) + OLD(2 * i - 1, 2 * j) * 2 +
+                          OLD(2 * i, 2 * j) * 4 + OLD(2 * i + 1, 2 * j) * 2 +
+                          OLD(2 * i - 1, 2 * j + 1) + OLD(2 * i, 2 * j + 1) * 2 +
+                          OLD(2 * i + 1, 2 * j + 1)) /
+                      16.0;
+        }
+    }
+}
+
+static void prolongate(Solver* s, int level, Comm* comm)
+{
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+
+    double* old = s->r[level + 1];
+    double* e   = s->r[level];
+
+    for (int j = 2; j < jmaxLocal + 1; j += 2) {
+        for (int i = 2; i < imaxLocal + 1; i += 2) {
+            E(i, j) = OLD(i / 2, j / 2);
+        }
+    }
+}
+
+static void correct(Solver* s, double* p, int level, Comm* comm)
+{
+    double* e     = s->e[level];
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+
+    for (int j = 1; j < jmaxLocal + 1; ++j) {
+        for (int i = 1; i < imaxLocal + 1; ++i) {
+            P(i, j) += E(i, j);
+        }
+    }
+}
+
+static void setBoundaryCondition(Solver* s, double* p, int imaxLocal, int jmaxLocal)
+{
+#ifdef _MPI
+    if (commIsBoundary(s->comm, BOTTOM)) { // set bottom bc
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            P(i, 0) = P(i, 1);
+        }
+    }
+
+    if (commIsBoundary(s->comm, TOP)) { // set top bc
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            P(i, jmaxLocal + 1) = P(i, jmaxLocal);
+        }
+    }
+
+    if (commIsBoundary(s->comm, LEFT)) { // set left bc
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            P(0, j) = P(1, j);
+        }
+    }
+
+    if (commIsBoundary(s->comm, RIGHT)) { // set right bc
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            P(imaxLocal + 1, j) = P(imaxLocal, j);
+        }
+    }
+#else
+    for (int i = 1; i < imaxLocal + 1; i++) {
+        P(i, 0)             = P(i, 1);
+        P(i, jmaxLocal + 1) = P(i, jmaxLocal);
+    }
+
+    for (int j = 1; j < jmaxLocal + 1; j++) {
+        P(0, j)             = P(1, j);
+        P(imaxLocal + 1, j) = P(imaxLocal, j);
+    }
+#endif
+}
+
+static void smooth(Solver* s, double* p, double* rhs, int level, Comm* comm)
+{
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+
+    int imax = s->grid->imax;
+    int jmax = s->grid->jmax;
+
+    double dx2  = s->grid->dx * s->grid->dx;
+    double dy2  = s->grid->dy * s->grid->dy;
+    double idx2 = 1.0 / dx2;
+    double idy2 = 1.0 / dy2;
+
+    double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2);
+    double* r     = s->r[level];
+
+    double res = 1.0;
+    int pass, jsw, isw;
+
+    jsw = 1;
+
+    for (pass = 0; pass < 2; pass++) {
+        isw = jsw;
+
+#ifdef _MPI
+        commExchange(comm, p);
+#endif
+
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = isw; i < imaxLocal + 1; i += 2) {
+
+                P(i, j) -= factor *
+                           (RHS(i, j) -
+                               ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
+                                   (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2));
+            }
+            isw = 3 - isw;
+        }
+        jsw = 3 - jsw;
+    }
+}
+
+static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm)
+{
+    int imax      = s->grid->imax;
+    int jmax      = s->grid->jmax;
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+
+    double dx2    = s->grid->dx * s->grid->dx;
+    double dy2    = s->grid->dy * s->grid->dy;
+    double idx2   = 1.0 / dx2;
+    double idy2   = 1.0 / dy2;
+    double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2);
+    double* r     = s->r[level];
+    double res    = 1.0;
+    int pass, jsw, isw;
+
+    jsw = 1;
+
+    for (pass = 0; pass < 2; pass++) {
+        isw = jsw;
+
+#ifdef _MPI
+        commExchange(comm, p);
+#endif
+
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = isw; i < imaxLocal + 1; i += 2) {
+
+                R(i, j) = RHS(i, j) -
+                          ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
+                              (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2);
+
+                res += (R(i, j) * R(i, j));
+            }
+            isw = 3 - isw;
+        }
+        jsw = 3 - jsw;
+    }
+
+#ifdef _MPI
+    commReduction(&res, SUM);
+#endif
+
+    res = res / (double)(imax * jmax);
+#ifdef DEBUG
+    if (commIsMaster(s->comm)) {
+        printf("%d Residuum: %e\n", it, res);
+    }
+#endif
+    return res;
+}
+
+static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm)
+{
+    double res = 0.0;
+
+    // coarsest level
+    if (level == COARSEST_LEVEL) {
+        for (int i = 0; i < 5; i++) {
+            smooth(s, p, rhs, level, comm);
+        }
+        return res;
+    }
+
+    // pre-smoothing
+    for (int i = 0; i < s->presmooth; i++) {
+        smooth(s, p, rhs, level, comm);
+        if (level == FINEST_LEVEL)
+            setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal);
+    }
+
+    // calculate residuals
+    res = calculateResidual(s, p, rhs, level, comm);
+
+    // restrict
+    restrictMG(s, level, comm);
+
+    // Create a new comm object withupdated imaxLocal and jmaxLocal
+    // along with their updated bufferTypes, sdispls, rdispls
+    Comm newcomm;
+    commUpdateDatatypes(s->comm, &newcomm, comm->imaxLocal, comm->jmaxLocal);
+
+    // MGSolver on residual and error.
+    multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm);
+
+    commFreeCommunicator(&newcomm);
+
+    // prolongate
+    prolongate(s, level, comm);
+
+    // correct p on finer level using residual
+    correct(s, p, level, comm);
+
+    if (level == FINEST_LEVEL)
+        setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal);
+
+    // post-smoothing
+    for (int i = 0; i < s->postsmooth; i++) {
+        smooth(s, p, rhs, level, comm);
+        if (level == FINEST_LEVEL)
+            setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal);
+    }
+
+    return res;
+}
+
+void initSolver(Solver* s, Discretization* d, Parameter* p)
+{
+    s->eps        = p->eps;
+    s->omega      = p->omg;
+    s->itermax    = p->itermax;
+    s->levels     = p->levels;
+    s->grid       = &d->grid;
+    s->comm       = &d->comm;
+    s->presmooth  = p->presmooth;
+    s->postsmooth = p->postsmooth;
+
+    int imax   = s->grid->imax;
+    int jmax   = s->grid->jmax;
+    int levels = s->levels;
+    printf("Using Multigrid solver with %d levels\n", levels);
+
+    s->r = malloc(levels * sizeof(double*));
+    s->e = malloc(levels * sizeof(double*));
+
+    size_t size = (imax + 2) * (jmax + 2) * sizeof(double);
+
+    for (int j = 0; j < levels; j++) {
+        s->r[j] = allocate(64, size);
+        s->e[j] = allocate(64, size);
+
+        for (int i = 0; i < (imax + 2) * (jmax + 2); i++) {
+            s->r[j][i] = 0.0;
+            s->e[j][i] = 0.0;
+        }
+    }
+}
+
+double solve(Solver* s, double* p, double* rhs)
+{
+    double res = multiGrid(s, p, rhs, 0, s->comm);
+
+#ifdef VERBOSE
+    if (commIsMaster(s->comm)) {
+        printf("Residuum: %.6f\n", res);
+    }
+#endif
+
+    return res;
+}

BasicSolver/2D-mpi/src/solver-rb.c

@@ -23,7 +23,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     s->comm    = &d->comm;
 }
 
-void solve(Solver* s, double* p, double* rhs)
+double solve(Solver* s, double* p, double* rhs)
 {
     int imax      = s->grid->imax;
     int jmax      = s->grid->jmax;
@@ -101,4 +101,6 @@ void solve(Solver* s, double* p, double* rhs)
         printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
     }
 #endif
+
+    return res;
 }

BasicSolver/2D-mpi/src/solver.h

@@ -9,6 +9,7 @@
 #include "comm.h"
 #include "discretization.h"
 #include "grid.h"
+#include "mpi.h"
 #include "parameter.h"
 
 typedef struct {
@@ -17,10 +18,12 @@ typedef struct {
     /* parameters */
     double eps, omega;
     int itermax;
+    int levels, presmooth, postsmooth;
+    double **r, **e;
     /* communication */
     Comm* comm;
 } Solver;
 
 void initSolver(Solver*, Discretization*, Parameter*);
-void solve(Solver*, double*, double*);
+double solve(Solver*, double*, double*);
 #endif

BasicSolver/2D-seq/Makefile

@@ -38,9 +38,22 @@ $(BUILD_DIR)/%.s:  %.c
 	$(info ===>  GENERATE ASM  $@)
 	$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
 
-.PHONY: clean distclean tags info asm format
+.PHONY: clean distclean vis vis_clean tags info asm format
 
-clean:
+vis:
+	$(info ===>  GENERATE VISUALIZATION)
+	@gnuplot -e "filename='pressure.dat'" ./surface.plot
+	@gnuplot -e "filename='velocity.dat'" ./vector.plot
+	@gnuplot -e "filename='residual.dat'" ./residual.plot
+
+vis_clean:
+	$(info ===>  CLEAN VISUALIZATION)
+	@rm -f *.dat
+	@rm -f *.png
+	@rm -f ./vis_files/*.dat
+	@rm -f ./vis_files/*.gif
+
+clean: vis_clean
 	$(info ===>  CLEAN)
 	@rm -rf $(BUILD_DIR)
 	@rm -f tags

BasicSolver/2D-seq/canal.par

@@ -36,6 +36,13 @@ te      100.0   # final time
 dt      0.02    # time stepsize
 tau     0.5     # safety factor for time stepsize control (<0 constant delt)
 
+# Multigrid data:
+# ---------
+
+levels        3         # Multigrid levels
+presmooth     5         # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
 # Pressure Iteration Data:
 # -----------------------
 

BasicSolver/2D-seq/config.mk

@@ -1,12 +1,12 @@
 # Supported: GCC, CLANG, ICC
-TAG ?= CLANG
+TAG ?= ICC
 ENABLE_OPENMP ?= false
 # Supported: sor, rb, mg
-SOLVER ?= rb
+SOLVER ?= mg
 # Run in debug settings
 DEBUG ?= false
 
 #Feature options
 OPTIONS +=  -DARRAY_ALIGNMENT=64
-#OPTIONS +=  -DVERBOSE
+OPTIONS +=  -DVERBOSE
 #OPTIONS +=  -DDEBUG

BasicSolver/2D-seq/dcavity.par

@@ -36,6 +36,13 @@ te      10.0		# final time
 dt      0.02	    # time stepsize
 tau     0.5	    	# safety factor for time stepsize control (<0 constant delt)
 
+# Multigrid data:
+# ---------
+
+levels        2         # Multigrid levels
+presmooth     20         # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
 # Solver Data:
 # -----------------------
 
@@ -44,5 +51,4 @@ eps      0.001		# stopping tolerance for pressure iteration
 rho      0.5
 omg      1.7		# relaxation parameter for SOR iteration
 gamma    0.9		# upwind differencing factor gamma
-levels   5         # Multigrid levels
 #===============================================================================

BasicSolver/2D-seq/residual.plot

@@ -0,0 +1,9 @@
+set terminal png size 1800,768 enhanced font ,12
+set output 'residual.png'
+set datafile separator whitespace
+set xlabel "Timestep"
+set ylabel "Residual"
+
+set logscale y 2
+
+plot 'residual.dat' using 1:2 title "Residual"

BasicSolver/2D-seq/src/main.c

@@ -22,6 +22,8 @@ int main(int argc, char** argv)
     Solver s;
 
     initParameter(&p);
+    FILE* fp;
+    fp = initResidualWriter();
 
     if (argc != 2) {
         printf("Usage: %s <configFile>\n", argv[0]);
@@ -41,6 +43,7 @@ int main(int argc, char** argv)
     double te  = d.te;
     double t   = 0.0;
     int nt     = 0;
+    double res = 0.0;
 
     timeStart  = getTimeStamp();
     while (t <= te) {
@@ -50,17 +53,21 @@ int main(int argc, char** argv)
         computeFG(&d);
         computeRHS(&d);
         if (nt % 100 == 0) normalizePressure(&d);
-        solve(&s, d.p, d.rhs);
+        res = solve(&s, d.p, d.rhs);
         adaptUV(&d);
+
+        writeResidual(fp, t, res);
+
         t += d.dt;
         nt++;
 
 #ifdef VERBOSE
-        printf("TIME %f , TIMESTEP %f\n", t, solver.dt);
+        printf("TIME %f , TIMESTEP %f\n", t, d.dt);
 #else
         printProgress(t);
 #endif
     }
+    fclose(fp);
     timeStop = getTimeStamp();
     stopProgress();
     printf("Solution took %.2fs\n", timeStop - timeStart);

BasicSolver/2D-seq/src/parameter.c

@@ -24,6 +24,8 @@ void initParameter(Parameter* param)
     param->gamma   = 0.9;
     param->tau     = 0.5;
     param->levels  = 5;
+    param->presmooth = 5;
+    param->postsmooth = 5;
 }
 
 void readParameter(Parameter* param, const char* filename)
@@ -79,6 +81,8 @@ void readParameter(Parameter* param, const char* filename)
             PARSE_REAL(u_init);
             PARSE_REAL(v_init);
             PARSE_REAL(p_init);
+            PARSE_INT(presmooth);
+            PARSE_INT(postsmooth);
         }
     }
 

BasicSolver/2D-seq/src/parameter.h

@@ -18,6 +18,7 @@ typedef struct {
     char* name;
     int bcLeft, bcRight, bcBottom, bcTop;
     double u_init, v_init, p_init;
+    int presmooth, postsmooth;
 } Parameter;
 
 void initParameter(Parameter*);

BasicSolver/2D-seq/src/progress.c

@@ -49,3 +49,22 @@ void stopProgress()
     printf("\n");
     fflush(stdout);
 }
+
+FILE* initResidualWriter()
+{
+    FILE* fp;
+    fp = fopen("residual.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return fp;
+
+}
+
+void writeResidual(FILE* fp, double ts, double res)
+{
+    fprintf(fp, "%f, %f\n", ts, res);
+}

BasicSolver/2D-seq/src/progress.h

@@ -10,5 +10,7 @@
 extern void initProgress(double);
 extern void printProgress(double);
 extern void stopProgress(void);
+extern FILE* initResidualWriter(void);
+extern void writeResidual(FILE*, double, double);
 
 #endif

BasicSolver/2D-seq/src/solver-mg.c

@@ -84,6 +84,37 @@ static double smooth(Solver* s, double* p, double* rhs, int level, int imax, int
 
     jsw = 1;
 
+    for (pass = 0; pass < 2; pass++) {
+        isw = jsw;
+
+        for (int j = 1; j < jmax + 1; j++) {
+            for (int i = isw; i < imax + 1; i += 2) {
+
+                P(i, j) -= factor * (RHS(i, j) -
+                          ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
+                              (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2));
+
+            }
+            isw = 3 - isw;
+        }
+        jsw = 3 - jsw;
+    }
+
+}
+
+static double calculateResidual(Solver* s, double* p, double* rhs, int level, int imax, int jmax)
+{
+    double dx2    = s->grid->dx * s->grid->dx;
+    double dy2    = s->grid->dy * s->grid->dy;
+    double idx2   = 1.0 / dx2;
+    double idy2   = 1.0 / dy2;
+    double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2);
+    double* r     = s->r[level];
+    double res    = 1.0;
+    int pass, jsw, isw;
+
+    jsw = 1;
+
     for (pass = 0; pass < 2; pass++) {
         isw = jsw;
 
@@ -94,7 +125,6 @@ static double smooth(Solver* s, double* p, double* rhs, int level, int imax, int
                           ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
                               (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2);
 
-                P(i, j) -= (factor * R(i, j));
                 res += (R(i, j) * R(i, j));
             }
             isw = 3 - isw;
@@ -110,7 +140,7 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax,
 {
     double res = 0.0;
 
-    // coarsest level TODO: Use direct solver?
+    // coarsest level
     if (level == COARSEST_LEVEL) {
         for (int i = 0; i < 5; i++) {
             smooth(s, p, rhs, level, imax, jmax);
@@ -118,17 +148,18 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax,
         return res;
     }
 
-    // pre-smoothing TODO: Make smoothing steps configurable?
-    for (int i = 0; i < 5; i++) {
+    // pre-smoothing
+    for (int i = 0; i < s->presmooth; i++) {
         smooth(s, p, rhs, level, imax, jmax);
         if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax);
     }
 
+    res = calculateResidual(s, p, rhs, level, imax, jmax);
+
     // restrict
     restrictMG(s, level, imax, jmax);
 
     // MGSolver on residual and error.
-    // TODO: What if there is a rest?
     multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, imax / 2, jmax / 2);
 
     // prolongate
@@ -139,8 +170,8 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax,
     if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax);
 
     // post-smoothing
-    for (int i = 0; i < 5; i++) {
-        res = smooth(s, p, rhs, level, imax, jmax);
+    for (int i = 0; i < s->postsmooth; i++) {
+        smooth(s, p, rhs, level, imax, jmax);
         if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax);
     }
 
@@ -154,6 +185,8 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     s->itermax = p->itermax;
     s->levels  = p->levels;
     s->grid    = &d->grid;
+    s->presmooth = p->presmooth;
+    s->postsmooth = p->postsmooth;
 
     int imax   = s->grid->imax;
     int jmax   = s->grid->jmax;
@@ -176,11 +209,13 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     }
 }
 
-void solve(Solver* s, double* p, double* rhs)
+double solve(Solver* s, double* p, double* rhs)
 {
     double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax);
 
 #ifdef VERBOSE
     printf("Residuum: %.6f\n", res);
 #endif
+
+return res;
 }

BasicSolver/2D-seq/src/solver-rb.c

@@ -15,7 +15,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     s->omega   = p->omg;
 }
 
-void solve(Solver* solver, double* p, double* rhs)
+double solve(Solver* solver, double* p, double* rhs)
 {
     int imax      = solver->grid->imax;
     int jmax      = solver->grid->jmax;
@@ -73,4 +73,6 @@ void solve(Solver* solver, double* p, double* rhs)
 #ifdef VERBOSE
     printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
 #endif
+
+    return res;
 }

BasicSolver/2D-seq/src/solver-sor.c

@@ -15,7 +15,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     s->omega   = p->omg;
 }
 
-void solve(Solver* solver, double* p, double* rhs)
+double solve(Solver* solver, double* p, double* rhs)
 {
     int imax      = solver->grid->imax;
     int jmax      = solver->grid->jmax;
@@ -65,4 +65,6 @@ void solve(Solver* solver, double* p, double* rhs)
 #ifdef VERBOSE
     printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
 #endif
+
+    return res;
 }

BasicSolver/2D-seq/src/solver.h

@@ -18,9 +18,10 @@ typedef struct {
     int itermax;
     int levels;
     double **r, **e;
+        int presmooth, postsmooth;
 } Solver;
 
 extern void initSolver(Solver*, Discretization*, Parameter*);
-extern void solve(Solver*, double*, double*);
+extern double solve(Solver*, double*, double*);
 
 #endif

BasicSolver/3D-mpi/Makefile

@@ -22,6 +22,7 @@ SRC       = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)
 ASM       = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC))
 OBJ       = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC))
 OBJ      += $(BUILD_DIR)/vtkWriter-$(VTK_OUTPUT_FMT).o
+OBJ      += $(BUILD_DIR)/solver-$(SOLVER).o
 SOURCES   = $(SRC) $(wildcard $(SRC_DIR)/*.h)
 ifeq ($(VTK_OUTPUT_FMT),mpi)
 DEFINES  += -D_VTK_WRITER_MPI
@@ -42,9 +43,19 @@ $(BUILD_DIR)/%.s:  %.c
 	$(info ===>  GENERATE ASM  $@)
 	$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
 
-.PHONY: clean distclean tags info asm format
+.PHONY: clean distclean vis vis_clean tags info asm format
 
-clean:
+vis:
+	$(info ===>  GENERATE VISUALIZATION)
+	@gnuplot -e "filename='residual.dat'" ./residual.plot
+
+vis_clean:
+	$(info ===>  CLEAN VISUALIZATION)
+	@rm -f *.dat
+	@rm -f *.vtk
+	@rm -f *.png
+
+clean: vis_clean
 	$(info ===>  CLEAN)
 	@rm -rf $(BUILD_DIR)
 	@rm -f tags

BasicSolver/3D-mpi/canal.par

@@ -32,21 +32,28 @@ xlength       30.0     # domain size in x-direction
 ylength       4.0	   # domain size in y-direction
 zlength       4.0	   # domain size in z-direction
 imax          200      # number of interior cells in x-direction
-jmax          50	   # number of interior cells in y-direction
-kmax          50	   # number of interior cells in z-direction
+jmax          40	   # number of interior cells in y-direction
+kmax          40	   # number of interior cells in z-direction
 
 # Time Data:
 # ---------
 
-te       100.0   # final time
+te       60.0   # final time
 dt       0.02    # time stepsize
 tau      0.5     # safety factor for time stepsize control (<0 constant delt)
 
+# Multigrid data:
+# ---------
+
+levels        2         # Multigrid levels
+presmooth     20         # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
 # Pressure Iteration Data:
 # -----------------------
 
 itermax       500       # maximal number of pressure iteration in one time step
 eps           0.0001   # stopping tolerance for pressure iteration
-omg           1.3       # relaxation parameter for SOR iteration
+omg           1.7       # relaxation parameter for SOR iteration
 gamma         0.9       # upwind differencing factor gamma
 #===============================================================================

BasicSolver/3D-mpi/config.mk

@@ -1,7 +1,10 @@
 # Supported: GCC, CLANG, ICC
-TAG ?= CLANG
+TAG ?= ICC
+# Supported: true, false
 ENABLE_MPI ?= true
 ENABLE_OPENMP ?= false
+# Supported: rb, mg
+SOLVER ?= mg
 # Supported: seq, mpi
 VTK_OUTPUT_FMT ?= seq
 

BasicSolver/3D-mpi/dcavity.par

@@ -42,6 +42,13 @@ te       10.0		# final time
 dt       0.02	    # time stepsize
 tau      0.5		# safety factor for time stepsize control (<0 constant delt)
 
+# Multigrid data:
+# ---------
+
+levels        3         # Multigrid levels
+presmooth     20        # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
 # Pressure Iteration Data:
 # -----------------------
 

BasicSolver/3D-mpi/residual.plot

@@ -0,0 +1,9 @@
+set terminal png size 1800,768 enhanced font ,12
+set output 'residual.png'
+set datafile separator whitespace
+set xlabel "Timestep"
+set ylabel "Residual"
+
+set logscale y 2
+
+plot 'residual.dat' using 1:2 title "Residual"

BasicSolver/3D-mpi/src/comm.c

@@ -167,11 +167,12 @@ static void assembleResult(Comm* c,
     MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
 }
 
-static int sum(int* sizes, int position)
+// subroutines local to this module
+static int sum(int* sizes, int init, int offset, int coord)
 {
     int sum = 0;
 
-    for (int i = 0; i < position; i++) {
+    for (int i = init - offset; coord > 0; i -= offset, --coord) {
         sum += sizes[i];
     }
 
@@ -336,7 +337,6 @@ void commCollectResult(Comm* c,
     int imaxLocal = c->imaxLocal;
     int jmaxLocal = c->jmaxLocal;
     int kmaxLocal = c->kmaxLocal;
-
 #if defined(_MPI)
     int offset[c->size * NDIMS];
     int imaxLocalAll[c->size];
@@ -351,9 +351,13 @@ void commCollectResult(Comm* c,
         for (int i = 0; i < c->size; i++) {
             int coords[NCORDS];
             MPI_Cart_coords(c->comm, i, NDIMS, coords);
-            offset[i * NDIMS + IDIM] = sum(imaxLocalAll, coords[ICORD]);
-            offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, coords[JCORD]);
-            offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, coords[KCORD]);
+            offset[i * NDIMS + IDIM] = sum(imaxLocalAll,
+                i,
+                c->dims[IDIM] * c->dims[JDIM],
+                coords[ICORD]);
+            offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, i, c->dims[IDIM], coords[JCORD]);
+            offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, i, 1, coords[KCORD]);
+
             printf("Rank: %d, Coords(k,j,i): %d %d %d, Size(k,j,i): %d %d %d, "
                    "Offset(k,j,i): %d %d %d\n",
                 i,
@@ -562,9 +566,9 @@ void commPartition(Comm* c, int kmax, int jmax, int imax)
     MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]);
     MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords);
 
-    c->imaxLocal = sizeOfRank(c->rank, dims[ICORD], imax);
-    c->jmaxLocal = sizeOfRank(c->rank, dims[JCORD], jmax);
-    c->kmaxLocal = sizeOfRank(c->rank, dims[KCORD], kmax);
+    c->imaxLocal = sizeOfRank(c->coords[KDIM], dims[ICORD], imax);
+    c->jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JCORD], jmax);
+    c->kmaxLocal = sizeOfRank(c->coords[IDIM], dims[KCORD], kmax);
 
     // setup buffer types for communication
     setupCommunication(c, LEFT, BULK);
@@ -597,3 +601,47 @@ void commFinalize(Comm* c)
     MPI_Finalize();
 #endif
 }
+
+void commUpdateDatatypes(
+    Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal)
+{
+#if defined _MPI
+
+    int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
+
+    if (result == MPI_ERR_COMM) {
+        printf("\nNull communicator. Duplication failed !!\n");
+    }
+
+    newcomm->rank = oldcomm->rank;
+    newcomm->size = oldcomm->size;
+
+    newcomm->imaxLocal = imaxLocal / 2;
+    newcomm->jmaxLocal = jmaxLocal / 2;
+    newcomm->kmaxLocal = kmaxLocal / 2;
+
+    setupCommunication(newcomm, LEFT, BULK);
+    setupCommunication(newcomm, LEFT, HALO);
+    setupCommunication(newcomm, RIGHT, BULK);
+    setupCommunication(newcomm, RIGHT, HALO);
+    setupCommunication(newcomm, BOTTOM, BULK);
+    setupCommunication(newcomm, BOTTOM, HALO);
+    setupCommunication(newcomm, TOP, BULK);
+    setupCommunication(newcomm, TOP, HALO);
+    setupCommunication(newcomm, FRONT, BULK);
+    setupCommunication(newcomm, FRONT, HALO);
+    setupCommunication(newcomm, BACK, BULK);
+    setupCommunication(newcomm, BACK, HALO);
+#else
+    newcomm->imaxLocal = imaxLocal;
+    newcomm->jmaxLocal = jmaxLocal;
+    newcomm->kmaxLocal = kmaxLocal;
+#endif
+}
+
+void commFreeCommunicator(Comm* comm)
+{
+#ifdef _MPI
+    MPI_Comm_free(&comm->comm);
+#endif
+}

BasicSolver/3D-mpi/src/comm.h

@@ -23,6 +23,7 @@ typedef enum dimension { KDIM = 0, JDIM, IDIM, NDIMS } Dimension;
 enum layer { HALO = 0, BULK };
 enum op { MAX = 0, SUM };
 
+
 typedef struct {
     int rank;
     int size;
@@ -45,6 +46,9 @@ extern void commShift(Comm* c, double* f, double* g, double* h);
 extern void commReduction(double* v, int op);
 extern int commIsBoundary(Comm* c, Direction direction);
 extern void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax);
+extern void commFreeCommunicator(Comm* comm);
+extern void commUpdateDatatypes(
+    Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal);
 extern void commCollectResult(Comm* c,
     double* ug,
     double* vg,

BasicSolver/3D-mpi/src/discretization.c

@@ -0,0 +1,729 @@
+/*
+ * Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "allocate.h"
+#include "discretization.h"
+#include "parameter.h"
+#include "util.h"
+
+#define P(i, j, k)                                                                       \
+    p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define F(i, j, k)                                                                       \
+    f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define G(i, j, k)                                                                       \
+    g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define H(i, j, k)                                                                       \
+    h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define U(i, j, k)                                                                       \
+    u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define V(i, j, k)                                                                       \
+    v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define W(i, j, k)                                                                       \
+    w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define RHS(i, j, k)                                                                     \
+    rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+
+static void printConfig(Discretization* s)
+{
+    if (commIsMaster(&s->comm)) {
+        printf("Parameters for #%s#\n", s->problem);
+        printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n",
+            s->bcLeft,
+            s->bcRight,
+            s->bcBottom,
+            s->bcTop,
+            s->bcFront,
+            s->bcBack);
+        printf("\tReynolds number: %.2f\n", s->re);
+        printf("\tGx Gy: %.2f %.2f %.2f\n", s->gx, s->gy, s->gz);
+        printf("Geometry data:\n");
+        printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n",
+            s->grid.xlength,
+            s->grid.ylength,
+            s->grid.zlength);
+        printf("\tCells (x, y, z): %d, %d, %d\n",
+            s->grid.imax,
+            s->grid.jmax,
+            s->grid.kmax);
+        printf("\tCell size (dx, dy, dz): %f, %f, %f\n",
+            s->grid.dx,
+            s->grid.dy,
+            s->grid.dz);
+        printf("Timestep parameters:\n");
+        printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te);
+        printf("\tdt bound: %.6f\n", s->dtBound);
+        printf("\tTau factor: %.2f\n", s->tau);
+        printf("Iterative parameters:\n");
+        printf("\tMax iterations: %d\n", s->itermax);
+        printf("\tepsilon (stopping tolerance) : %f\n", s->eps);
+        printf("\tgamma factor: %f\n", s->gamma);
+        printf("\tomega (SOR relaxation): %f\n", s->omega);
+    }
+    commPrintConfig(&s->comm);
+}
+
+void initDiscretization(Discretization* s, Parameter* params)
+{
+    s->problem  = params->name;
+    s->bcLeft   = params->bcLeft;
+    s->bcRight  = params->bcRight;
+    s->bcBottom = params->bcBottom;
+    s->bcTop    = params->bcTop;
+    s->bcFront  = params->bcFront;
+    s->bcBack   = params->bcBack;
+
+    s->grid.imax    = params->imax;
+    s->grid.jmax    = params->jmax;
+    s->grid.kmax    = params->kmax;
+    s->grid.xlength = params->xlength;
+    s->grid.ylength = params->ylength;
+    s->grid.zlength = params->zlength;
+    s->grid.dx      = params->xlength / params->imax;
+    s->grid.dy      = params->ylength / params->jmax;
+    s->grid.dz      = params->zlength / params->kmax;
+
+    s->eps     = params->eps;
+    s->omega   = params->omg;
+    s->itermax = params->itermax;
+    s->re      = params->re;
+    s->gx      = params->gx;
+    s->gy      = params->gy;
+    s->gz      = params->gz;
+    s->dt      = params->dt;
+    s->te      = params->te;
+    s->tau     = params->tau;
+    s->gamma   = params->gamma;
+
+    /* allocate arrays */
+    int imaxLocal = s->comm.imaxLocal;
+    int jmaxLocal = s->comm.jmaxLocal;
+    int kmaxLocal = s->comm.kmaxLocal;
+    size_t size   = (imaxLocal + 2) * (jmaxLocal + 2) * (kmaxLocal + 2);
+
+    s->u   = allocate(64, size * sizeof(double));
+    s->v   = allocate(64, size * sizeof(double));
+    s->w   = allocate(64, size * sizeof(double));
+    s->p   = allocate(64, size * sizeof(double));
+    s->rhs = allocate(64, size * sizeof(double));
+    s->f   = allocate(64, size * sizeof(double));
+    s->g   = allocate(64, size * sizeof(double));
+    s->h   = allocate(64, size * sizeof(double));
+
+    for (int i = 0; i < size; i++) {
+        s->u[i]   = params->u_init;
+        s->v[i]   = params->v_init;
+        s->w[i]   = params->w_init;
+        s->p[i]   = params->p_init;
+        s->rhs[i] = 0.0;
+        s->f[i]   = 0.0;
+        s->g[i]   = 0.0;
+        s->h[i]   = 0.0;
+    }
+
+    double dx = s->grid.dx;
+    double dy = s->grid.dy;
+    double dz = s->grid.dz;
+
+    double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz);
+    s->dtBound       = 0.5 * s->re * 1.0 / invSqrSum;
+
+#ifdef VERBOSE
+    printConfig(s);
+#endif /* VERBOSE */
+}
+
+void setBoundaryConditions(Discretization* s)
+{
+    int imaxLocal = s->comm.imaxLocal;
+    int jmaxLocal = s->comm.jmaxLocal;
+    int kmaxLocal = s->comm.kmaxLocal;
+
+    double* u = s->u;
+    double* v = s->v;
+    double* w = s->w;
+
+    if (commIsBoundary(&s->comm, TOP)) {
+        switch (s->bcTop) {
+        case NOSLIP:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, jmaxLocal + 1, k) = -U(i, jmaxLocal, k);
+                    V(i, jmaxLocal, k)     = 0.0;
+                    W(i, jmaxLocal + 1, k) = -W(i, jmaxLocal, k);
+                }
+            }
+            break;
+        case SLIP:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k);
+                    V(i, jmaxLocal, k)     = 0.0;
+                    W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k);
+                }
+            }
+            break;
+        case OUTFLOW:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k);
+                    V(i, jmaxLocal, k)     = V(i, jmaxLocal - 1, k);
+                    W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k);
+                }
+            }
+            break;
+        case PERIODIC:
+            break;
+        }
+    }
+
+    if (commIsBoundary(&s->comm, BOTTOM)) {
+        switch (s->bcBottom) {
+        case NOSLIP:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, 0, k) = -U(i, 1, k);
+                    V(i, 0, k) = 0.0;
+                    W(i, 0, k) = -W(i, 1, k);
+                }
+            }
+            break;
+        case SLIP:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, 0, k) = U(i, 1, k);
+                    V(i, 0, k) = 0.0;
+                    W(i, 0, k) = W(i, 1, k);
+                }
+            }
+            break;
+        case OUTFLOW:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, 0, k) = U(i, 1, k);
+                    V(i, 0, k) = V(i, 1, k);
+                    W(i, 0, k) = W(i, 1, k);
+                }
+            }
+            break;
+        case PERIODIC:
+            break;
+        }
+    }
+
+    if (commIsBoundary(&s->comm, LEFT)) {
+        switch (s->bcLeft) {
+        case NOSLIP:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    U(0, j, k) = 0.0;
+                    V(0, j, k) = -V(1, j, k);
+                    W(0, j, k) = -W(1, j, k);
+                }
+            }
+            break;
+        case SLIP:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    U(0, j, k) = 0.0;
+                    V(0, j, k) = V(1, j, k);
+                    W(0, j, k) = W(1, j, k);
+                }
+            }
+            break;
+        case OUTFLOW:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    U(0, j, k) = U(1, j, k);
+                    V(0, j, k) = V(1, j, k);
+                    W(0, j, k) = W(1, j, k);
+                }
+            }
+            break;
+        case PERIODIC:
+            break;
+        }
+    }
+
+    if (commIsBoundary(&s->comm, RIGHT)) {
+        switch (s->bcRight) {
+        case NOSLIP:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    U(imaxLocal, j, k)     = 0.0;
+                    V(imaxLocal + 1, j, k) = -V(imaxLocal, j, k);
+                    W(imaxLocal + 1, j, k) = -W(imaxLocal, j, k);
+                }
+            }
+            break;
+        case SLIP:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    U(imaxLocal, j, k)     = 0.0;
+                    V(imaxLocal + 1, j, k) = V(imaxLocal, j, k);
+                    W(imaxLocal + 1, j, k) = W(imaxLocal, j, k);
+                }
+            }
+            break;
+        case OUTFLOW:
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    U(imaxLocal, j, k)     = U(imaxLocal - 1, j, k);
+                    V(imaxLocal + 1, j, k) = V(imaxLocal, j, k);
+                    W(imaxLocal + 1, j, k) = W(imaxLocal, j, k);
+                }
+            }
+            break;
+        case PERIODIC:
+            break;
+        }
+    }
+
+    if (commIsBoundary(&s->comm, FRONT)) {
+        switch (s->bcFront) {
+        case NOSLIP:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, j, 0) = -U(i, j, 1);
+                    V(i, j, 0) = -V(i, j, 1);
+                    W(i, j, 0) = 0.0;
+                }
+            }
+            break;
+        case SLIP:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, j, 0) = U(i, j, 1);
+                    V(i, j, 0) = V(i, j, 1);
+                    W(i, j, 0) = 0.0;
+                }
+            }
+            break;
+        case OUTFLOW:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, j, 0) = U(i, j, 1);
+                    V(i, j, 0) = V(i, j, 1);
+                    W(i, j, 0) = W(i, j, 1);
+                }
+            }
+            break;
+        case PERIODIC:
+            break;
+        }
+    }
+
+    if (commIsBoundary(&s->comm, BACK)) {
+        switch (s->bcBack) {
+        case NOSLIP:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, j, kmaxLocal + 1) = -U(i, j, kmaxLocal);
+                    V(i, j, kmaxLocal + 1) = -V(i, j, kmaxLocal);
+                    W(i, j, kmaxLocal)     = 0.0;
+                }
+            }
+            break;
+        case SLIP:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal);
+                    V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal);
+                    W(i, j, kmaxLocal)     = 0.0;
+                }
+            }
+            break;
+        case OUTFLOW:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal);
+                    V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal);
+                    W(i, j, kmaxLocal)     = W(i, j, kmaxLocal - 1);
+                }
+            }
+            break;
+        case PERIODIC:
+            break;
+        }
+    }
+}
+
+void computeRHS(Discretization* s)
+{
+    int imaxLocal = s->comm.imaxLocal;
+    int jmaxLocal = s->comm.jmaxLocal;
+    int kmaxLocal = s->comm.kmaxLocal;
+
+    double idx = 1.0 / s->grid.dx;
+    double idy = 1.0 / s->grid.dy;
+    double idz = 1.0 / s->grid.dz;
+    double idt = 1.0 / s->dt;
+
+    double* rhs = s->rhs;
+    double* f   = s->f;
+    double* g   = s->g;
+    double* h   = s->h;
+
+    commShift(&s->comm, f, g, h);
+
+    for (int k = 1; k < kmaxLocal + 1; k++) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx +
+                                   (G(i, j, k) - G(i, j - 1, k)) * idy +
+                                   (H(i, j, k) - H(i, j, k - 1)) * idz) *
+                               idt;
+            }
+        }
+    }
+}
+
+void setSpecialBoundaryCondition(Discretization* s)
+{
+    int imaxLocal = s->comm.imaxLocal;
+    int jmaxLocal = s->comm.jmaxLocal;
+    int kmaxLocal = s->comm.kmaxLocal;
+
+    double* u = s->u;
+
+    if (strcmp(s->problem, "dcavity") == 0) {
+        if (commIsBoundary(&s->comm, TOP)) {
+            for (int k = 1; k < kmaxLocal; k++) {
+                for (int i = 1; i < imaxLocal; i++) {
+                    U(i, jmaxLocal + 1, k) = 2.0 - U(i, jmaxLocal, k);
+                }
+            }
+        }
+    } else if (strcmp(s->problem, "canal") == 0) {
+        if (commIsBoundary(&s->comm, LEFT)) {
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    U(0, j, k) = 2.0;
+                }
+            }
+        }
+    }
+}
+
+static double maxElement(Discretization* s, double* m)
+{
+    int size = (s->comm.imaxLocal + 2) * (s->comm.jmaxLocal + 2) *
+               (s->comm.kmaxLocal + 2);
+    double maxval = DBL_MIN;
+
+    for (int i = 0; i < size; i++) {
+        maxval = MAX(maxval, fabs(m[i]));
+    }
+    commReduction(&maxval, MAX);
+    return maxval;
+}
+
+void normalizePressure(Discretization* s)
+{
+    int imaxLocal = s->comm.imaxLocal;
+    int jmaxLocal = s->comm.jmaxLocal;
+    int kmaxLocal = s->comm.kmaxLocal;
+
+    double* p   = s->p;
+    double avgP = 0.0;
+
+    for (int k = 1; k < kmaxLocal + 1; k++) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                avgP += P(i, j, k);
+            }
+        }
+    }
+    commReduction(&avgP, SUM);
+    avgP /= (s->grid.imax * s->grid.jmax * s->grid.kmax);
+
+    for (int k = 1; k < kmaxLocal + 1; k++) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                P(i, j, k) = P(i, j, k) - avgP;
+            }
+        }
+    }
+}
+
+void computeTimestep(Discretization* s)
+{
+    double dt = s->dtBound;
+    double dx = s->grid.dx;
+    double dy = s->grid.dy;
+    double dz = s->grid.dz;
+
+    double umax = maxElement(s, s->u);
+    double vmax = maxElement(s, s->v);
+    double wmax = maxElement(s, s->w);
+
+    if (umax > 0) {
+        dt = (dt > dx / umax) ? dx / umax : dt;
+    }
+    if (vmax > 0) {
+        dt = (dt > dy / vmax) ? dy / vmax : dt;
+    }
+    if (wmax > 0) {
+        dt = (dt > dz / wmax) ? dz / wmax : dt;
+    }
+
+    s->dt = dt * s->tau;
+}
+
+
+
+void computeFG(Discretization* s)
+{
+    int imaxLocal = s->comm.imaxLocal;
+    int jmaxLocal = s->comm.jmaxLocal;
+    int kmaxLocal = s->comm.kmaxLocal;
+
+    double* u = s->u;
+    double* v = s->v;
+    double* w = s->w;
+    double* f = s->f;
+    double* g = s->g;
+    double* h = s->h;
+
+    double gx = s->gx;
+    double gy = s->gy;
+    double gz = s->gz;
+    double dt = s->dt;
+
+    double gamma     = s->gamma;
+    double inverseRe = 1.0 / s->re;
+    double inverseDx = 1.0 / s->grid.dx;
+    double inverseDy = 1.0 / s->grid.dy;
+    double inverseDz = 1.0 / s->grid.dz;
+    double du2dx, dv2dy, dw2dz;
+    double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz;
+    double du2dx2, du2dy2, du2dz2;
+    double dv2dx2, dv2dy2, dv2dz2;
+    double dw2dx2, dw2dy2, dw2dz2;
+
+    commExchange(&s->comm, u);
+    commExchange(&s->comm, v);
+    commExchange(&s->comm, w);
+
+    for (int k = 1; k < kmaxLocal + 1; k++) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                du2dx = inverseDx * 0.25 *
+                            ((U(i, j, k) + U(i + 1, j, k)) *
+                                    (U(i, j, k) + U(i + 1, j, k)) -
+                                (U(i, j, k) + U(i - 1, j, k)) *
+                                    (U(i, j, k) + U(i - 1, j, k))) +
+                        gamma * inverseDx * 0.25 *
+                            (fabs(U(i, j, k) + U(i + 1, j, k)) *
+                                    (U(i, j, k) - U(i + 1, j, k)) +
+                                fabs(U(i, j, k) + U(i - 1, j, k)) *
+                                    (U(i, j, k) - U(i - 1, j, k)));
+
+                duvdy = inverseDy * 0.25 *
+                            ((V(i, j, k) + V(i + 1, j, k)) *
+                                    (U(i, j, k) + U(i, j + 1, k)) -
+                                (V(i, j - 1, k) + V(i + 1, j - 1, k)) *
+                                    (U(i, j, k) + U(i, j - 1, k))) +
+                        gamma * inverseDy * 0.25 *
+                            (fabs(V(i, j, k) + V(i + 1, j, k)) *
+                                    (U(i, j, k) - U(i, j + 1, k)) +
+                                fabs(V(i, j - 1, k) + V(i + 1, j - 1, k)) *
+                                    (U(i, j, k) - U(i, j - 1, k)));
+
+                duwdz = inverseDz * 0.25 *
+                            ((W(i, j, k) + W(i + 1, j, k)) *
+                                    (U(i, j, k) + U(i, j, k + 1)) -
+                                (W(i, j, k - 1) + W(i + 1, j, k - 1)) *
+                                    (U(i, j, k) + U(i, j, k - 1))) +
+                        gamma * inverseDz * 0.25 *
+                            (fabs(W(i, j, k) + W(i + 1, j, k)) *
+                                    (U(i, j, k) - U(i, j, k + 1)) +
+                                fabs(W(i, j, k - 1) + W(i + 1, j, k - 1)) *
+                                    (U(i, j, k) - U(i, j, k - 1)));
+
+                du2dx2 = inverseDx * inverseDx *
+                         (U(i + 1, j, k) - 2.0 * U(i, j, k) + U(i - 1, j, k));
+                du2dy2 = inverseDy * inverseDy *
+                         (U(i, j + 1, k) - 2.0 * U(i, j, k) + U(i, j - 1, k));
+                du2dz2 = inverseDz * inverseDz *
+                         (U(i, j, k + 1) - 2.0 * U(i, j, k) + U(i, j, k - 1));
+                F(i, j, k) = U(i, j, k) + dt * (inverseRe * (du2dx2 + du2dy2 + du2dz2) -
+                                                   du2dx - duvdy - duwdz + gx);
+
+                duvdx = inverseDx * 0.25 *
+                            ((U(i, j, k) + U(i, j + 1, k)) *
+                                    (V(i, j, k) + V(i + 1, j, k)) -
+                                (U(i - 1, j, k) + U(i - 1, j + 1, k)) *
+                                    (V(i, j, k) + V(i - 1, j, k))) +
+                        gamma * inverseDx * 0.25 *
+                            (fabs(U(i, j, k) + U(i, j + 1, k)) *
+                                    (V(i, j, k) - V(i + 1, j, k)) +
+                                fabs(U(i - 1, j, k) + U(i - 1, j + 1, k)) *
+                                    (V(i, j, k) - V(i - 1, j, k)));
+
+                dv2dy = inverseDy * 0.25 *
+                            ((V(i, j, k) + V(i, j + 1, k)) *
+                                    (V(i, j, k) + V(i, j + 1, k)) -
+                                (V(i, j, k) + V(i, j - 1, k)) *
+                                    (V(i, j, k) + V(i, j - 1, k))) +
+                        gamma * inverseDy * 0.25 *
+                            (fabs(V(i, j, k) + V(i, j + 1, k)) *
+                                    (V(i, j, k) - V(i, j + 1, k)) +
+                                fabs(V(i, j, k) + V(i, j - 1, k)) *
+                                    (V(i, j, k) - V(i, j - 1, k)));
+
+                dvwdz = inverseDz * 0.25 *
+                            ((W(i, j, k) + W(i, j + 1, k)) *
+                                    (V(i, j, k) + V(i, j, k + 1)) -
+                                (W(i, j, k - 1) + W(i, j + 1, k - 1)) *
+                                    (V(i, j, k) + V(i, j, k + 1))) +
+                        gamma * inverseDz * 0.25 *
+                            (fabs(W(i, j, k) + W(i, j + 1, k)) *
+                                    (V(i, j, k) - V(i, j, k + 1)) +
+                                fabs(W(i, j, k - 1) + W(i, j + 1, k - 1)) *
+                                    (V(i, j, k) - V(i, j, k + 1)));
+
+                dv2dx2 = inverseDx * inverseDx *
+                         (V(i + 1, j, k) - 2.0 * V(i, j, k) + V(i - 1, j, k));
+                dv2dy2 = inverseDy * inverseDy *
+                         (V(i, j + 1, k) - 2.0 * V(i, j, k) + V(i, j - 1, k));
+                dv2dz2 = inverseDz * inverseDz *
+                         (V(i, j, k + 1) - 2.0 * V(i, j, k) + V(i, j, k - 1));
+                G(i, j, k) = V(i, j, k) + dt * (inverseRe * (dv2dx2 + dv2dy2 + dv2dz2) -
+                                                   duvdx - dv2dy - dvwdz + gy);
+
+                duwdx = inverseDx * 0.25 *
+                            ((U(i, j, k) + U(i, j, k + 1)) *
+                                    (W(i, j, k) + W(i + 1, j, k)) -
+                                (U(i - 1, j, k) + U(i - 1, j, k + 1)) *
+                                    (W(i, j, k) + W(i - 1, j, k))) +
+                        gamma * inverseDx * 0.25 *
+                            (fabs(U(i, j, k) + U(i, j, k + 1)) *
+                                    (W(i, j, k) - W(i + 1, j, k)) +
+                                fabs(U(i - 1, j, k) + U(i - 1, j, k + 1)) *
+                                    (W(i, j, k) - W(i - 1, j, k)));
+
+                dvwdy = inverseDy * 0.25 *
+                            ((V(i, j, k) + V(i, j, k + 1)) *
+                                    (W(i, j, k) + W(i, j + 1, k)) -
+                                (V(i, j - 1, k + 1) + V(i, j - 1, k)) *
+                                    (W(i, j, k) + W(i, j - 1, k))) +
+                        gamma * inverseDy * 0.25 *
+                            (fabs(V(i, j, k) + V(i, j, k + 1)) *
+                                    (W(i, j, k) - W(i, j + 1, k)) +
+                                fabs(V(i, j - 1, k + 1) + V(i, j - 1, k)) *
+                                    (W(i, j, k) - W(i, j - 1, k)));
+
+                dw2dz = inverseDz * 0.25 *
+                            ((W(i, j, k) + W(i, j, k + 1)) *
+                                    (W(i, j, k) + W(i, j, k + 1)) -
+                                (W(i, j, k) + W(i, j, k - 1)) *
+                                    (W(i, j, k) + W(i, j, k - 1))) +
+                        gamma * inverseDz * 0.25 *
+                            (fabs(W(i, j, k) + W(i, j, k + 1)) *
+                                    (W(i, j, k) - W(i, j, k + 1)) +
+                                fabs(W(i, j, k) + W(i, j, k - 1)) *
+                                    (W(i, j, k) - W(i, j, k - 1)));
+
+                dw2dx2 = inverseDx * inverseDx *
+                         (W(i + 1, j, k) - 2.0 * W(i, j, k) + W(i - 1, j, k));
+                dw2dy2 = inverseDy * inverseDy *
+                         (W(i, j + 1, k) - 2.0 * W(i, j, k) + W(i, j - 1, k));
+                dw2dz2 = inverseDz * inverseDz *
+                         (W(i, j, k + 1) - 2.0 * W(i, j, k) + W(i, j, k - 1));
+                H(i, j, k) = W(i, j, k) + dt * (inverseRe * (dw2dx2 + dw2dy2 + dw2dz2) -
+                                                   duwdx - dvwdy - dw2dz + gz);
+            }
+        }
+    }
+
+    /* ----------------------------- boundary of F ---------------------------
+     */
+    if (commIsBoundary(&s->comm, LEFT)) {
+        for (int k = 1; k < kmaxLocal + 1; k++) {
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                F(0, j, k) = U(0, j, k);
+            }
+        }
+    }
+
+    if (commIsBoundary(&s->comm, RIGHT)) {
+        for (int k = 1; k < kmaxLocal + 1; k++) {
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                F(imaxLocal, j, k) = U(imaxLocal, j, k);
+            }
+        }
+    }
+
+    /* ----------------------------- boundary of G ---------------------------
+     */
+    if (commIsBoundary(&s->comm, BOTTOM)) {
+        for (int k = 1; k < kmaxLocal + 1; k++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                G(i, 0, k) = V(i, 0, k);
+            }
+        }
+    }
+
+    if (commIsBoundary(&s->comm, TOP)) {
+        for (int k = 1; k < kmaxLocal + 1; k++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                G(i, jmaxLocal, k) = V(i, jmaxLocal, k);
+            }
+        }
+    }
+
+    /* ----------------------------- boundary of H ---------------------------
+     */
+    if (commIsBoundary(&s->comm, FRONT)) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                H(i, j, 0) = W(i, j, 0);
+            }
+        }
+    }
+
+    if (commIsBoundary(&s->comm, BACK)) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                H(i, j, kmaxLocal) = W(i, j, kmaxLocal);
+            }
+        }
+    }
+}
+
+void adaptUV(Discretization* s)
+{
+    int imaxLocal = s->comm.imaxLocal;
+    int jmaxLocal = s->comm.jmaxLocal;
+    int kmaxLocal = s->comm.kmaxLocal;
+
+    double* p = s->p;
+    double* u = s->u;
+    double* v = s->v;
+    double* w = s->w;
+    double* f = s->f;
+    double* g = s->g;
+    double* h = s->h;
+
+    double factorX = s->dt / s->grid.dx;
+    double factorY = s->dt / s->grid.dy;
+    double factorZ = s->dt / s->grid.dz;
+
+    for (int k = 1; k < kmaxLocal + 1; k++) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                U(i, j, k) = F(i, j, k) - (P(i + 1, j, k) - P(i, j, k)) * factorX;
+                V(i, j, k) = G(i, j, k) - (P(i, j + 1, k) - P(i, j, k)) * factorY;
+                W(i, j, k) = H(i, j, k) - (P(i, j, k + 1) - P(i, j, k)) * factorZ;
+            }
+        }
+    }
+}

BasicSolver/3D-mpi/src/discretization.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#ifndef __DISCRETIZATION_H_
+#define __DISCRETIZATION_H_
+
+#include "grid.h"
+#include "parameter.h"
+#include "comm.h"
+
+enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
+
+typedef struct {
+    /* geometry and grid information */
+    Grid grid;
+    /* arrays */
+    double *p, *rhs;
+    double *f, *g, *h;
+    double *u, *v, *w;
+    /* parameters */
+    double eps, omega;
+    double re, tau, gamma;
+    double gx, gy, gz;
+    /* time stepping */
+    int itermax;
+    double dt, te;
+    double dtBound;
+    char* problem;
+    int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
+    Comm comm;
+} Discretization;
+
+extern void initDiscretization(Discretization*, Parameter*);
+extern void computeRHS(Discretization*);
+extern void normalizePressure(Discretization*);
+extern void computeTimestep(Discretization*);
+extern void setBoundaryConditions(Discretization*);
+extern void setSpecialBoundaryCondition(Discretization*);
+extern void computeFG(Discretization*);
+extern void adaptUV(Discretization*);
+#endif

BasicSolver/3D-mpi/src/main.c

@@ -9,7 +9,7 @@
 #include <unistd.h>
 
 #include "allocate.h"
-#include "comm.h"
+#include "discretization.h"
 #include "parameter.h"
 #include "progress.h"
 #include "solver.h"
@@ -21,9 +21,12 @@ int main(int argc, char** argv)
     double timeStart, timeStop;
     Parameter p;
     Solver s;
+    Discretization d;
 
-    commInit(&s.comm, argc, argv);
+    commInit(&d.comm, argc, argv);
     initParameter(&p);
+    FILE* fp;
+    if (commIsMaster(&d.comm)) fp = initResidualWriter();
 
     if (argc != 2) {
         printf("Usage: %s <configFile>\n", argv[0]);
@@ -31,33 +34,44 @@ int main(int argc, char** argv)
     }
 
     readParameter(&p, argv[1]);
-    commPartition(&s.comm, p.kmax, p.jmax, p.imax);
-    if (commIsMaster(&s.comm)) {
+    commPartition(&d.comm, p.kmax, p.jmax, p.imax);
+
+    if (commIsMaster(&d.comm)) {
         printParameter(&p);
     }
-    initSolver(&s, &p);
+    
+    initDiscretization(&d, &p);
+    initSolver(&s, &d, &p);
+
 #ifndef VERBOSE
-    initProgress(s.te);
+    initProgress(d.te);
 #endif
 
-    double tau = s.tau;
-    double te  = s.te;
+    double tau = d.tau;
+    double te  = d.te;
     double t   = 0.0;
+    int nt     = 0;
+    double res = 0.0;
 
     timeStart = getTimeStamp();
     while (t <= te) {
-        if (tau > 0.0) computeTimestep(&s);
-        setBoundaryConditions(&s);
-        setSpecialBoundaryCondition(&s);
-        computeFG(&s);
-        computeRHS(&s);
-        solve(&s);
-        adaptUV(&s);
-        t += s.dt;
+        if (tau > 0.0) computeTimestep(&d);
+        setBoundaryConditions(&d);
+        setSpecialBoundaryCondition(&d);
+        computeFG(&d);
+        computeRHS(&d);
+        if (nt % 100 == 0) normalizePressure(&d);
+        res = solve(&s, d.p, d.rhs);
+        adaptUV(&d);
+
+        if (commIsMaster(&d.comm)) writeResidual(fp, t, res);
+
+        t += d.dt;
+        nt++;
 
 #ifdef VERBOSE
-        if (commIsMaster(&s.comm)) {
-            printf("TIME %f , TIMESTEP %f\n", t, s.dt);
+        if (commIsMaster(s.comm)) {
+            printf("TIME %f , TIMESTEP %f\n", t, d.dt);
         }
 #else
         printProgress(t);
@@ -67,7 +81,7 @@ int main(int argc, char** argv)
 #ifndef VERBOSE
     stopProgress();
 #endif
-    if (commIsMaster(&s.comm)) {
+    if (commIsMaster(s.comm)) {
         printf("Solution took %.2fs\n", timeStop - timeStart);
     }
 
@@ -75,14 +89,16 @@ int main(int argc, char** argv)
 #ifdef _VTK_WRITER_MPI
     VtkOptions opts = { .grid = s.grid, .comm = s.comm };
     vtkOpen(&opts, s.problem);
-    vtkScalar(&opts, "pressure", s.p);
-    vtkVector(&opts, "velocity", (VtkVector) { s.u, s.v, s.w });
+    vtkScalar(&opts, "pressure", d.p);
+    vtkVector(&opts, "velocity", (VtkVector) { d.u, d.v, d.w });
     vtkClose(&opts);
 #else
+    if (commIsMaster(&d.comm)) fclose(fp);
+
     double *pg, *ug, *vg, *wg;
 
-    if (commIsMaster(&s.comm)) {
-        size_t bytesize = s.grid.imax * s.grid.jmax * s.grid.kmax * sizeof(double);
+    if (commIsMaster(s.comm)) {
+        size_t bytesize = s.grid->imax * s.grid->jmax * s.grid->kmax * sizeof(double);
 
         pg = allocate(64, bytesize);
         ug = allocate(64, bytesize);
@@ -90,34 +106,35 @@ int main(int argc, char** argv)
         wg = allocate(64, bytesize);
     }
 
-    commCollectResult(&s.comm,
+    commCollectResult(s.comm,
         ug,
         vg,
         wg,
         pg,
-        s.u,
-        s.v,
-        s.w,
-        s.p,
-        s.grid.kmax,
-        s.grid.jmax,
-        s.grid.imax);
+        d.u,
+        d.v,
+        d.w,
+        d.p,
+        s.grid->kmax,
+        s.grid->jmax,
+        s.grid->imax);
 
-    if (commIsMaster(&s.comm)) {
+    if (commIsMaster(s.comm)) {
         VtkOptions opts = { .grid = s.grid };
         vtkOpen(&opts, s.problem);
         vtkScalar(&opts, "pressure", pg);
         vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg });
         vtkClose(&opts);
     }
+
 #endif
 
     timeStop = getTimeStamp();
 
-    if (commIsMaster(&s.comm)) {
+    if (commIsMaster(s.comm)) {
         printf("Result output took %.2fs\n", timeStop - timeStart);
     }
 
-    commFinalize(&s.comm);
+    commFinalize(s.comm);
     return EXIT_SUCCESS;
 }

BasicSolver/3D-mpi/src/parameter.c

@@ -26,6 +26,10 @@ void initParameter(Parameter* param)
     param->re          = 100.0;
     param->gamma       = 0.9;
     param->tau         = 0.5;
+    param->levels     = 5;
+    param->presmooth  = 5;
+    param->postsmooth = 5;
+
 }
 
 void readParameter(Parameter* param, const char* filename)
@@ -65,6 +69,9 @@ void readParameter(Parameter* param, const char* filename)
             PARSE_INT(jmax);
             PARSE_INT(kmax);
             PARSE_INT(itermax);
+            PARSE_INT(levels);
+            PARSE_INT(presmooth);
+            PARSE_INT(postsmooth);
             PARSE_REAL(eps);
             PARSE_REAL(omg);
             PARSE_REAL(re);

BasicSolver/3D-mpi/src/parameter.h

@@ -18,6 +18,7 @@ typedef struct {
     char* name;
     int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
     double u_init, v_init, w_init, p_init;
+    int levels, presmooth, postsmooth;
 } Parameter;
 
 void initParameter(Parameter*);

BasicSolver/3D-mpi/src/progress.c

@@ -4,12 +4,12 @@
  * Use of this source code is governed by a MIT style
  * license that can be found in the LICENSE file.
  */
+#include "progress.h"
 #include <math.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
-#include "progress.h"
-
 static double _end;
 static int _current;
 
@@ -48,3 +48,18 @@ void stopProgress()
     printf("\n");
     fflush(stdout);
 }
+
+FILE* initResidualWriter()
+{
+    FILE* fp;
+    fp = fopen("residual.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return fp;
+}
+
+void writeResidual(FILE* fp, double ts, double res) { fprintf(fp, "%f, %f\n", ts, res); }

BasicSolver/3D-mpi/src/progress.h

@@ -4,11 +4,14 @@
  * Use of this source code is governed by a MIT-style
  * license that can be found in the LICENSE file.
  */
+
+#include <stdio.h>
+
 #ifndef __PROGRESS_H_
 #define __PROGRESS_H_
-
 extern void initProgress(double);
 extern void printProgress(double);
 extern void stopProgress(void);
-
+extern FILE* initResidualWriter(void);
+extern void writeResidual(FILE*, double, double);
 #endif

BasicSolver/3D-mpi/src/solver-mg.c

@@ -0,0 +1,403 @@
+/*
+ * Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "allocate.h"
+#include "solver.h"
+#include "util.h"
+
+#define FINEST_LEVEL   0
+#define COARSEST_LEVEL (s->levels - 1)
+#define S(i, j, k)                                                                       \
+    s[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define E(i, j, k)                                                                       \
+    e[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define R(i, j, k)                                                                       \
+    r[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define OLD(i, j, k)                                                                     \
+    old[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+
+static void restrictMG(Solver* s, int level, Comm* comm)
+{
+    double* r   = s->r[level + 1];
+    double* old = s->r[level];
+
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+    int kmaxLocal = comm->kmaxLocal;
+
+    commExchange(comm, old);
+
+    for (int k = 1; k < comm->kmaxLocal + 1; k++) {
+        for (int j = 1; j < comm->jmaxLocal + 1; j++) {
+            for (int i = 1; i < comm->imaxLocal + 1; ++i) {
+                R(i, j, k) = (OLD(2 * i - 1, 2 * j - 1, 2 * k) +
+                                 OLD(2 * i, 2 * j - 1, 2 * k) * 2 +
+                                 OLD(2 * i + 1, 2 * j - 1, 2 * k) +
+                                 OLD(2 * i - 1, 2 * j, 2 * k) * 2 +
+                                 OLD(2 * i, 2 * j, 2 * k) * 8 +
+                                 OLD(2 * i + 1, 2 * j, 2 * k) * 2 +
+                                 OLD(2 * i - 1, 2 * j + 1, 2 * k) +
+                                 OLD(2 * i, 2 * j + 1, 2 * k) * 2 +
+                                 OLD(2 * i + 1, 2 * j + 1, 2 * k) +
+
+                                 OLD(2 * i - 1, 2 * j - 1, 2 * k - 1) +
+                                 OLD(2 * i, 2 * j - 1, 2 * k - 1) * 2 +
+                                 OLD(2 * i + 1, 2 * j - 1, 2 * k - 1) +
+                                 OLD(2 * i - 1, 2 * j, 2 * k - 1) * 2 +
+                                 OLD(2 * i, 2 * j, 2 * k - 1) * 4 +
+                                 OLD(2 * i + 1, 2 * j, 2 * k - 1) * 2 +
+                                 OLD(2 * i - 1, 2 * j + 1, 2 * k - 1) +
+                                 OLD(2 * i, 2 * j + 1, 2 * k - 1) * 2 +
+                                 OLD(2 * i + 1, 2 * j + 1, 2 * k - 1) +
+
+                                 OLD(2 * i - 1, 2 * j - 1, 2 * k + 1) +
+                                 OLD(2 * i, 2 * j - 1, 2 * k + 1) * 2 +
+                                 OLD(2 * i + 1, 2 * j - 1, 2 * k + 1) +
+                                 OLD(2 * i - 1, 2 * j, 2 * k + 1) * 2 +
+                                 OLD(2 * i, 2 * j, 2 * k + 1) * 4 +
+                                 OLD(2 * i + 1, 2 * j, 2 * k + 1) * 2 +
+                                 OLD(2 * i - 1, 2 * j + 1, 2 * k + 1) +
+                                 OLD(2 * i, 2 * j + 1, 2 * k + 1) * 2 +
+                                 OLD(2 * i + 1, 2 * j + 1, 2 * k + 1)) /
+                             64.0;
+            }
+        }
+    }
+}
+
+static void prolongate(Solver* s, int level, Comm* comm)
+{
+    double* old = s->r[level + 1];
+    double* e   = s->r[level];
+
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+    int kmaxLocal = comm->kmaxLocal;
+
+    for (int k = 2; k < kmaxLocal + 1; k += 2) {
+        for (int j = 2; j < jmaxLocal + 1; j += 2) {
+            for (int i = 2; i < imaxLocal + 1; i += 2) {
+                E(i, j, k) = OLD(i / 2, j / 2, k / 2);
+            }
+        }
+    }
+}
+
+static void correct(Solver* s, double* p, int level, Comm* comm)
+{
+    double* e = s->e[level];
+
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+    int kmaxLocal = comm->kmaxLocal;
+
+    for (int k = 1; k < kmaxLocal + 1; ++k) {
+        for (int j = 1; j < jmaxLocal + 1; ++j) {
+            for (int i = 1; i < imaxLocal + 1; ++i) {
+                P(i, j, k) += E(i, j, k);
+            }
+        }
+    }
+}
+
+static void setBoundaryCondition(
+    Solver* s, double* p, int imaxLocal, int jmaxLocal, int kmaxLocal)
+{
+#ifdef _MPI
+    if (commIsBoundary(s->comm, FRONT)) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                P(i, j, 0) = P(i, j, 1);
+            }
+        }
+    }
+
+    if (commIsBoundary(s->comm, BACK)) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal);
+            }
+        }
+    }
+
+    if (commIsBoundary(s->comm, BOTTOM)) {
+        for (int k = 1; k < kmaxLocal + 1; k++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                P(i, 0, k) = P(i, 1, k);
+            }
+        }
+    }
+
+    if (commIsBoundary(s->comm, TOP)) {
+        for (int k = 1; k < kmaxLocal + 1; k++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k);
+            }
+        }
+    }
+
+    if (commIsBoundary(s->comm, LEFT)) {
+        for (int k = 1; k < kmaxLocal + 1; k++) {
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                P(0, j, k) = P(1, j, k);
+            }
+        }
+    }
+
+    if (commIsBoundary(s->comm, RIGHT)) {
+        for (int k = 1; k < kmaxLocal + 1; k++) {
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                P(imaxLocal + 1, j, k) = P(imaxLocal, j, k);
+            }
+        }
+    }
+#else
+    for (int j = 1; j < jmaxLocal + 1; j++) {
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            P(i, j, 0)             = P(i, j, 1);
+            P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal);
+        }
+    }
+
+    for (int k = 1; k < kmaxLocal + 1; k++) {
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            P(i, 0, k)             = P(i, 1, k);
+            P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k);
+        }
+    }
+
+    for (int k = 1; k < kmaxLocal + 1; k++) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            P(0, j, k)             = P(1, j, k);
+            P(imaxLocal + 1, j, k) = P(imaxLocal, j, k);
+        }
+    }
+#endif
+}
+
+static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm)
+{
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+    int kmaxLocal = comm->kmaxLocal;
+
+    double eps    = s->eps;
+    int itermax   = s->itermax;
+    double dx2    = s->grid->dx * s->grid->dx;
+    double dy2    = s->grid->dy * s->grid->dy;
+    double dz2    = s->grid->dz * s->grid->dz;
+    double idx2   = 1.0 / dx2;
+    double idy2   = 1.0 / dy2;
+    double idz2   = 1.0 / dz2;
+    double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
+                    (dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
+    double* r    = s->r[level];
+    double epssq = eps * eps;
+    int it       = 0;
+    int pass, ksw, jsw, isw;
+    double res = 1.0;
+
+    ksw = 1;
+
+    for (pass = 0; pass < 2; pass++) {
+        jsw = ksw;
+
+        commExchange(comm, p);
+
+        for (int k = 1; k < kmaxLocal + 1; k++) {
+            isw = jsw;
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                for (int i = isw; i < imaxLocal + 1; i += 2) {
+
+                    P(i, j, k) -=
+                        factor *
+                        (RHS(i, j, k) -
+                            ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
+                                (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
+                                    idy2 +
+                                (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
+                                    idz2));
+                }
+                isw = 3 - isw;
+            }
+            jsw = 3 - jsw;
+        }
+        ksw = 3 - ksw;
+    }
+}
+
+static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm)
+{
+
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+    int kmaxLocal = comm->kmaxLocal;
+
+    double eps    = s->eps;
+    int itermax   = s->itermax;
+    double dx2    = s->grid->dx * s->grid->dx;
+    double dy2    = s->grid->dy * s->grid->dy;
+    double dz2    = s->grid->dz * s->grid->dz;
+    double idx2   = 1.0 / dx2;
+    double idy2   = 1.0 / dy2;
+    double idz2   = 1.0 / dz2;
+    double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
+                    (dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
+    double* r    = s->r[level];
+    double epssq = eps * eps;
+    int it       = 0;
+    int pass, ksw, jsw, isw;
+    double res = 1.0;
+
+    ksw = 1;
+
+    for (pass = 0; pass < 2; pass++) {
+        jsw = ksw;
+
+        commExchange(comm, p);
+
+        for (int k = 1; k < kmaxLocal + 1; k++) {
+            isw = jsw;
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                for (int i = isw; i < imaxLocal + 1; i += 2) {
+
+                    R(i,
+                        j,
+                        k) = (RHS(i, j, k) -
+                              ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) *
+                                      idx2 +
+                                  (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
+                                      idy2 +
+                                  (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
+                                      idz2));
+
+                    res += (R(i, j, k) * R(i, j, k));
+                }
+                isw = 3 - isw;
+            }
+            jsw = 3 - jsw;
+        }
+        ksw = 3 - ksw;
+    }
+
+    commReduction(&res, SUM);
+
+    res = res / (double)(imaxLocal * jmaxLocal * kmaxLocal);
+#ifdef DEBUG
+    if (commIsMaster(s->comm)) {
+        printf("%d Residuum: %e\n", it, res);
+    }
+#endif
+    return res;
+}
+
+static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm)
+{
+    int imaxLocal = comm->imaxLocal;
+    int jmaxLocal = comm->jmaxLocal;
+    int kmaxLocal = comm->kmaxLocal;
+
+    double res = 0.0;
+
+    // coarsest level
+    if (level == COARSEST_LEVEL) {
+        for (int i = 0; i < 5; i++) {
+            smooth(s, p, rhs, level, comm);
+        }
+        return res;
+    }
+
+    // pre-smoothing
+    for (int i = 0; i < s->presmooth; i++) {
+        smooth(s, p, rhs, level, comm);
+        if (level == FINEST_LEVEL)
+            setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal);
+    }
+
+    res = calculateResidual(s, p, rhs, level, comm);
+
+    // restrict
+    restrictMG(s, level, comm);
+
+    Comm newcomm;
+    commUpdateDatatypes(s->comm,
+        &newcomm,
+        imaxLocal,
+        jmaxLocal,
+        kmaxLocal);
+
+    // MGSolver on residual and error.
+    multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm);
+
+    commFreeCommunicator(&newcomm);
+
+    // prolongate
+    prolongate(s, level, comm);
+
+    // correct p on finer level using residual
+    correct(s, p, level, comm);
+    if (level == FINEST_LEVEL)
+        setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal);
+
+    // post-smoothing
+    for (int i = 0; i < s->postsmooth; i++) {
+        smooth(s, p, rhs, level, comm);
+        if (level == FINEST_LEVEL)
+            setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal);
+    }
+
+    return res;
+}
+
+void initSolver(Solver* s, Discretization* d, Parameter* p)
+{
+    s->eps        = p->eps;
+    s->omega      = p->omg;
+    s->itermax    = p->itermax;
+    s->levels     = p->levels;
+    s->grid       = &d->grid;
+    s->presmooth  = p->presmooth;
+    s->postsmooth = p->postsmooth;
+    s->comm       = &d->comm;
+    s->problem    = p->name;
+
+    int imax   = s->grid->imax;
+    int jmax   = s->grid->jmax;
+    int kmax   = s->grid->kmax;
+    int levels = s->levels;
+    if (commIsMaster(s->comm)) printf("Using Multigrid solver with %d levels\n", levels);
+
+    s->r = malloc(levels * sizeof(double*));
+    s->e = malloc(levels * sizeof(double*));
+
+    size_t size = (imax + 2) * (jmax + 2) * (kmax + 2);
+
+    for (int j = 0; j < levels; j++) {
+        s->r[j] = allocate(64, size * sizeof(double));
+        s->e[j] = allocate(64, size * sizeof(double));
+
+        for (size_t i = 0; i < size; i++) {
+            s->r[j][i] = 0.0;
+            s->e[j][i] = 0.0;
+        }
+    }
+}
+
+double solve(Solver* s, double* p, double* rhs)
+{
+    double res = multiGrid(s, p, rhs, 0, s->comm);
+
+#ifdef VERBOSE
+    if (commIsMaster(s->comm)) {
+        printf("Residuum: %.6f\n", res);
+    }
+#endif
+
+    return res;
+}

BasicSolver/3D-mpi/src/solver-rb.c

@@ -0,0 +1,175 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "allocate.h"
+#include "comm.h"
+#include "parameter.h"
+#include "solver.h"
+#include "util.h"
+
+void initSolver(Solver* s, Discretization* d, Parameter* p)
+{
+    s->eps        = p->eps;
+    s->omega      = p->omg;
+    s->itermax    = p->itermax;
+    s->levels     = p->levels;
+    s->grid       = &d->grid;
+    s->presmooth  = p->presmooth;
+    s->postsmooth = p->postsmooth;
+    s->comm       = &d->comm;
+    s->problem    = p->name;
+}
+
+double solve(Solver* s, double* p, double* rhs)
+{
+    int imaxLocal = s->comm->imaxLocal;
+    int jmaxLocal = s->comm->jmaxLocal;
+    int kmaxLocal = s->comm->kmaxLocal;
+
+    int imax = s->grid->imax;
+    int jmax = s->grid->jmax;
+    int kmax = s->grid->kmax;
+
+    double eps  = s->eps;
+    int itermax = s->itermax;
+    double dx2  = s->grid->dx * s->grid->dx;
+    double dy2  = s->grid->dy * s->grid->dy;
+    double dz2  = s->grid->dz * s->grid->dz;
+    double idx2 = 1.0 / dx2;
+    double idy2 = 1.0 / dy2;
+    double idz2 = 1.0 / dz2;
+
+    double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
+                    (dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
+    double epssq = eps * eps;
+    int it       = 0;
+    double res   = 1.0;
+    int pass, ksw, jsw, isw;
+
+    while ((res >= epssq) && (it < itermax)) {
+        ksw = 1;
+
+        for (pass = 0; pass < 2; pass++) {
+            jsw = ksw;
+            commExchange(s->comm, p);
+
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                isw = jsw;
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    for (int i = isw; i < imaxLocal + 1; i += 2) {
+
+                        double r =
+                            RHS(i, j, k) -
+                            ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
+                                (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
+                                    idy2 +
+                                (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
+                                    idz2);
+
+                        P(i, j, k) -= (factor * r);
+                        res += (r * r);
+                    }
+                    isw = 3 - isw;
+                }
+                jsw = 3 - jsw;
+            }
+            ksw = 3 - ksw;
+        }
+#ifdef _MPI
+        if (commIsBoundary(s->comm, FRONT)) {
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    P(i, j, 0) = P(i, j, 1);
+                }
+            }
+        }
+
+        if (commIsBoundary(s->comm, BACK)) {
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal);
+                }
+            }
+        }
+
+        if (commIsBoundary(s->comm, BOTTOM)) {
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    P(i, 0, k) = P(i, 1, k);
+                }
+            }
+        }
+
+        if (commIsBoundary(s->comm, TOP)) {
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int i = 1; i < imaxLocal + 1; i++) {
+                    P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k);
+                }
+            }
+        }
+
+        if (commIsBoundary(s->comm, LEFT)) {
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    P(0, j, k) = P(1, j, k);
+                }
+            }
+        }
+
+        if (commIsBoundary(s->comm, RIGHT)) {
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    P(imaxLocal + 1, j, k) = P(imaxLocal, j, k);
+                }
+            }
+        }
+#else
+        for (int j = 1; j < jmax + 1; j++) {
+            for (int i = 1; i < imax + 1; i++) {
+                P(i, j, 0)        = P(i, j, 1);
+                P(i, j, kmax + 1) = P(i, j, kmax);
+            }
+        }
+
+        for (int k = 1; k < kmax + 1; k++) {
+            for (int i = 1; i < imax + 1; i++) {
+                P(i, 0, k)        = P(i, 1, k);
+                P(i, jmax + 1, k) = P(i, jmax, k);
+            }
+        }
+
+        for (int k = 1; k < kmax + 1; k++) {
+            for (int j = 1; j < jmax + 1; j++) {
+                P(0, j, k)        = P(1, j, k);
+                P(imax + 1, j, k) = P(imax, j, k);
+            }
+        }
+#endif
+        commReduction(&res, SUM);
+        res = res / (double)(imax * jmax * kmax);
+#ifdef DEBUG
+        if (commIsMaster(&s->comm)) {
+            printf("%d Residuum: %e\n", it, res);
+        }
+#endif
+        commExchange(s->comm, p);
+        it++;
+    }
+
+#ifdef VERBOSE
+    if (commIsMaster(s->comm)) {
+        printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
+    }
+#endif
+
+return res;
+}

BasicSolver/3D-mpi/src/solver.h

@@ -9,12 +9,11 @@
 #include "comm.h"
 #include "grid.h"
 #include "parameter.h"
-
-enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
+#include "discretization.h"
 
 typedef struct {
     /* geometry and grid information */
-    Grid grid;
+    Grid* grid;
     /* arrays */
     double *p, *rhs;
     double *f, *g, *h;
@@ -30,16 +29,11 @@ typedef struct {
     char* problem;
     int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
     /* communication */
-    Comm comm;
+    double **r, **e;
+    int levels, presmooth, postsmooth;
+    Comm* comm;
 } Solver;
 
-extern void initSolver(Solver*, Parameter*);
-extern void computeRHS(Solver*);
-extern void solve(Solver*);
-extern void normalizePressure(Solver*);
-extern void computeTimestep(Solver*);
-extern void setBoundaryConditions(Solver*);
-extern void setSpecialBoundaryCondition(Solver*);
-extern void computeFG(Solver*);
-extern void adaptUV(Solver*);
+extern double solve(Solver* , double* , double* );
+extern void initSolver(Solver*, Discretization*, Parameter*);
 #endif

BasicSolver/3D-mpi/src/test.c

@@ -14,10 +14,10 @@
 
 void testInit(Solver* s)
 {
-    int imaxLocal = s->comm.imaxLocal;
-    int jmaxLocal = s->comm.jmaxLocal;
-    int kmaxLocal = s->comm.kmaxLocal;
-    int myrank    = s->comm.rank;
+    int imaxLocal = s->comm->imaxLocal;
+    int jmaxLocal = s->comm->jmaxLocal;
+    int kmaxLocal = s->comm->kmaxLocal;
+    int myrank    = s->comm->rank;
     double* p     = s->p;
     double* f     = s->f;
     double* g     = s->g;
@@ -76,11 +76,11 @@ static char* direction2String(Direction dir)
 
 static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir)
 {
-    int imaxLocal = s->comm.imaxLocal;
-    int jmaxLocal = s->comm.jmaxLocal;
-    int kmaxLocal = s->comm.kmaxLocal;
+    int imaxLocal = s->comm->imaxLocal;
+    int jmaxLocal = s->comm->jmaxLocal;
+    int kmaxLocal = s->comm->kmaxLocal;
     char filename[50];
-    snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm.rank);
+    snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm->rank);
     FILE* fh = fopen(filename, "w");
 
     for (int y = 0; y < ymax; y++) {
@@ -116,9 +116,9 @@ static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir)
 
 void testPrintHalo(Solver* s, double* a)
 {
-    int imaxLocal = s->comm.imaxLocal;
-    int jmaxLocal = s->comm.jmaxLocal;
-    int kmaxLocal = s->comm.kmaxLocal;
+    int imaxLocal = s->comm->imaxLocal;
+    int jmaxLocal = s->comm->jmaxLocal;
+    int kmaxLocal = s->comm->kmaxLocal;
 
     printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, BOTTOM);
     printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, TOP);

BasicSolver/3D-mpi/src/util.h

@@ -19,4 +19,13 @@
 #define ABS(a) ((a) >= 0 ? (a) : -(a))
 #endif
 
+#define P(i, j, k)   p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define F(i, j, k)   f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define G(i, j, k)   g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define H(i, j, k)   h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define U(i, j, k)   u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define V(i, j, k)   v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define W(i, j, k)   w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define RHS(i, j, k) rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+
 #endif // __UTIL_H_

BasicSolver/3D-mpi/src/vtkWriter-mpi.c

@@ -46,18 +46,18 @@ static void writeHeader(VtkOptions* o)
     cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n");
     cursor += sprintf(cursor,
         "DIMENSIONS %d %d %d\n",
-        o->grid.imax,
-        o->grid.jmax,
-        o->grid.kmax);
+        o->grid->imax,
+        o->grid->jmax,
+        o->grid->kmax);
     cursor += sprintf(cursor,
         "ORIGIN %f %f %f\n",
-        o->grid.dx * 0.5,
-        o->grid.dy * 0.5,
-        o->grid.dz * 0.5);
-    cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz);
+        o->grid->dx * 0.5,
+        o->grid->dy * 0.5,
+        o->grid->dz * 0.5);
+    cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz);
     cursor += sprintf(cursor,
         "POINT_DATA %d\n",
-        o->grid.imax * o->grid.jmax * o->grid.kmax);
+        o->grid->imax * o->grid->jmax * o->grid->kmax);
 
     if (commIsMaster(&o->comm)) {
         MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
@@ -67,7 +67,6 @@ static void writeHeader(VtkOptions* o)
 void vtkOpen(VtkOptions* o, char* problem)
 {
     char filename[50];
-
     snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size);
     MPI_File_open(o->comm.comm,
         filename,
@@ -98,7 +97,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
     }
 
     int offsets[NDIMS];
-    commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax);
+    commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax);
 
     // set global view in file
     MPI_Offset disp;
@@ -108,7 +107,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
     MPI_File_get_size(o->fh, &disp);
 
     MPI_Type_create_subarray(NDIMS,
-        (int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax },
+        (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax },
         (int[NDIMS]) { o->comm.kmaxLocal, o->comm.jmaxLocal, o->comm.imaxLocal },
         offsets,
         MPI_ORDER_C,
@@ -177,7 +176,7 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec)
     }
 
     int offsets[NDIMS];
-    commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax);
+    commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax);
 
     // set global view in file
     MPI_Offset disp;
@@ -190,7 +189,7 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec)
     MPI_Type_commit(&vectorType);
 
     MPI_Type_create_subarray(NDIMS,
-        (int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax },
+        (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax },
         (int[NDIMS]) { kmaxLocal, jmaxLocal, imaxLocal },
         offsets,
         MPI_ORDER_C,

BasicSolver/3D-mpi/src/vtkWriter-seq.c

@@ -41,14 +41,14 @@ static void writeHeader(VtkOptions* o)
     }
 
     fprintf(o->fh, "DATASET STRUCTURED_POINTS\n");
-    fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid.imax, o->grid.jmax, o->grid.kmax);
+    fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid->imax, o->grid->jmax, o->grid->kmax);
     fprintf(o->fh,
         "ORIGIN %f %f %f\n",
-        o->grid.dx * 0.5,
-        o->grid.dy * 0.5,
-        o->grid.dz * 0.5);
-    fprintf(o->fh, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz);
-    fprintf(o->fh, "POINT_DATA %d\n", o->grid.imax * o->grid.jmax * o->grid.kmax);
+        o->grid->dx * 0.5,
+        o->grid->dy * 0.5,
+        o->grid->dz * 0.5);
+    fprintf(o->fh, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz);
+    fprintf(o->fh, "POINT_DATA %d\n", o->grid->imax * o->grid->jmax * o->grid->kmax);
 }
 
 void vtkOpen(VtkOptions* o, char* problem)
@@ -64,9 +64,9 @@ void vtkOpen(VtkOptions* o, char* problem)
 
 static void writeScalar(VtkOptions* o, double* s)
 {
-    int imax = o->grid.imax;
-    int jmax = o->grid.jmax;
-    int kmax = o->grid.kmax;
+    int imax = o->grid->imax;
+    int jmax = o->grid->jmax;
+    int kmax = o->grid->kmax;
 
     for (int k = 0; k < kmax; k++) {
         for (int j = 0; j < jmax; j++) {
@@ -105,9 +105,9 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
 
 static void writeVector(VtkOptions* o, VtkVector vec)
 {
-    int imax = o->grid.imax;
-    int jmax = o->grid.jmax;
-    int kmax = o->grid.kmax;
+    int imax = o->grid->imax;
+    int jmax = o->grid->jmax;
+    int kmax = o->grid->kmax;
 
     for (int k = 0; k < kmax; k++) {
         for (int j = 0; j < jmax; j++) {

BasicSolver/3D-mpi/src/vtkWriter.h

@@ -14,7 +14,7 @@
 typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat;
 
 typedef struct VtkOptions {
-    Grid grid;
+    Grid* grid;
 #ifdef _VTK_WRITER_MPI
     MPI_File fh;
 #else

BasicSolver/3D-seq/Makefile

@@ -40,6 +40,18 @@ $(BUILD_DIR)/%.s:  %.c
 
 .PHONY: clean distclean tags info asm format
 
+vis:
+	$(info ===>  GENERATE VISUALIZATION)
+	@gnuplot -e "filename='residual.dat'" ./residual.plot
+
+vis_clean:
+	$(info ===>  CLEAN VISUALIZATION)
+	@rm -f *.dat
+	@rm -f *.vtk
+	@rm -f *.png
+
+clean: vis_clean
+
 clean:
 	$(info ===>  CLEAN)
 	@rm -rf $(BUILD_DIR)

BasicSolver/3D-seq/canal.par

@@ -38,10 +38,17 @@ kmax          50	   # number of interior cells in z-direction
 # Time Data:
 # ---------
 
-te       100.0   # final time
+te       60.0   # final time
 dt       0.02    # time stepsize
 tau      0.5     # safety factor for time stepsize control (<0 constant delt)
 
+# Multigrid data:
+# ---------
+
+levels        3         # Multigrid levels
+presmooth     5         # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
 # Pressure Iteration Data:
 # -----------------------
 

BasicSolver/3D-seq/config.mk

@@ -1,12 +1,12 @@
 # Supported: GCC, CLANG, ICC
-TAG ?= CLANG
+TAG ?= ICC
 ENABLE_OPENMP ?= false
-# Supported: sor, mg
+# Supported: rb, mg
 SOLVER ?= mg
 # Run in debug settings
 DEBUG ?= false
 
 #Feature options
 OPTIONS +=  -DARRAY_ALIGNMENT=64
-#OPTIONS +=  -DVERBOSE
+OPTIONS +=  -DVERBOSE
 #OPTIONS +=  -DDEBUG

BasicSolver/3D-seq/dcavity.par

@@ -38,10 +38,17 @@ kmax       128		# number of interior cells in z-direction
 # Time Data:
 # ---------
 
-te      2.0		# final time
+te       10.0		# final time
 dt       0.02	    # time stepsize
 tau      0.5		# safety factor for time stepsize control (<0 constant delt)
 
+# Multigrid data:
+# ---------
+
+levels        3         # Multigrid levels
+presmooth     20         # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
 # Pressure Iteration Data:
 # -----------------------
 
@@ -50,5 +57,4 @@ eps      0.001		# stopping tolerance for pressure iteration
 rho      0.5
 omg      1.7		# relaxation parameter for SOR iteration
 gamma    0.9		# upwind differencing factor gamma
-levels   5         # Multigrid levels
 #===============================================================================

BasicSolver/3D-seq/residual.plot

@@ -0,0 +1,9 @@
+set terminal png size 1800,768 enhanced font ,12
+set output 'residual.png'
+set datafile separator whitespace
+set xlabel "Timestep"
+set ylabel "Residual"
+
+set logscale y 2
+
+plot 'residual.dat' using 1:2 title "Residual"

BasicSolver/3D-seq/src/discretization.c

@@ -106,7 +106,7 @@ void initDiscretization(Discretization* d, Parameter* p)
     d->dtBound       = 0.5 * d->re * 1.0 / invSqrSum;
 
 #ifdef VERBOSE
-    printConfig(s);
+    printConfig(d);
 #endif /* VERBOSE */
 }
 

BasicSolver/3D-seq/src/main.c

@@ -73,6 +73,9 @@ int main(int argc, char** argv)
     Solver s;
     initParameter(&p);
 
+    FILE* fp;
+    fp = initResidualWriter();
+
     if (argc != 2) {
         printf("Usage: %s <configFile>\n", argv[0]);
         exit(EXIT_SUCCESS);
@@ -90,6 +93,7 @@ int main(int argc, char** argv)
     double te  = d.te;
     double t   = 0.0;
     int nt     = 0;
+    double res = 0.0;
 
     timeStart = getTimeStamp();
     while (t <= te) {
@@ -99,13 +103,16 @@ int main(int argc, char** argv)
         computeFG(&d);
         computeRHS(&d);
         if (nt % 100 == 0) normalizePressure(&d);
-        solve(&s, d.p, d.rhs);
+        res = solve(&s, d.p, d.rhs);
         adaptUV(&d);
+
+        writeResidual(fp, t, res);
+
         t += d.dt;
         nt++;
 
 #ifdef VERBOSE
-        printf("TIME %f , TIMESTEP %f\n", t, solver.dt);
+        printf("TIME %f , TIMESTEP %f\n", t, d.dt);
 #else
         printProgress(t);
 #endif
@@ -124,6 +131,7 @@ int main(int argc, char** argv)
     vg = allocate(64, bytesize);
     wg = allocate(64, bytesize);
     
+    fclose(fp);
     createBulkArrays(&d, pg, ug, vg, wg);
     VtkOptions opts = { .grid = d.grid };
     vtkOpen(&opts, d.problem);

BasicSolver/3D-seq/src/parameter.c

@@ -27,6 +27,8 @@ void initParameter(Parameter* param)
     param->gamma      = 0.9;
     param->tau        = 0.5;
     param->levels     = 5;
+    param->presmooth  = 5;
+    param->postsmooth = 5;
 }
 
 void readParameter(Parameter* param, const char* filename)

BasicSolver/3D-seq/src/parameter.h

@@ -18,6 +18,7 @@ typedef struct {
     char* name;
     int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
     double u_init, v_init, w_init, p_init;
+    int presmooth, postsmooth;
 } Parameter;
 
 void initParameter(Parameter*);

BasicSolver/3D-seq/src/progress.c

@@ -49,3 +49,22 @@ void stopProgress()
     printf("\n");
     fflush(stdout);
 }
+
+FILE* initResidualWriter()
+{
+    FILE* fp;
+    fp = fopen("residual.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return fp;
+
+}
+
+void writeResidual(FILE* fp, double ts, double res)
+{
+    fprintf(fp, "%f, %f\n", ts, res);
+}

BasicSolver/3D-seq/src/progress.h

@@ -10,5 +10,6 @@
 extern void initProgress(double);
 extern void printProgress(double);
 extern void stopProgress(void);
-
+extern FILE* initResidualWriter(void);
+extern void writeResidual(FILE*, double, double);
 #endif

BasicSolver/3D-seq/src/solver-mg.c

@@ -141,13 +141,62 @@ static double smooth(
             for (int j = 1; j < jmax + 1; j++) {
                 for (int i = isw; i < imax + 1; i += 2) {
 
-                    R(i, j, k) =
-                        RHS(i, j, k) -
+                    P(i, j, k) -=
+                        factor *
+                        (RHS(i, j, k) -
                             ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
-                            (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * idy2 +
-                            (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * idz2);
+                                (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
+                                    idy2 +
+                                (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
+                                    idz2));
+                }
+                isw = 3 - isw;
+            }
+            jsw = 3 - jsw;
+        }
+        ksw = 3 - ksw;
+    }
+}
+
+static double calculateResidual(
+    Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax)
+{
+    double eps    = s->eps;
+    int itermax   = s->itermax;
+    double dx2    = s->grid->dx * s->grid->dx;
+    double dy2    = s->grid->dy * s->grid->dy;
+    double dz2    = s->grid->dz * s->grid->dz;
+    double idx2   = 1.0 / dx2;
+    double idy2   = 1.0 / dy2;
+    double idz2   = 1.0 / dz2;
+    double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
+                    (dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
+    double* r    = s->r[level];
+    double epssq = eps * eps;
+    int it       = 0;
+    int pass, ksw, jsw, isw;
+    double res = 1.0;
+
+    ksw = 1;
+
+    for (pass = 0; pass < 2; pass++) {
+        jsw = ksw;
+
+        for (int k = 1; k < kmax + 1; k++) {
+            isw = jsw;
+            for (int j = 1; j < jmax + 1; j++) {
+                for (int i = isw; i < imax + 1; i += 2) {
+
+                    R(i,
+                        j,
+                        k) = (RHS(i, j, k) -
+                              ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) *
+                                      idx2 +
+                                  (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
+                                      idy2 +
+                                  (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
+                                      idz2));
 
-                    P(i, j, k) -= (factor * R(i, j, k));
                     res += (R(i, j, k) * R(i, j, k));
                 }
                 isw = 3 - isw;
@@ -167,7 +216,7 @@ static double multiGrid(
 {
     double res = 0.0;
 
-    // coarsest level TODO: Use direct solver?
+    // coarsest level
     if (level == COARSEST_LEVEL) {
         for (int i = 0; i < 5; i++) {
             smooth(s, p, rhs, level, imax, jmax, kmax);
@@ -175,17 +224,18 @@ static double multiGrid(
         return res;
     }
 
-    // pre-smoothing TODO: Make smoothing steps configurable?
-    for (int i = 0; i < 5; i++) {
+    // pre-smoothing
+    for (int i = 0; i < s->presmooth; i++) {
         smooth(s, p, rhs, level, imax, jmax, kmax);
         if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax);
     }
 
+    res = calculateResidual(s, p, rhs, level, imax, jmax, kmax);
+
     // restrict
     restrictMG(s, level, imax, jmax, kmax);
 
     // MGSolver on residual and error.
-    // TODO: What if there is a rest?
     multiGrid(s,
         s->e[level + 1],
         s->r[level + 1],
@@ -202,8 +252,8 @@ static double multiGrid(
     if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax);
 
     // post-smoothing
-    for (int i = 0; i < 5; i++) {
-        res = smooth(s, p, rhs, level, imax, jmax, kmax);
+    for (int i = 0; i < s->postsmooth; i++) {
+        smooth(s, p, rhs, level, imax, jmax, kmax);
         if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax);
     }
 
@@ -217,6 +267,8 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     s->itermax    = p->itermax;
     s->levels     = p->levels;
     s->grid       = &d->grid;
+    s->presmooth  = p->presmooth;
+    s->postsmooth = p->postsmooth;
 
     int imax   = s->grid->imax;
     int jmax   = s->grid->jmax;
@@ -240,11 +292,13 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     }
 }
 
-void solve(Solver* s, double* p, double* rhs)
+double solve(Solver* s, double* p, double* rhs)
 {
     double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax, s->grid->kmax);
 
 #ifdef VERBOSE
     printf("Residuum: %.6f\n", res);
 #endif
+
+return res;
 }

BasicSolver/3D-seq/src/solver-rb.c

@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include "solver.h"
+#include "util.h"
+
+void initSolver(Solver* s, Discretization* d, Parameter* p)
+{
+    s->grid    = &d->grid;
+    s->itermax = p->itermax;
+    s->eps     = p->eps;
+    s->omega   = p->omg;
+}
+
+double solve(Solver* s, double* p, double* rhs)
+{
+    int imax      = s->grid->imax;
+    int jmax      = s->grid->jmax;
+    int kmax      = s->grid->kmax;
+    double eps    = s->eps;
+    int itermax   = s->itermax;
+    double dx2    = s->grid->dx * s->grid->dx;
+    double dy2    = s->grid->dy * s->grid->dy;
+    double dz2    = s->grid->dz * s->grid->dz;
+    double idx2   = 1.0 / dx2;
+    double idy2   = 1.0 / dy2;
+    double idz2   = 1.0 / dz2;
+    double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) /
+                    (dy2 * dz2 + dx2 * dz2 + dx2 * dy2);
+    double epssq = eps * eps;
+    int it       = 0;
+    double res   = 1.0;
+    int pass, ksw, jsw, isw;
+
+    while ((res >= epssq) && (it < itermax)) {
+        res = 0.0;
+        ksw = 1;
+
+        for (pass = 0; pass < 2; pass++) {
+            jsw = ksw;
+
+            for (int k = 1; k < kmax + 1; k++) {
+                isw = jsw;
+                for (int j = 1; j < jmax + 1; j++) {
+                    for (int i = isw; i < imax + 1; i += 2) {
+
+                        double r =
+                            RHS(i, j, k) -
+                            ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
+                                (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
+                                    idy2 +
+                                (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
+                                    idz2);
+
+                        P(i, j, k) -= (factor * r);
+                        res += (r * r);
+                    }
+                    isw = 3 - isw;
+                }
+                jsw = 3 - jsw;
+            }
+            ksw = 3 - ksw;
+        }
+
+        for (int j = 1; j < jmax + 1; j++) {
+            for (int i = 1; i < imax + 1; i++) {
+                P(i, j, 0)        = P(i, j, 1);
+                P(i, j, kmax + 1) = P(i, j, kmax);
+            }
+        }
+
+        for (int k = 1; k < kmax + 1; k++) {
+            for (int i = 1; i < imax + 1; i++) {
+                P(i, 0, k)        = P(i, 1, k);
+                P(i, jmax + 1, k) = P(i, jmax, k);
+            }
+        }
+
+        for (int k = 1; k < kmax + 1; k++) {
+            for (int j = 1; j < jmax + 1; j++) {
+                P(0, j, k)        = P(1, j, k);
+                P(imax + 1, j, k) = P(imax, j, k);
+            }
+        }
+
+        res = res / (double)(imax * jmax * kmax);
+#ifdef DEBUG
+        printf("%d Residuum: %e\n", it, res);
+#endif
+        it++;
+    }
+
+#ifdef VERBOSE
+    printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
+#endif
+
+return res;
+}

BasicSolver/3D-seq/src/solver.h

@@ -18,9 +18,10 @@ typedef struct {
     int itermax;
     int levels;
     double **r, **e;
+    int presmooth, postsmooth;
 } Solver;
 
 extern void initSolver(Solver*, Discretization*, Parameter*);
-extern void solve(Solver*, double*, double*);
+extern double solve(Solver*, double*, double*);
 
 #endif

BasicSolver/3D-seq/src/vtkWriter.c

@@ -70,6 +70,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s)
         exit(EXIT_FAILURE);
     }
     fprintf(o->fh, "SCALARS %s float\n", name);
+    fprintf(o->fh, "LOOKUP_TABLE default\n");
 
     for (int k = 0; k < kmax; k++) {
         for (int j = 0; j < jmax; j++) {

EnhancedSolver/2D-mpi/Makefile

@@ -0,0 +1,88 @@
+#=======================================================================================
+# Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+# All rights reserved.
+# Use of this source code is governed by a MIT-style
+# license that can be found in the LICENSE file.
+#=======================================================================================
+
+#CONFIGURE BUILD SYSTEM
+TARGET	   = exe-$(TAG)
+BUILD_DIR  = ./$(TAG)
+SRC_DIR    = ./src
+MAKE_DIR   = ./
+Q         ?= @
+
+#DO NOT EDIT BELOW
+include $(MAKE_DIR)/config.mk
+include $(MAKE_DIR)/include_$(TAG).mk
+INCLUDES  += -I$(SRC_DIR) -I$(BUILD_DIR)
+
+VPATH     = $(SRC_DIR)
+SRC       = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c))
+ASM       = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC))
+OBJ       = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC))
+OBJ      += $(BUILD_DIR)/comm-$(COMM_TYPE).o
+OBJ      += $(BUILD_DIR)/solver-$(SOLVER).o
+SOURCES   = $(SRC) $(wildcard $(SRC_DIR)/*.h)
+CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES)
+
+${TARGET}: $(BUILD_DIR) $(OBJ)
+	$(info ===>  LINKING  $(TARGET))
+	$(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS)
+
+$(BUILD_DIR)/%.o:  %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk
+	$(info ===>  COMPILE  $@)
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
+	$(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM  $< > $(BUILD_DIR)/$*.d
+
+$(BUILD_DIR)/%.s:  %.c
+	$(info ===>  GENERATE ASM  $@)
+	$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
+
+.PHONY: clean distclean vis_clean vis tags info asm format
+
+vis:
+	$(info ===>  GENERATE VISUALIZATION)
+	@gnuplot -e "filename='pressure.dat'" ./surface.plot
+	@gnuplot -e "filename='velocity.dat'" ./vector.plot
+	@gnuplot -e "filename='residual.dat'" ./residual.plot
+
+vis_clean:
+	$(info ===>  CLEAN VISUALIZATION)
+	@rm -f *.dat
+	@rm -f *.png
+	@rm -f ./vis_files/*.dat
+	@rm -f ./vis_files/*.gif
+
+clean: vis_clean
+	$(info ===>  CLEAN)
+	@rm -rf $(BUILD_DIR)
+	@rm -f tags
+
+distclean: clean
+	$(info ===>  DIST CLEAN)
+	@rm -f $(TARGET)
+	@rm -f *.dat
+	@rm -f *.png
+
+info:
+	$(info $(CFLAGS))
+	$(Q)$(CC) $(VERSION)
+
+asm:  $(BUILD_DIR) $(ASM)
+
+tags:
+	$(info ===>  GENERATE TAGS)
+	$(Q)ctags -R
+
+format:
+	@for src in $(SOURCES) ; do \
+		echo "Formatting $$src" ; \
+		clang-format -i $$src ; \
+	done
+	@echo "Done"
+
+$(BUILD_DIR):
+	@mkdir $(BUILD_DIR)
+
+-include $(OBJ:.o=.d)

EnhancedSolver/2D-mpi/README.md

@@ -0,0 +1,48 @@
+# C source skeleton
+
+## Build
+
+1. Configure the toolchain and additional options in `config.mk`:
+```
+# Supported: GCC, CLANG, ICC
+TAG ?= GCC
+ENABLE_OPENMP ?= false
+
+OPTIONS +=  -DARRAY_ALIGNMENT=64
+#OPTIONS +=  -DVERBOSE_AFFINITY
+#OPTIONS +=  -DVERBOSE_DATASIZE
+#OPTIONS +=  -DVERBOSE_TIMER
+```
+
+The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution.
+
+
+2. Build with:
+```
+make
+```
+
+You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set.
+Intermediate build results are located in the `<TOOLCHAIN>` directory.
+
+To output the executed commands use:
+```
+make Q=
+```
+
+3. Clean up with:
+```
+make clean
+```
+to clean intermediate build results.
+
+```
+make distclean
+```
+to clean intermediate build results and binary.
+
+4. (Optional) Generate assembler:
+```
+make asm
+```
+The assembler files will also be located in the `<TOOLCHAIN>` directory.

EnhancedSolver/2D-mpi/backstep.par

@@ -0,0 +1,79 @@
+#==============================================================================
+#                            Laminar Canal Flow
+#==============================================================================
+
+# Problem specific Data:
+# ---------------------
+
+name backstep             # name of flow setup
+
+bcTop      1			#  flags for boundary conditions
+bcBottom   1			#  1 = no-slip      3 = outflow
+bcLeft     3			#  2 = free-slip    4 = periodic
+bcRight    3			#
+
+gx    0.0      # Body forces (e.g. gravity)
+gy    0.0      #
+
+re    36000.0	   # Reynolds number
+
+u_init    1.0      # initial value for velocity in x-direction
+v_init    0.0      # initial value for velocity in y-direction
+p_init    1.0      # initial value for pressure
+
+# Geometry Data:
+# -------------
+
+xlength    7.0     # domain size in x-direction
+ylength    1.5	   # domain size in y-direction
+imax       200     # number of interior cells in x-direction
+jmax       40	   # number of interior cells in y-direction
+
+# Time Data:
+# ---------
+
+te      60.0    # final time
+dt      0.02    # time stepsize
+tau     0.5     # safety factor for time stepsize control (<0 constant delt)
+
+# Pressure Iteration Data:
+# -----------------------
+
+itermax  500       # maximal number of pressure iteration in one time step
+eps      0.0001    # stopping tolerance for pressure iteration
+rho      0.52
+omg      1.7       # relaxation parameter for SOR iteration
+gamma    0.9       # upwind differencing factor gamma
+
+# Multigrid data:
+# ---------
+
+levels        3         # Multigrid levels
+presmooth     55        # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
+# Particle Tracing Data:
+# -----------------------
+
+numberOfParticles   500
+startTime           0     #if you want to see particles trapped in recirculation zone, startTime should be set to 0 
+injectTimePeriod    1.0
+writeTimePeriod     0.5
+
+x1                  0.0
+y1                  0.5
+x2                  0.0
+y2                  1.5
+
+# Obstacle Geometry Data:
+# -----------------------
+# Shape 0 disable, 1 Rectangle/Square, 2 Circle
+
+shape               1 
+xCenter             0.0
+yCenter             0.0
+xRectLength         2.0
+yRectLength         1.0
+circleRadius        1.0
+
+#===============================================================================

EnhancedSolver/2D-mpi/canal.par

@@ -0,0 +1,78 @@
+#==============================================================================
+#                            Laminar Canal Flow
+#==============================================================================
+
+# Problem specific Data:
+# ---------------------
+
+name canal             # name of flow setup
+
+bcTop      1			#  flags for boundary conditions
+bcBottom   1			#  1 = no-slip      3 = outflow
+bcLeft     3			#  2 = free-slip    4 = periodic
+bcRight    3			#
+
+gx     0.0      # Body forces (e.g. gravity)
+gy     0.0      #
+
+re            100.0	   # Reynolds number
+
+u_init        1.0      # initial value for velocity in x-direction
+v_init        0.0      # initial value for velocity in y-direction
+p_init        0.0      # initial value for pressure
+
+# Geometry Data:
+# -------------
+
+xlength       30.0     # domain size in x-direction
+ylength       4.0	   # domain size in y-direction
+imax          200      # number of interior cells in x-direction
+jmax          40	   # number of interior cells in y-direction
+
+# Time Data:
+# ---------
+
+te       60.0   # final time
+dt       0.02    # time stepsize
+tau      0.5     # safety factor for time stepsize control (<0 constant delt)
+
+# Multigrid data:
+# ---------
+
+levels        3         # Multigrid levels
+presmooth     5         # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
+# Pressure Iteration Data:
+# -----------------------
+
+itermax       500       # maximal number of pressure iteration in one time step
+eps           0.00001   # stopping tolerance for pressure iteration
+omg           1.7       # relaxation parameter for SOR iteration
+gamma         0.9       # upwind differencing factor gamma
+
+# Particle Tracing Data:
+# -----------------------
+
+numberOfParticles   60
+startTime           10.0
+injectTimePeriod    4.0
+writeTimePeriod     1.0
+
+x1                  1.0
+y1                  0.0
+x2                  1.0
+y2                  4.0
+
+# Obstacle Geometry Data:
+# -----------------------
+# Shape 0 disable, 1 Rectangle/Square, 2 Circle
+
+shape               0 
+xCenter             10.0
+yCenter             2
+xRectLength         6.0
+yRectLength         1.0
+circleRadius        1.0
+
+#===============================================================================

EnhancedSolver/2D-mpi/config.mk

@@ -0,0 +1,17 @@
+# Supported: GCC, CLANG, ICC
+TAG ?= ICC
+# Supported: true, false
+ENABLE_MPI ?= true
+ENABLE_OPENMP ?= false
+# Supported: rb, mg
+SOLVER ?= mg
+# Run in debug settings ?= mg
+COMM_TYPE ?= v3
+
+#Feature options
+OPTIONS +=  -DARRAY_ALIGNMENT=64
+OPTIONS +=  -DVERBOSE
+# OPTIONS +=  -DTEST
+#OPTIONS +=  -DVERBOSE_AFFINITY
+#OPTIONS +=  -DVERBOSE_DATASIZE
+#OPTIONS +=  -DVERBOSE_TIMER

EnhancedSolver/2D-mpi/dcavity.par

@@ -0,0 +1,79 @@
+#==============================================================================
+#                              Driven Cavity
+#==============================================================================
+
+# Problem specific Data:
+# ---------------------
+
+name dcavity        # name of flow setup
+
+bcTop      1			#  flags for boundary conditions
+bcBottom   1			#  1 = no-slip      3 = outflow
+bcLeft     1			#  2 = free-slip    4 = periodic
+bcRight    1			#
+
+gx    0.0			# Body forces (e.g. gravity)
+gy    0.0			#
+
+re    10.0		    # Reynolds number
+
+u_init    1.0		# initial value for velocity in x-direction
+v_init    0.0		# initial value for velocity in y-direction
+p_init    0.0		# initial value for pressure
+
+# Geometry Data:
+# -------------
+
+xlength    1.0		# domain size in x-direction
+ylength    1.0		# domain size in y-direction
+imax       128		# number of interior cells in x-direction
+jmax       128		# number of interior cells in y-direction
+
+# Time Data:
+# ---------
+
+te      10.0		# final time
+dt      0.02	    # time stepsize
+tau     0.5	    	# safety factor for time stepsize control (<0 constant delt)
+
+# Pressure Iteration Data:
+# -----------------------
+
+itermax  1000		# maximal number of pressure iteration in one time step
+eps      0.001		# stopping tolerance for pressure iteration
+rho      0.5
+omg      1.7		# relaxation parameter for SOR iteration
+gamma    0.9		# upwind differencing factor gamma
+
+# Multigrid data:
+# ---------
+
+levels        3         # Multigrid levels
+presmooth     20        # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
+# Particle Tracing Data:
+# -----------------------
+
+numberOfParticles   200
+startTime           2.0
+injectTimePeriod    0.5
+writeTimePeriod     0.2
+
+x1                  0.1
+y1                  0.9
+x2                  0.9
+y2                  0.9
+
+
+# Obstacle Geometry Data:
+# -----------------------
+# Shape 0 disable, 1 Rectangle/Square, 2 Circle
+
+shape               0 
+xCenter             0.5
+yCenter             0.5
+xRectLength         0.5
+yRectLength         0.5
+circleRadius        0.5
+#===============================================================================

EnhancedSolver/2D-mpi/include_CLANG.mk

@@ -0,0 +1,21 @@
+ifeq ($(ENABLE_MPI),true)
+CC   = mpicc
+DEFINES  = -D_MPI
+else
+CC   = cc
+endif
+
+GCC  = cc
+LINKER = $(CC)
+
+ifeq ($(ENABLE_OPENMP),true)
+OPENMP   = -fopenmp
+#OPENMP   = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp
+LIBS     = # -lomp
+endif
+
+VERSION  = --version
+CFLAGS   = -Ofast -std=c17
+LFLAGS   = $(OPENMP) -lm
+DEFINES  += -D_GNU_SOURCE# -DDEBUG
+INCLUDES = -I/opt/homebrew/include

EnhancedSolver/2D-mpi/include_GCC.mk

@@ -0,0 +1,20 @@
+ifeq ($(ENABLE_MPI),true)
+CC   = mpicc
+DEFINES  = -D_MPI
+else
+CC   = gcc
+endif
+
+GCC  = gcc
+LINKER = $(CC)
+
+ifeq ($(ENABLE_OPENMP),true)
+OPENMP   = -fopenmp
+endif
+
+VERSION  = --version
+CFLAGS   = -Ofast -ffreestanding -std=c99 $(OPENMP)
+LFLAGS   = $(OPENMP)
+DEFINES  += -D_GNU_SOURCE
+INCLUDES =
+LIBS     =

EnhancedSolver/2D-mpi/karman.par

@@ -0,0 +1,79 @@
+#==============================================================================
+#                            Laminar Canal Flow
+#==============================================================================
+
+# Problem specific Data:
+# ---------------------
+
+name karman             # name of flow setup
+
+bcTop      1			#  flags for boundary conditions
+bcBottom   1			#  1 = no-slip      3 = outflow
+bcLeft     3			#  2 = free-slip    4 = periodic
+bcRight    3			#
+
+gx    0.0      # Body forces (e.g. gravity)
+gy    0.0      #
+
+re    5050.0	   # Reynolds number
+
+u_init    1.0      # initial value for velocity in x-direction
+v_init    0.0      # initial value for velocity in y-direction
+p_init    0.0      # initial value for pressure
+
+# Geometry Data:
+# -------------
+
+xlength    30.0     # domain size in x-direction
+ylength    8.0	   # domain size in y-direction
+imax       400      # number of interior cells in x-direction
+jmax       200	   # number of interior cells in y-direction
+
+# Time Data:
+# ---------
+
+te      150.0   # final time
+dt      0.02    # time stepsize
+tau     0.5     # safety factor for time stepsize control (<0 constant delt)
+
+# Pressure Iteration Data:
+# -----------------------
+
+itermax  200       # maximal number of pressure iteration in one time step
+eps      0.001   # stopping tolerance for pressure iteration
+rho      0.52
+omg      1.75      # relaxation parameter for SOR iteration
+gamma    0.9       # upwind differencing factor gamma
+
+# Multigrid data:
+# ---------
+
+levels        3         # Multigrid levels
+presmooth     15         # Pre-smoothning iterations
+postsmooth    5         # Post-smoothning iterations
+
+# Particle Tracing Data:
+# -----------------------
+
+numberOfParticles   200
+startTime           50
+injectTimePeriod    1.0
+writeTimePeriod     0.5
+
+x1                  0.0
+y1                  3.8
+x2                  0.0
+y2                  4.1
+
+# Obstacle Geometry Data:
+# -----------------------
+# Shape 0 disable, 1 Rectangle/Square, 2 Circle
+
+shape               2 
+xCenter             5.0
+yCenter             4.0
+xRectLength         2.0
+yRectLength         1.0
+circleRadius        1.0
+
+#===============================================================================

EnhancedSolver/2D-mpi/residual.plot

@@ -0,0 +1,9 @@
+set terminal png size 1800,768 enhanced font ,12
+set output 'residual.png'
+set datafile separator whitespace
+set xlabel "Timestep"
+set ylabel "Residual"
+
+set logscale y 2
+
+plot 'residual.dat' using 1:2 title "Residual"

EnhancedSolver/2D-mpi/src/affinity.c

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+#ifdef __linux__
+#ifdef _OPENMP
+#include <pthread.h>
+#include <sched.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#define MAX_NUM_THREADS 128
+#define gettid()        syscall(SYS_gettid)
+
+static int getProcessorID(cpu_set_t* cpu_set)
+{
+    int processorId;
+
+    for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) {
+        if (CPU_ISSET(processorId, cpu_set)) {
+            break;
+        }
+    }
+    return processorId;
+}
+
+int affinity_getProcessorId()
+{
+    cpu_set_t cpu_set;
+    CPU_ZERO(&cpu_set);
+    sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set);
+
+    return getProcessorID(&cpu_set);
+}
+
+void affinity_pinThread(int processorId)
+{
+    cpu_set_t cpuset;
+    pthread_t thread;
+
+    thread = pthread_self();
+    CPU_ZERO(&cpuset);
+    CPU_SET(processorId, &cpuset);
+    pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
+}
+
+void affinity_pinProcess(int processorId)
+{
+    cpu_set_t cpuset;
+
+    CPU_ZERO(&cpuset);
+    CPU_SET(processorId, &cpuset);
+    sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
+}
+#endif /*_OPENMP*/
+#endif /*__linux__*/

EnhancedSolver/2D-mpi/src/affinity.h

@@ -0,0 +1,14 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+#ifndef AFFINITY_H
+#define AFFINITY_H
+
+extern int affinity_getProcessorId();
+extern void affinity_pinProcess(int);
+extern void affinity_pinThread(int);
+
+#endif /*AFFINITY_H*/

EnhancedSolver/2D-mpi/src/allocate.c

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+#include <errno.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "allocate.h"
+
+void* allocate(size_t alignment, size_t bytesize)
+{
+    int errorCode;
+    void* ptr;
+
+    errorCode = posix_memalign(&ptr, alignment, bytesize);
+
+    if (errorCode) {
+        if (errorCode == EINVAL) {
+            fprintf(stderr, "Error: Alignment parameter is not a power of two\n");
+            exit(EXIT_FAILURE);
+        }
+        if (errorCode == ENOMEM) {
+            fprintf(stderr, "Error: Insufficient memory to fulfill the request\n");
+            exit(EXIT_FAILURE);
+        }
+    }
+
+    if (ptr == NULL) {
+        fprintf(stderr, "Error: posix_memalign failed!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return ptr;
+}

EnhancedSolver/2D-mpi/src/allocate.h

@@ -0,0 +1,13 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+#ifndef __ALLOCATE_H_
+#define __ALLOCATE_H_
+#include <stdlib.h>
+
+extern void* allocate(size_t alignment, size_t bytesize);
+
+#endif

EnhancedSolver/2D-mpi/src/comm-v1.c

@@ -0,0 +1,233 @@
+/*
+ * Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <stdlib.h>
+
+#include "comm.h"
+
+#ifdef _MPI
+// subroutines local to this module
+static int sum(int* sizes, int position)
+{
+    int sum = 0;
+
+    for (int i = 0; i < position; i += position) {
+        sum += sizes[i];
+    }
+
+    return sum;
+}
+
+static void gatherArray(
+    Comm* c, int cnt, int* rcvCounts, int* displs, double* src, double* dst)
+{
+    double* sendbuffer = src + (c->imaxLocal + 2);
+
+    if (c->rank == 0) {
+        sendbuffer = src;
+    }
+
+    MPI_Gatherv(sendbuffer,
+        cnt,
+        MPI_DOUBLE,
+        dst,
+        rcvCounts,
+        displs,
+        MPI_DOUBLE,
+        0,
+        MPI_COMM_WORLD);
+}
+#endif // defined _MPI
+
+// exported subroutines
+int commIsBoundary(Comm* c, int direction)
+{
+#ifdef _MPI
+    switch (direction) {
+    case L:
+        return 1;
+        break;
+    case R:
+        return 1;
+        break;
+    case B:
+        return c->rank == 0;
+        break;
+    case T:
+        return c->rank == (c->size - 1);
+        break;
+    }
+#endif
+
+    return 1;
+}
+
+void commExchange(Comm* c, double* grid)
+{
+#ifdef _MPI
+    MPI_Request requests[4] = { MPI_REQUEST_NULL,
+        MPI_REQUEST_NULL,
+        MPI_REQUEST_NULL,
+        MPI_REQUEST_NULL };
+
+    /* exchange ghost cells with top neighbor */
+    if (c->rank + 1 < c->size) {
+        int top     = c->rank + 1;
+        double* src = grid + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
+        double* dst = grid + (c->jmaxLocal + 1) * (c->imaxLocal + 2) + 1;
+
+        MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, top, 1, MPI_COMM_WORLD, &requests[0]);
+        MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, top, 2, MPI_COMM_WORLD, &requests[1]);
+    }
+
+    /* exchange ghost cells with bottom neighbor */
+    if (c->rank > 0) {
+        int bottom  = c->rank - 1;
+        double* src = grid + (c->imaxLocal + 2) + 1;
+        double* dst = grid + 1;
+
+        MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]);
+        MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]);
+    }
+
+    MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
+#endif
+}
+
+void commShift(Comm* c, double* f, double* g)
+{
+#ifdef _MPI
+    MPI_Request requests[2] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL };
+
+    /* shift G */
+    /* receive ghost cells from bottom neighbor */
+    if (c->rank > 0) {
+        int bottom = c->rank - 1;
+        MPI_Irecv(g + 1,
+            c->imaxLocal,
+            MPI_DOUBLE,
+            bottom,
+            0,
+            MPI_COMM_WORLD,
+            &requests[0]);
+    }
+
+    if (c->rank + 1 < c->size) {
+        int top     = c->rank + 1;
+        double* buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
+        /* send ghost cells to top neighbor */
+        MPI_Isend(buf, c->imaxLocal, MPI_DOUBLE, top, 0, MPI_COMM_WORLD, &requests[1]);
+    }
+
+    MPI_Waitall(2, requests, MPI_STATUSES_IGNORE);
+#endif
+}
+
+void commCollectResult(Comm* c,
+    double* ug,
+    double* vg,
+    double* pg,
+    double* u,
+    double* v,
+    double* p,
+    int jmax,
+    int imax)
+{
+#ifdef _MPI
+    int *rcvCounts, *displs;
+    int cnt = c->jmaxLocal * (imax + 2);
+
+    if (c->rank == 0) {
+        rcvCounts = (int*)malloc(c->size * sizeof(int));
+        displs    = (int*)malloc(c->size * sizeof(int));
+    }
+
+    if (c->rank == 0 && c->size == 1) {
+        cnt = (c->jmaxLocal + 2) * (imax + 2);
+    } else if (c->rank == 0 || c->rank == (c->size - 1)) {
+        cnt = (c->jmaxLocal + 1) * (imax + 2);
+    }
+
+    MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD);
+
+    if (c->rank == 0) {
+        displs[0]  = 0;
+        int cursor = rcvCounts[0];
+
+        for (int i = 1; i < c->size; i++) {
+            displs[i] = cursor;
+            cursor += rcvCounts[i];
+        }
+    }
+
+    gatherArray(c, cnt, rcvCounts, displs, p, pg);
+    gatherArray(c, cnt, rcvCounts, displs, u, ug);
+    gatherArray(c, cnt, rcvCounts, displs, v, vg);
+#endif
+}
+
+void commPartition(Comm* c, int jmax, int imax)
+{
+#ifdef _MPI
+    c->imaxLocal = imax;
+    c->jmaxLocal = sizeOfRank(c->coords[JDIM], c->size, jmax);
+
+    c->neighbours[BOTTOM] = c->rank == 0 ? -1 : c->rank - 1;
+    c->neighbours[TOP]    = c->rank == (c->size - 1) ? -1 : c->rank + 1;
+    c->neighbours[LEFT]   = -1;
+    c->neighbours[RIGHT]  = -1;
+
+    c->coords[IDIM] = 0;
+    c->coords[JDIM] = c->rank;
+
+    c->dims[IDIM] = 1;
+    c->dims[JDIM] = c->size;
+#else
+    c->imaxLocal = imax;
+    c->jmaxLocal = jmax;
+#endif
+}
+
+void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
+{
+
+#if defined _MPI
+    newcomm->comm = MPI_COMM_NULL;
+    int result    = MPI_Comm_dup(MPI_COMM_WORLD, &newcomm->comm);
+
+    if (result == MPI_ERR_COMM) {
+        printf("\nNull communicator. Duplication failed !!\n");
+    }
+
+    newcomm->rank = oldcomm->rank;
+    newcomm->size = oldcomm->size;
+
+    newcomm->imaxLocal = imaxLocal / 2;
+    newcomm->jmaxLocal = jmaxLocal / 2;
+
+    newcomm->neighbours[BOTTOM] = newcomm->rank == 0 ? -1 : newcomm->rank - 1;
+    newcomm->neighbours[TOP]    = newcomm->rank == (newcomm->size - 1) ? -1 : newcomm->rank + 1;
+    newcomm->neighbours[LEFT]   = -1;
+    newcomm->neighbours[RIGHT]  = -1;
+
+    newcomm->coords[IDIM] = 0;
+    newcomm->coords[JDIM] = newcomm->rank;
+
+    newcomm->dims[IDIM] = 1;
+    newcomm->dims[JDIM] = newcomm->size;
+
+
+#endif
+    newcomm->imaxLocal = imaxLocal;
+    newcomm->jmaxLocal = jmaxLocal;
+}
+
+void commFreeCommunicator(Comm* comm)
+{
+#ifdef _MPI
+    MPI_Comm_free(&comm->comm);
+#endif
+}

EnhancedSolver/2D-mpi/src/comm-v2.c

@@ -0,0 +1,342 @@
+/*
+ * Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include "comm.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _MPI
+// subroutines local to this module
+static int sum(int* sizes, int init, int offset, int coord)
+{
+    int sum = 0;
+
+    for (int i = init - offset; coord > 0; i -= offset, --coord) {
+        sum += sizes[i];
+    }
+
+    return sum;
+}
+
+static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax)
+{
+    MPI_Request* requests;
+    int numRequests = 1;
+
+    if (c->rank == 0) {
+        numRequests = c->size + 1;
+    } else {
+        numRequests = 1;
+    }
+
+    requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
+
+    /* all ranks send their bulk array, including the external boundary layer */
+    MPI_Datatype bulkType;
+    int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
+    int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
+    int starts[NDIMS]   = { 1, 1 };
+
+    if (commIsBoundary(c, L)) {
+        newSizes[CIDIM] += 1;
+        starts[CIDIM] = 0;
+    }
+    if (commIsBoundary(c, R)) {
+        newSizes[CIDIM] += 1;
+    }
+    if (commIsBoundary(c, B)) {
+        newSizes[CJDIM] += 1;
+        starts[CJDIM] = 0;
+    }
+    if (commIsBoundary(c, T)) {
+        newSizes[CJDIM] += 1;
+    }
+
+    MPI_Type_create_subarray(NDIMS,
+        oldSizes,
+        newSizes,
+        starts,
+        MPI_ORDER_C,
+        MPI_DOUBLE,
+        &bulkType);
+    MPI_Type_commit(&bulkType);
+    MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
+
+    int newSizesI[c->size];
+    int newSizesJ[c->size];
+    MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD);
+    MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+    /* rank 0 assembles the subdomains */
+    if (c->rank == 0) {
+        for (int i = 0; i < c->size; i++) {
+            MPI_Datatype domainType;
+            int oldSizes[NDIMS] = { jmax + 2, imax + 2 };
+            int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
+            int coords[NDIMS];
+            MPI_Cart_coords(c->comm, i, NDIMS, coords);
+            int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]),
+                sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) };
+            printf(
+                "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
+                "Starts(i,j): %d %d\n",
+                i,
+                coords[IDIM],
+                coords[JDIM],
+                oldSizes[CIDIM],
+                oldSizes[CJDIM],
+                newSizes[CIDIM],
+                newSizes[CJDIM],
+                starts[CIDIM],
+                starts[CJDIM]);
+
+            MPI_Type_create_subarray(NDIMS,
+                oldSizes,
+                newSizes,
+                starts,
+                MPI_ORDER_C,
+                MPI_DOUBLE,
+                &domainType);
+            MPI_Type_commit(&domainType);
+
+            MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]);
+            MPI_Type_free(&domainType);
+        }
+    }
+
+    MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
+}
+#endif // defined _MPI
+
+// exported subroutines
+int commIsBoundary(Comm* c, int direction)
+{
+#ifdef _MPI
+    switch (direction) {
+    case L:
+        return c->coords[IDIM] == 0;
+        break;
+    case R:
+        return c->coords[IDIM] == (c->dims[IDIM] - 1);
+        break;
+    case B:
+        return c->coords[JDIM] == 0;
+        break;
+    case T:
+        return c->coords[JDIM] == (c->dims[JDIM] - 1);
+        break;
+    }
+#endif
+
+    return 1;
+}
+
+void commExchange(Comm* c, double* grid)
+{
+#ifdef _MPI
+    MPI_Request requests[8];
+    for (int i = 0; i < 8; i++)
+        requests[i] = MPI_REQUEST_NULL;
+
+    for (int i = 0; i < NDIRS; i++) {
+        double* sbuf = grid + c->sdispls[i];
+        double* rbuf = grid + c->rdispls[i];
+
+        int tag = 0;
+        if (c->neighbours[i] != MPI_PROC_NULL) {
+            // printf("DEBUG: Rank %d - SendRecv with %d\n", c->rank, c->neighbours[i]);
+            tag = c->neighbours[i];
+        }
+        MPI_Irecv(rbuf,
+            1,
+            c->bufferTypes[i],
+            c->neighbours[i],
+            tag,
+            c->comm,
+            &requests[i * 2]);
+        MPI_Isend(sbuf,
+            1,
+            c->bufferTypes[i],
+            c->neighbours[i],
+            c->rank,
+            c->comm,
+            &requests[i * 2 + 1]);
+    }
+
+    MPI_Waitall(8, requests, MPI_STATUSES_IGNORE);
+#endif
+}
+
+void commShift(Comm* c, double* f, double* g)
+{
+#ifdef _MPI
+    MPI_Request requests[4] = { MPI_REQUEST_NULL,
+        MPI_REQUEST_NULL,
+        MPI_REQUEST_NULL,
+        MPI_REQUEST_NULL };
+
+    /* shift G */
+    /* receive ghost cells from bottom neighbor */
+    double* buf = g + 1;
+    MPI_Irecv(buf,
+        1,
+        c->bufferTypes[B],
+        c->neighbours[B],
+        0,
+        c->comm,
+        &requests[0]);
+
+    /* send ghost cells to top neighbor */
+    buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
+    MPI_Isend(buf, 1, c->bufferTypes[T], c->neighbours[T], 0, c->comm, &requests[1]);
+
+    /* shift F */
+    /* receive ghost cells from left neighbor */
+    buf = f + (c->imaxLocal + 2);
+    MPI_Irecv(buf,
+        1,
+        c->bufferTypes[L],
+        c->neighbours[L],
+        1,
+        c->comm,
+        &requests[2]);
+
+    /* send ghost cells to right neighbor */
+    buf = f + (c->imaxLocal + 2) + (c->imaxLocal);
+    MPI_Isend(buf,
+        1,
+        c->bufferTypes[R],
+        c->neighbours[R],
+        1,
+        c->comm,
+        &requests[3]);
+
+    MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
+#endif
+}
+
+void commCollectResult(Comm* c,
+    double* ug,
+    double* vg,
+    double* pg,
+    double* u,
+    double* v,
+    double* p,
+    int imax,
+    int jmax)
+{
+#ifdef _MPI
+    /* collect P */
+    assembleResult(c, p, pg, imax, jmax);
+
+    /* collect U */
+    assembleResult(c, u, ug, imax, jmax);
+
+    /* collect V */
+    assembleResult(c, v, vg, imax, jmax);
+#endif
+}
+
+void commPartition(Comm* c, int jmax, int imax)
+{
+#ifdef _MPI
+    int dims[NDIMS]    = { 0, 0 };
+    int periods[NDIMS] = { 0, 0 };
+    MPI_Dims_create(c->size, NDIMS, dims);
+    MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm);
+    MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[L], &c->neighbours[R]);
+    MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[B], &c->neighbours[T]);
+    MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
+
+    int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax);
+    int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax);
+
+    c->imaxLocal = imaxLocal;
+    c->jmaxLocal = jmaxLocal;
+
+    MPI_Datatype jBufferType;
+    MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
+    MPI_Type_commit(&jBufferType);
+
+    MPI_Datatype iBufferType;
+    MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
+    MPI_Type_commit(&iBufferType);
+
+    c->bufferTypes[L]   = iBufferType;
+    c->bufferTypes[R]  = iBufferType;
+    c->bufferTypes[B] = jBufferType;
+    c->bufferTypes[T]    = jBufferType;
+
+    c->sdispls[L]   = (imaxLocal + 2) + 1;
+    c->sdispls[R]  = (imaxLocal + 2) + imaxLocal;
+    c->sdispls[B] = (imaxLocal + 2) + 1;
+    c->sdispls[T]    = jmaxLocal * (imaxLocal + 2) + 1;
+
+    c->rdispls[L]   = (imaxLocal + 2);
+    c->rdispls[R]  = (imaxLocal + 2) + (imaxLocal + 1);
+    c->rdispls[B] = 1;
+    c->rdispls[T]    = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
+#else
+    c->imaxLocal = imax;
+    c->jmaxLocal = jmax;
+#endif
+}
+
+void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
+{
+#if defined _MPI
+    newcomm->comm = MPI_COMM_NULL;
+    int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
+
+    if (result == MPI_ERR_COMM) {
+        printf("\nNull communicator. Duplication failed !!\n");
+    }
+
+    newcomm->rank = oldcomm->rank;
+    newcomm->size = oldcomm->size;
+
+    memcpy(&newcomm->neighbours, &oldcomm->neighbours, sizeof(oldcomm->neighbours));
+    memcpy(&newcomm->coords, &oldcomm->coords, sizeof(oldcomm->coords));
+    memcpy(&newcomm->dims, &oldcomm->dims, sizeof(oldcomm->dims));
+
+    newcomm->imaxLocal = imaxLocal/2;
+    newcomm->jmaxLocal = jmaxLocal/2;
+
+    MPI_Datatype jBufferType;
+    MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
+    MPI_Type_commit(&jBufferType);
+
+    MPI_Datatype iBufferType;
+    MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
+    MPI_Type_commit(&iBufferType);
+
+    newcomm->bufferTypes[L]   = iBufferType;
+    newcomm->bufferTypes[R]  = iBufferType;
+    newcomm->bufferTypes[B] = jBufferType;
+    newcomm->bufferTypes[T]    = jBufferType;
+
+    newcomm->sdispls[L]   = (imaxLocal + 2) + 1;
+    newcomm->sdispls[R]  = (imaxLocal + 2) + imaxLocal;
+    newcomm->sdispls[B] = (imaxLocal + 2) + 1;
+    newcomm->sdispls[T]    = jmaxLocal * (imaxLocal + 2) + 1;
+
+    newcomm->rdispls[L]   = (imaxLocal + 2);
+    newcomm->rdispls[R]  = (imaxLocal + 2) + (imaxLocal + 1);
+    newcomm->rdispls[B] = 1;
+    newcomm->rdispls[T]    = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
+#else
+    newcomm->imaxLocal = imaxLocal;
+    newcomm->jmaxLocal = jmaxLocal;
+#endif
+}
+
+void commFreeCommunicator(Comm* comm)
+{
+    #ifdef _MPI
+        MPI_Comm_free(&comm->comm);
+    #endif
+}

EnhancedSolver/2D-mpi/src/comm-v3.c

@@ -0,0 +1,320 @@
+/*
+ * Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "comm.h"
+
+#ifdef _MPI
+// subroutines local to this module
+static int sum(int* sizes, int init, int offset, int coord)
+{
+    int sum = 0;
+
+    for (int i = init - offset; coord > 0; i -= offset, --coord) {
+        sum += sizes[i];
+    }
+
+    return sum;
+}
+
+static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax)
+{
+    MPI_Request* requests;
+    int numRequests = 1;
+
+    if (c->rank == 0) {
+        numRequests = c->size + 1;
+    } else {
+        numRequests = 1;
+    }
+
+    requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
+
+    /* all ranks send their bulk array, including the external boundary layer */
+    MPI_Datatype bulkType;
+    int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
+    int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
+    int starts[NDIMS]   = { 1, 1 };
+
+    if (commIsBoundary(c, L)) {
+        newSizes[CIDIM] += 1;
+        starts[CIDIM] = 0;
+    }
+    if (commIsBoundary(c, R)) {
+        newSizes[CIDIM] += 1;
+    }
+    if (commIsBoundary(c, B)) {
+        newSizes[CJDIM] += 1;
+        starts[CJDIM] = 0;
+    }
+    if (commIsBoundary(c, T)) {
+        newSizes[CJDIM] += 1;
+    }
+
+    MPI_Type_create_subarray(NDIMS,
+        oldSizes,
+        newSizes,
+        starts,
+        MPI_ORDER_C,
+        MPI_DOUBLE,
+        &bulkType);
+    MPI_Type_commit(&bulkType);
+    MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
+
+    int newSizesI[c->size];
+    int newSizesJ[c->size];
+    MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD);
+    MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+    /* rank 0 assembles the subdomains */
+    if (c->rank == 0) {
+        for (int i = 0; i < c->size; i++) {
+            MPI_Datatype domainType;
+            int oldSizes[NDIMS] = { jmax + 2, imax + 2 };
+            int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
+            int coords[NDIMS];
+            MPI_Cart_coords(c->comm, i, NDIMS, coords);
+            int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]),
+                sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) };
+            printf(
+                "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
+                "Starts(i,j): %d %d\n",
+                i,
+                coords[IDIM],
+                coords[JDIM],
+                oldSizes[CIDIM],
+                oldSizes[CJDIM],
+                newSizes[CIDIM],
+                newSizes[CJDIM],
+                starts[CIDIM],
+                starts[CJDIM]);
+
+            MPI_Type_create_subarray(NDIMS,
+                oldSizes,
+                newSizes,
+                starts,
+                MPI_ORDER_C,
+                MPI_DOUBLE,
+                &domainType);
+            MPI_Type_commit(&domainType);
+
+            MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]);
+            MPI_Type_free(&domainType);
+        }
+    }
+
+    MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
+}
+#endif // defined _MPI
+
+// exported subroutines
+int commIsBoundary(Comm* c, int direction)
+{
+#ifdef _MPI
+    switch (direction) {
+    case L:
+        return c->coords[IDIM] == 0;
+        break;
+    case R:
+        return c->coords[IDIM] == (c->dims[IDIM] - 1);
+        break;
+    case B:
+        return c->coords[JDIM] == 0;
+        break;
+    case T:
+        return c->coords[JDIM] == (c->dims[JDIM] - 1);
+        break;
+    }
+#endif
+
+    return 1;
+}
+
+void commExchange(Comm* c, double* grid)
+{
+#ifdef _MPI
+    int counts[NDIRS] = { 1, 1, 1, 1 };
+    MPI_Neighbor_alltoallw(grid,
+        counts,
+        c->sdispls,
+        c->bufferTypes,
+        grid,
+        counts,
+        c->rdispls,
+        c->bufferTypes,
+        c->comm);
+#endif
+}
+
+void commShift(Comm* c, double* f, double* g)
+{
+#ifdef _MPI
+    MPI_Request requests[4] = { MPI_REQUEST_NULL,
+        MPI_REQUEST_NULL,
+        MPI_REQUEST_NULL,
+        MPI_REQUEST_NULL };
+
+    /* shift G */
+    /* receive ghost cells from bottom neighbor */
+    double* buf = g + 1;
+    MPI_Irecv(buf,
+        1,
+        c->bufferTypes[B],
+        c->neighbours[B],
+        0,
+        c->comm,
+        &requests[0]);
+
+    /* send ghost cells to top neighbor */
+    buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
+    MPI_Isend(buf, 1, c->bufferTypes[T], c->neighbours[T], 0, c->comm, &requests[1]);
+
+    /* shift F */
+    /* receive ghost cells from left neighbor */
+    buf = f + (c->imaxLocal + 2);
+    MPI_Irecv(buf,
+        1,
+        c->bufferTypes[L],
+        c->neighbours[L],
+        1,
+        c->comm,
+        &requests[2]);
+
+    /* send ghost cells to right neighbor */
+    buf = f + (c->imaxLocal + 2) + (c->imaxLocal);
+    MPI_Isend(buf,
+        1,
+        c->bufferTypes[R],
+        c->neighbours[R],
+        1,
+        c->comm,
+        &requests[3]);
+
+    MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
+#endif
+}
+
+void commCollectResult(Comm* c,
+    double* ug,
+    double* vg,
+    double* pg,
+    double* u,
+    double* v,
+    double* p,
+    int imax,
+    int jmax)
+{
+#ifdef _MPI
+    /* collect P */
+    assembleResult(c, p, pg, imax, jmax);
+
+    /* collect U */
+    assembleResult(c, u, ug, imax, jmax);
+
+    /* collect V */
+    assembleResult(c, v, vg, imax, jmax);
+#endif
+}
+
+void commPartition(Comm* c, int jmax, int imax)
+{
+#ifdef _MPI
+    int dims[NDIMS]    = { 0, 0 };
+    int periods[NDIMS] = { 0, 0 };
+    MPI_Dims_create(c->size, NDIMS, dims);
+    MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm);
+    MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[L], &c->neighbours[R]);
+    MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[B], &c->neighbours[T]);
+    MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
+
+    int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax);
+    int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax);
+
+    c->imaxLocal = imaxLocal;
+    c->jmaxLocal = jmaxLocal;
+
+    MPI_Datatype jBufferType;
+    MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
+    MPI_Type_commit(&jBufferType);
+
+    MPI_Datatype iBufferType;
+    MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
+    MPI_Type_commit(&iBufferType);
+
+    c->bufferTypes[L]   = iBufferType;
+    c->bufferTypes[R]  = iBufferType;
+    c->bufferTypes[B] = jBufferType;
+    c->bufferTypes[T]    = jBufferType;
+
+    size_t dblsize     = sizeof(double);
+    c->sdispls[L]   = ((imaxLocal + 2) + 1) * dblsize;
+    c->sdispls[R]  = ((imaxLocal + 2) + imaxLocal) * dblsize;
+    c->sdispls[B] = ((imaxLocal + 2) + 1) * dblsize;
+    c->sdispls[T]    = (jmaxLocal * (imaxLocal + 2) + 1) * dblsize;
+
+    c->rdispls[L]   = (imaxLocal + 2) * dblsize;
+    c->rdispls[R]  = ((imaxLocal + 2) + (imaxLocal + 1)) * dblsize;
+    c->rdispls[B] = 1 * dblsize;
+    c->rdispls[T]    = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * dblsize;
+#else
+    c->imaxLocal = imax;
+    c->jmaxLocal = jmax;
+#endif
+}
+
+void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
+{
+#if defined _MPI
+
+    int result    = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
+
+    if (result == MPI_ERR_COMM) {
+        printf("\nNull communicator. Duplication failed !!\n");
+    }
+
+    newcomm->rank = oldcomm->rank;
+    newcomm->size = oldcomm->size;
+    
+
+    newcomm->imaxLocal = imaxLocal / 2;
+    newcomm->jmaxLocal = jmaxLocal / 2;
+
+    MPI_Datatype jBufferType;
+    MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
+    MPI_Type_commit(&jBufferType);
+
+    MPI_Datatype iBufferType;
+    MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
+    MPI_Type_commit(&iBufferType);
+
+    newcomm->bufferTypes[L]   = iBufferType;
+    newcomm->bufferTypes[R]  = iBufferType;
+    newcomm->bufferTypes[B] = jBufferType;
+    newcomm->bufferTypes[T]    = jBufferType;
+
+    newcomm->sdispls[L]   = (imaxLocal + 2) + 1;
+    newcomm->sdispls[R]  = (imaxLocal + 2) + imaxLocal;
+    newcomm->sdispls[B] = (imaxLocal + 2) + 1;
+    newcomm->sdispls[T]    = jmaxLocal * (imaxLocal + 2) + 1;
+
+    newcomm->rdispls[L]   = (imaxLocal + 2);
+    newcomm->rdispls[R]  = (imaxLocal + 2) + (imaxLocal + 1);
+    newcomm->rdispls[B] = 1;
+    newcomm->rdispls[T]    = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
+#else
+    newcomm->imaxLocal = imaxLocal;
+    newcomm->jmaxLocal = jmaxLocal;
+#endif
+}
+
+void commFreeCommunicator(Comm* comm)
+{
+    #ifdef _MPI
+        MPI_Comm_free(&comm->comm);
+    #endif
+}

EnhancedSolver/2D-mpi/src/comm.c

@@ -0,0 +1,129 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "comm.h"
+
+// subroutines local to this module
+int sizeOfRank(int rank, int size, int N)
+{
+    return N / size + ((N % size > rank) ? 1 : 0);
+}
+
+void commReduction(double* v, int op)
+{
+#ifdef _MPI
+    if (op == MAX) {
+        MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
+    } else if (op == SUM) {
+        MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+    }
+#endif
+}
+
+void commPrintConfig(Comm* c)
+{
+#ifdef _MPI
+    fflush(stdout);
+    MPI_Barrier(MPI_COMM_WORLD);
+    if (commIsMaster(c)) {
+        printf("Communication setup:\n");
+    }
+
+    for (int i = 0; i < c->size; i++) {
+        if (i == c->rank) {
+            printf("\tRank %d of %d\n", c->rank, c->size);
+            printf("\tNeighbours (bottom, top, left, right): %d %d, %d, %d\n",
+                c->neighbours[B],
+                c->neighbours[T],
+                c->neighbours[L],
+                c->neighbours[R]);
+            printf("\tIs boundary:\n");
+            printf("\t\tLEFT: %d\n", commIsBoundary(c, L));
+            printf("\t\tRIGHT: %d\n", commIsBoundary(c, R));
+            printf("\t\tBOTTOM: %d\n", commIsBoundary(c, B));
+            printf("\t\tTOP: %d\n", commIsBoundary(c, T));
+            printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]);
+            printf("\tDims (i,j) %d %d\n", c->dims[IDIM], c->dims[JDIM]);
+            printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal);
+            fflush(stdout);
+        }
+        MPI_Barrier(MPI_COMM_WORLD);
+    }
+#endif
+}
+
+void commInit(Comm* c, int argc, char** argv)
+{
+#ifdef _MPI
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
+    MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
+#else
+    c->rank = 0;
+    c->size = 1;
+#endif
+}
+
+void commTestInit(Comm* c, double* p, double* f, double* g)
+{
+    int imax = c->imaxLocal;
+    int jmax = c->jmaxLocal;
+    int rank = c->rank;
+
+    for (int j = 0; j < jmax + 2; j++) {
+        for (int i = 0; i < imax + 2; i++) {
+            p[j * (imax + 2) + i] = rank;
+            f[j * (imax + 2) + i] = rank;
+            g[j * (imax + 2) + i] = rank;
+        }
+    }
+}
+
+static void testWriteFile(char* filename, double* grid, int imax, int jmax)
+{
+    FILE* fp = fopen(filename, "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    for (int j = 0; j < jmax + 2; j++) {
+        for (int i = 0; i < imax + 2; i++) {
+            fprintf(fp, "%.2f ", grid[j * (imax + 2) + i]);
+        }
+        fprintf(fp, "\n");
+    }
+
+    fclose(fp);
+}
+
+void commTestWrite(Comm* c, double* p, double* f, double* g)
+{
+    int imax = c->imaxLocal;
+    int jmax = c->jmaxLocal;
+    int rank = c->rank;
+
+    char filename[30];
+    snprintf(filename, 30, "ptest-%d.dat", rank);
+    testWriteFile(filename, p, imax, jmax);
+
+    snprintf(filename, 30, "ftest-%d.dat", rank);
+    testWriteFile(filename, f, imax, jmax);
+
+    snprintf(filename, 30, "gtest-%d.dat", rank);
+    testWriteFile(filename, g, imax, jmax);
+}
+
+void commFinalize(Comm* c)
+{
+#ifdef _MPI
+    MPI_Finalize();
+#endif
+}

EnhancedSolver/2D-mpi/src/comm.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#ifndef __COMM_H_
+#define __COMM_H_
+#if defined(_MPI)
+#include <mpi.h>
+#endif
+
+enum direction { L = 0, R, B, T, NDIRS }; // L = Left, R = Right, B = Bottom, T =Top
+enum dimension { IDIM = 0, JDIM, NDIMS };
+enum cdimension { CJDIM = 0, CIDIM };
+enum layer { HALO = 0, BULK };
+enum op { MAX = 0, SUM };
+
+typedef struct {
+    int rank;
+    int size;
+#if defined(_MPI)
+    MPI_Comm comm;
+    MPI_Datatype bufferTypes[NDIRS];
+    MPI_Aint sdispls[NDIRS];
+    MPI_Aint rdispls[NDIRS];
+#endif
+    int neighbours[NDIRS];
+    int coords[NDIMS], dims[NDIMS];
+    int imaxLocal, jmaxLocal;
+} Comm;
+
+extern int sizeOfRank(int rank, int size, int N);
+extern void commInit(Comm* c, int argc, char** argv);
+extern void commTestInit(Comm* c, double* p, double* f, double* g);
+extern void commTestWrite(Comm* c, double* p, double* f, double* g);
+extern void commFinalize(Comm* c);
+extern void commPartition(Comm* c, int jmax, int imax);
+extern void commPrintConfig(Comm*);
+extern void commExchange(Comm*, double*);
+extern void commShift(Comm* c, double* f, double* g);
+extern void commReduction(double* v, int op);
+extern int commIsBoundary(Comm* c, int direction);
+extern void commUpdateDatatypes(Comm*, Comm*, int, int);
+extern void commFreeCommunicator(Comm*);
+extern void commCollectResult(Comm* c,
+    double* ug,
+    double* vg,
+    double* pg,
+    double* u,
+    double* v,
+    double* p,
+    int jmax,
+    int imax);
+
+static inline int commIsMaster(Comm* c) { return c->rank == 0; }
+#endif // __COMM_H_

EnhancedSolver/2D-mpi/src/discretization.c

@@ -0,0 +1,724 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "allocate.h"
+#include "comm.h"
+#include "discretization.h"
+#include "parameter.h"
+#include "util.h"
+
+static double distance(double i, double j, double iCenter, double jCenter)
+{
+    return sqrt(pow(iCenter - i, 2) + pow(jCenter - j, 2) * 1.0);
+}
+
+double sumOffset(double* sizes, int init, int offset, int coord)
+{
+    double sum = 0;
+
+    for (int i = init - offset; coord > 0; i -= offset, --coord) {
+        sum += sizes[i];
+    }
+
+    return sum;
+}
+
+void print(Discretization* d, double* grid)
+{
+    int imaxLocal = d->comm.imaxLocal;
+
+    for (int i = 0; i < d->comm.size; i++) {
+        if (i == d->comm.rank) {
+            sleep(1 * d->comm.rank);
+            printf("### RANK %d LVL "
+                   "###################################################### #\n ",
+                d->comm.rank);
+            for (int j = 0; j < d->comm.jmaxLocal + 2; j++) {
+                printf("%02d: ", j);
+                for (int i = 0; i < d->comm.imaxLocal + 2; i++) {
+                    printf("%2.2f  ", grid[j * (imaxLocal + 2) + i]);
+                }
+                printf("\n");
+            }
+            fflush(stdout);
+        }
+    }
+}
+
+static void printConfig(Discretization* d)
+{
+    if (commIsMaster(&d->comm)) {
+        printf("Parameters for #%s#\n", d->problem);
+        printf("BC Left:%d Right:%d Bottom:%d Top:%d\n",
+            d->bcLeft,
+            d->bcRight,
+            d->bcBottom,
+            d->bcTop);
+        printf("\tReynolds number: %.2f\n", d->re);
+        printf("\tGx Gy: %.2f %.2f\n", d->gx, d->gy);
+        printf("Geometry data:\n");
+        printf("\tDomain box size (x, y): %.2f, %.2f\n",
+            d->grid.xlength,
+            d->grid.ylength);
+        printf("\tCells (x, y): %d, %d\n", d->grid.imax, d->grid.jmax);
+        printf("\tCell size (dx, dy): %f, %f\n", d->grid.dx, d->grid.dy);
+        printf("Timestep parameters:\n");
+        printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te);
+        printf("\tdt bound: %.6f\n", d->dtBound);
+        printf("\tTau factor: %.2f\n", d->tau);
+        printf("Iterative s parameters:\n");
+        printf("\tgamma factor: %f\n", d->gamma);
+    }
+    commPrintConfig(&d->comm);
+}
+
+void initDiscretiztion(Discretization* d, Parameter* params)
+{
+    d->problem      = params->name;
+    d->bcLeft       = params->bcLeft;
+    d->bcRight      = params->bcRight;
+    d->bcBottom     = params->bcBottom;
+    d->bcTop        = params->bcTop;
+    d->grid.imax    = params->imax;
+    d->grid.jmax    = params->jmax;
+    d->grid.xlength = params->xlength;
+    d->grid.ylength = params->ylength;
+    d->grid.dx      = params->xlength / params->imax;
+    d->grid.dy      = params->ylength / params->jmax;
+    d->re           = params->re;
+    d->gx           = params->gx;
+    d->gy           = params->gy;
+    d->dt           = params->dt;
+    d->te           = params->te;
+    d->tau          = params->tau;
+    d->gamma        = params->gamma;
+
+    /* allocate arrays */
+    int imaxLocal = d->comm.imaxLocal;
+    int jmaxLocal = d->comm.jmaxLocal;
+    size_t size   = (imaxLocal + 2) * (jmaxLocal + 2);
+
+    d->u      = allocate(64, size * sizeof(double));
+    d->v      = allocate(64, size * sizeof(double));
+    d->p      = allocate(64, size * sizeof(double));
+    d->rhs    = allocate(64, size * sizeof(double));
+    d->f      = allocate(64, size * sizeof(double));
+    d->g      = allocate(64, size * sizeof(double));
+    d->grid.s = allocate(64, size * sizeof(double));
+
+    for (int i = 0; i < size; i++) {
+        d->u[i]      = params->u_init;
+        d->v[i]      = params->v_init;
+        d->p[i]      = params->p_init;
+        d->rhs[i]    = 0.0;
+        d->f[i]      = 0.0;
+        d->g[i]      = 0.0;
+        d->grid.s[i] = FLUID;
+    }
+
+    double dx = d->grid.dx;
+    double dy = d->grid.dy;
+
+    double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy);
+    d->dtBound       = 0.5 * d->re * 1.0 / invSqrSum;
+
+    d->xLocal = d->comm.imaxLocal * d->grid.dx;
+    d->yLocal = d->comm.jmaxLocal * d->grid.dy;
+
+    double xLocal[d->comm.size];
+    double yLocal[d->comm.size];
+
+#ifdef _MPI
+    MPI_Allgather(&d->xLocal, 1, MPI_DOUBLE, xLocal, 1, MPI_DOUBLE, d->comm.comm);
+    MPI_Allgather(&d->yLocal, 1, MPI_DOUBLE, yLocal, 1, MPI_DOUBLE, d->comm.comm);
+
+    d->xOffset    = sumOffset(xLocal,
+        d->comm.rank,
+        d->comm.dims[JDIM],
+        d->comm.coords[IDIM]);
+    d->yOffset    = sumOffset(yLocal, d->comm.rank, 1, d->comm.coords[JDIM]);
+    d->xOffsetEnd = d->xOffset + d->xLocal;
+    d->yOffsetEnd = d->yOffset + d->yLocal;
+#else
+
+    d->xOffset    = 0;
+    d->yOffset    = 0;
+    d->xOffsetEnd = d->xOffset + d->xLocal;
+    d->yOffsetEnd = d->yOffset + d->yLocal;
+#endif
+
+    printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : "
+           "%.2f\n",
+        d->comm.rank,
+        d->xOffset,
+        d->yOffset,
+        d->xOffsetEnd,
+        d->yOffsetEnd);
+
+    double* s   = d->grid.s;
+    int iOffset = 0, jOffset = 0;
+
+    double xCenter = 0, yCenter = 0, radius = 0;
+    double x1 = 0, x2 = 0, y1 = 0, y2 = 0;
+
+    switch (params->shape) {
+    case NOSHAPE:
+        break;
+    case RECT:
+        x1 = params->xCenter - params->xRectLength / 2;
+        x2 = params->xCenter + params->xRectLength / 2;
+        y1 = params->yCenter - params->yRectLength / 2;
+        y2 = params->yCenter + params->yRectLength / 2;
+
+        iOffset = d->xOffset / dx;
+        jOffset = d->yOffset / dy;
+
+        for (int j = 1; j < jmaxLocal + 1; ++j) {
+            for (int i = 1; i < imaxLocal + 1; ++i) {
+                if ((x1 <= ((i + iOffset) * dx)) && (((i + iOffset) * dx) <= x2) &&
+                    (y1 <= ((j + jOffset) * dy)) && (((j + jOffset) * dy) <= y2)) {
+                    S(i, j) = OBSTACLE;
+                }
+            }
+        }
+
+        break;
+    case CIRCLE:
+        xCenter = params->xCenter;
+        yCenter = params->yCenter;
+        radius  = params->circleRadius;
+
+        iOffset = d->xOffset / dx;
+        jOffset = d->yOffset / dy;
+
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                if (distance(((i + iOffset) * dx),
+                        ((j + jOffset) * dy),
+                        xCenter,
+                        yCenter) <= radius) {
+                    S(i, j) = OBSTACLE;
+                }
+            }
+        }
+
+        break;
+    default:
+        break;
+    }
+
+#ifdef _MPI
+    commExchange(&d->comm, s);
+#endif
+
+    for (int j = 1; j < jmaxLocal + 1; j++) {
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            if (S(i, j - 1) == FLUID && S(i, j + 1) == OBSTACLE && S(i, j) == OBSTACLE)
+                S(i, j) = BOTTOM; // BOTTOM
+            if (S(i - 1, j) == FLUID && S(i + 1, j) == OBSTACLE && S(i, j) == OBSTACLE)
+                S(i, j) = LEFT; // LEFT
+            if (S(i + 1, j) == FLUID && S(i - 1, j) == OBSTACLE && S(i, j) == OBSTACLE)
+                S(i, j) = RIGHT; // RIGHT
+            if (S(i, j + 1) == FLUID && S(i, j - 1) == OBSTACLE && S(i, j) == OBSTACLE)
+                S(i, j) = TOP; // TOP
+            if (S(i - 1, j - 1) == FLUID && S(i, j - 1) == FLUID &&
+                S(i - 1, j) == FLUID && S(i + 1, j + 1) == OBSTACLE &&
+                (S(i, j) == OBSTACLE || S(i, j) == LEFT || S(i, j) == BOTTOM))
+                S(i, j) = BOTTOMLEFT; // BOTTOMLEFT
+            if (S(i + 1, j - 1) == FLUID && S(i, j - 1) == FLUID &&
+                S(i + 1, j) == FLUID && S(i - 1, j + 1) == OBSTACLE &&
+                (S(i, j) == OBSTACLE || S(i, j) == RIGHT || S(i, j) == BOTTOM))
+                S(i, j) = BOTTOMRIGHT; // BOTTOMRIGHT
+            if (S(i - 1, j + 1) == FLUID && S(i - 1, j) == FLUID &&
+                S(i, j + 1) == FLUID && S(i + 1, j - 1) == OBSTACLE &&
+                (S(i, j) == OBSTACLE || S(i, j) == LEFT || S(i, j) == TOP))
+                S(i, j) = TOPLEFT; // TOPLEFT
+            if (S(i + 1, j + 1) == FLUID && S(i + 1, j) == FLUID &&
+                S(i, j + 1) == FLUID && S(i - 1, j - 1) == OBSTACLE &&
+                (S(i, j) == OBSTACLE || S(i, j) == RIGHT || S(i, j) == TOP))
+                S(i, j) = TOPRIGHT; // TOPRIGHT
+        }
+    }
+
+#ifdef VERBOSE
+    printConfig(d);
+#endif
+}
+
+void computeRHS(Discretization* d)
+{
+    int imaxLocal = d->comm.imaxLocal;
+    int jmaxLocal = d->comm.jmaxLocal;
+    double idx    = 1.0 / d->grid.dx;
+    double idy    = 1.0 / d->grid.dy;
+    double idt    = 1.0 / d->dt;
+    double* rhs   = d->rhs;
+    double* f     = d->f;
+    double* g     = d->g;
+
+    commShift(&d->comm, f, g);
+
+    for (int j = 1; j < jmaxLocal + 1; j++) {
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) *
+                        idt;
+        }
+    }
+}
+
+static double maxElement(Discretization* d, double* m)
+{
+    int imaxLocal = d->comm.imaxLocal;
+    int jmaxLocal = d->comm.jmaxLocal;
+    int size      = (imaxLocal + 2) * (jmaxLocal + 2);
+    double maxval = DBL_MIN;
+
+    for (int i = 0; i < size; i++) {
+        maxval = MAX(maxval, fabs(m[i]));
+    }
+
+    commReduction(&maxval, MAX);
+    return maxval;
+}
+
+void computeTimestep(Discretization* d)
+{
+    double dt   = d->dtBound;
+    double dx   = d->grid.dx;
+    double dy   = d->grid.dy;
+    double umax = maxElement(d, d->u);
+    double vmax = maxElement(d, d->v);
+
+    if (umax > 0) {
+        dt = (dt > dx / umax) ? dx / umax : dt;
+    }
+    if (vmax > 0) {
+        dt = (dt > dy / vmax) ? dy / vmax : dt;
+    }
+
+    d->dt = dt * d->tau;
+}
+
+void setBoundaryConditions(Discretization* d)
+{
+    int imaxLocal = d->comm.imaxLocal;
+    int jmaxLocal = d->comm.jmaxLocal;
+    double* u     = d->u;
+    double* v     = d->v;
+
+    if (commIsBoundary(&d->comm, T)) {
+        switch (d->bcTop) {
+        case NOSLIP:
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                V(i, jmaxLocal)     = 0.0;
+                U(i, jmaxLocal + 1) = -U(i, jmaxLocal);
+            }
+            break;
+        case SLIP:
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                V(i, jmaxLocal)     = 0.0;
+                U(i, jmaxLocal + 1) = U(i, jmaxLocal);
+            }
+            break;
+        case OUTFLOW:
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                U(i, jmaxLocal + 1) = U(i, jmaxLocal);
+                V(i, jmaxLocal)     = V(i, jmaxLocal - 1);
+            }
+            break;
+        case PERIODIC:
+            break;
+        }
+    }
+
+    if (commIsBoundary(&d->comm, B)) {
+        switch (d->bcBottom) {
+        case NOSLIP:
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                V(i, 0) = 0.0;
+                U(i, 0) = -U(i, 1);
+            }
+            break;
+        case SLIP:
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                V(i, 0) = 0.0;
+                U(i, 0) = U(i, 1);
+            }
+            break;
+        case OUTFLOW:
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                U(i, 0) = U(i, 1);
+                V(i, 0) = V(i, 1);
+            }
+            break;
+        case PERIODIC:
+            break;
+        }
+    }
+
+    if (commIsBoundary(&d->comm, R)) {
+        switch (d->bcRight) {
+        case NOSLIP:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                U(imaxLocal, j)     = 0.0;
+                V(imaxLocal + 1, j) = -V(imaxLocal, j);
+            }
+            break;
+        case SLIP:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                U(imaxLocal, j)     = 0.0;
+                V(imaxLocal + 1, j) = V(imaxLocal, j);
+            }
+            break;
+        case OUTFLOW:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                U(imaxLocal, j)     = U(imaxLocal - 1, j);
+                V(imaxLocal + 1, j) = V(imaxLocal, j);
+            }
+            break;
+        case PERIODIC:
+            break;
+        }
+    }
+
+    if (commIsBoundary(&d->comm, L)) {
+        switch (d->bcLeft) {
+        case NOSLIP:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                U(0, j) = 0.0;
+                V(0, j) = -V(1, j);
+            }
+            break;
+        case SLIP:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                U(0, j) = 0.0;
+                V(0, j) = V(1, j);
+            }
+            break;
+        case OUTFLOW:
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                U(0, j) = U(1, j);
+                V(0, j) = V(1, j);
+            }
+            break;
+        case PERIODIC:
+            break;
+        }
+    }
+}
+
+void setSpecialBoundaryCondition(Discretization* d)
+{
+    int imaxLocal = d->comm.imaxLocal;
+    int jmaxLocal = d->comm.jmaxLocal;
+    double* u     = d->u;
+    double* s     = d->grid.s;
+
+    if (strcmp(d->problem, "dcavity") == 0) {
+        if (commIsBoundary(&d->comm, T)) {
+            for (int i = 1; i < imaxLocal + 1; i++) {
+                U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal);
+            }
+        }
+    } else if (strcmp(d->problem, "canal") == 0) {
+        if (commIsBoundary(&d->comm, L)) {
+
+            double ylength = d->grid.ylength;
+            double dy      = d->grid.dy;
+            int rest       = d->grid.jmax % d->comm.dims[JDIM];
+            int yc         = d->comm.rank * (d->grid.jmax / d->comm.dims[JDIM]) +
+                     MIN(rest, d->comm.rank);
+            double ys = dy * (yc + 0.5);
+            double y;
+
+            // printf("RANK %d yc: %d ys: %f\n",d->comm.rank, yc, ys);
+
+            for (int j = 1; j < jmaxLocal + 1; j++) {
+                y       = ys + dy * (j - 0.5);
+                U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength);
+            }
+        }
+    } else if (strcmp(d->problem, "backstep") == 0) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            if (S(0, j) == FLUID) U(0, j) = 1.0;
+        }
+    } else if (strcmp(d->problem, "karman") == 0) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            U(0, j) = 1.0;
+        }
+    }
+    /* print(solver, solver->u); */
+}
+
+void setObjectBoundaryCondition(Discretization* d)
+{
+    int imaxLocal = d->comm.imaxLocal;
+    int jmaxLocal = d->comm.jmaxLocal;
+    double* u     = d->u;
+    double* v     = d->v;
+    double* s     = d->grid.s;
+
+    for (int j = 1; j < jmaxLocal + 1; j++) {
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            switch ((int)S(i, j)) {
+            case TOP:
+                U(i, j)     = -U(i, j + 1);
+                U(i - 1, j) = -U(i - 1, j + 1);
+                V(i, j)     = 0.0;
+                break;
+            case BOTTOM:
+                U(i, j)     = -U(i, j - 1);
+                U(i - 1, j) = -U(i - 1, j - 1);
+                V(i, j)     = 0.0;
+                break;
+            case LEFT:
+                U(i - 1, j) = 0.0;
+                V(i, j)     = -V(i - 1, j);
+                V(i, j - 1) = -V(i - 1, j - 1);
+                break;
+            case RIGHT:
+                U(i, j)     = 0.0;
+                V(i, j)     = -V(i + 1, j);
+                V(i, j - 1) = -V(i + 1, j - 1);
+                break;
+            case TOPLEFT:
+                U(i, j)     = -U(i, j + 1);
+                U(i - 1, j) = 0.0;
+                V(i, j)     = 0.0;
+                V(i, j - 1) = -V(i - 1, j - 1);
+                break;
+            case TOPRIGHT:
+                U(i, j)     = 0.0;
+                U(i - 1, j) = -U(i - 1, j + 1);
+                V(i, j)     = 0.0;
+                V(i, j - 1) = -V(i + 1, j - 1);
+                break;
+            case BOTTOMLEFT:
+                U(i, j)     = -U(i, j - 1);
+                U(i - 1, j) = 0.0;
+                V(i, j)     = -V(i - 1, j);
+                V(i, j - 1) = 0.0;
+                break;
+            case BOTTOMRIGHT:
+                U(i, j)     = 0.0;
+                U(i - 1, j) = -U(i - 1, j - 1);
+                V(i, j)     = -V(i, j + 1);
+                V(i, j - 1) = 0.0;
+                break;
+            }
+        }
+    }
+}
+
+void computeFG(Discretization* d)
+{
+    double* u = d->u;
+    double* v = d->v;
+    double* f = d->f;
+    double* g = d->g;
+    double* s = d->grid.s;
+
+    int imaxLocal = d->comm.imaxLocal;
+    int jmaxLocal = d->comm.jmaxLocal;
+
+    double gx        = d->gx;
+    double gy        = d->gy;
+    double gamma     = d->gamma;
+    double dt        = d->dt;
+    double inverseRe = 1.0 / d->re;
+    double inverseDx = 1.0 / d->grid.dx;
+    double inverseDy = 1.0 / d->grid.dy;
+    double du2dx, dv2dy, duvdx, duvdy;
+    double du2dx2, du2dy2, dv2dx2, dv2dy2;
+
+    commExchange(&d->comm, u);
+    commExchange(&d->comm, v);
+
+    for (int j = 1; j < jmaxLocal + 1; j++) {
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            if (S(i, j) == FLUID) {
+                du2dx = inverseDx * 0.25 *
+                            ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) -
+                                (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) +
+                        gamma * inverseDx * 0.25 *
+                            (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) +
+                                fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j)));
+
+                duvdy = inverseDy * 0.25 *
+                            ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) -
+                                (V(i, j - 1) + V(i + 1, j - 1)) *
+                                    (U(i, j) + U(i, j - 1))) +
+                        gamma * inverseDy * 0.25 *
+                            (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) +
+                                fabs(V(i, j - 1) + V(i + 1, j - 1)) *
+                                    (U(i, j) - U(i, j - 1)));
+
+                du2dx2 = inverseDx * inverseDx *
+                         (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j));
+                du2dy2 = inverseDy * inverseDy *
+                         (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1));
+                F(i, j) = U(i, j) +
+                          dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx);
+
+                duvdx = inverseDx * 0.25 *
+                            ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) -
+                                (U(i - 1, j) + U(i - 1, j + 1)) *
+                                    (V(i, j) + V(i - 1, j))) +
+                        gamma * inverseDx * 0.25 *
+                            (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) +
+                                fabs(U(i - 1, j) + U(i - 1, j + 1)) *
+                                    (V(i, j) - V(i - 1, j)));
+
+                dv2dy = inverseDy * 0.25 *
+                            ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) -
+                                (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) +
+                        gamma * inverseDy * 0.25 *
+                            (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) +
+                                fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1)));
+
+                dv2dx2 = inverseDx * inverseDx *
+                         (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j));
+                dv2dy2 = inverseDy * inverseDy *
+                         (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1));
+                G(i, j) = V(i, j) +
+                          dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy);
+            } else {
+                switch ((int)S(i, j)) {
+                case TOP:
+                    G(i, j) = V(i, j);
+                    break;
+                case BOTTOM:
+                    G(i, j - 1) = V(i, j - 1);
+                    break;
+                case LEFT:
+                    F(i - 1, j) = U(i - 1, j);
+                    break;
+                case RIGHT:
+                    F(i, j) = U(i, j);
+                    break;
+                case TOPLEFT:
+                    F(i - 1, j) = U(i - 1, j);
+                    G(i, j)     = V(i, j);
+                    break;
+                case TOPRIGHT:
+                    F(i, j) = U(i, j);
+                    G(i, j) = V(i, j);
+                    break;
+                case BOTTOMLEFT:
+                    F(i - 1, j) = U(i - 1, j);
+                    G(i, j - 1) = V(i, j - 1);
+                    break;
+                case BOTTOMRIGHT:
+                    F(i, j)     = U(i, j);
+                    G(i, j - 1) = V(i, j - 1);
+                    break;
+                }
+            }
+        }
+    }
+
+    /* ----------------------------- boundary of F --------------------------- */
+    if (commIsBoundary(&d->comm, L)) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            F(0, j) = U(0, j);
+        }
+    }
+
+    if (commIsBoundary(&d->comm, R)) {
+        for (int j = 1; j < jmaxLocal + 1; j++) {
+            F(imaxLocal, j) = U(imaxLocal, j);
+        }
+    }
+
+    /* ----------------------------- boundary of G --------------------------- */
+    if (commIsBoundary(&d->comm, B)) {
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            G(i, 0) = V(i, 0);
+        }
+    }
+
+    if (commIsBoundary(&d->comm, T)) {
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            G(i, jmaxLocal) = V(i, jmaxLocal);
+        }
+    }
+}
+
+void adaptUV(Discretization* d)
+{
+    int imaxLocal = d->comm.imaxLocal;
+    int jmaxLocal = d->comm.jmaxLocal;
+
+    double* p = d->p;
+    double* u = d->u;
+    double* v = d->v;
+    double* f = d->f;
+    double* g = d->g;
+
+    double factorX = d->dt / d->grid.dx;
+    double factorY = d->dt / d->grid.dy;
+
+    for (int j = 1; j < jmaxLocal + 1; j++) {
+        for (int i = 1; i < imaxLocal + 1; i++) {
+            U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX;
+            V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY;
+        }
+    }
+}
+
+void writeResult(Discretization* d, double* u, double* v, double* p)
+{
+    int imax  = d->grid.imax;
+    int jmax  = d->grid.jmax;
+    double dx = d->grid.dx;
+    double dy = d->grid.dy;
+    double x = 0.0, y = 0.0;
+
+    FILE* fp;
+    fp = fopen("pressure.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    for (int j = 1; j <= jmax; j++) {
+        y = (double)(j - 0.5) * dy;
+        for (int i = 1; i <= imax; i++) {
+            x = (double)(i - 0.5) * dx;
+            fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax + 2) + i]);
+        }
+        fprintf(fp, "\n");
+    }
+
+    fclose(fp);
+
+    fp = fopen("velocity.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    for (int j = 1; j <= jmax; j++) {
+        y = dy * (j - 0.5);
+        for (int i = 1; i <= imax; i++) {
+            x           = dx * (i - 0.5);
+            double velU = (u[j * (imax + 2) + i] + u[j * (imax + 2) + (i - 1)]) / 2.0;
+            double velV = (v[j * (imax + 2) + i] + v[(j - 1) * (imax + 2) + i]) / 2.0;
+            double len  = sqrt((velU * velU) + (velV * velV));
+            fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, velU, velV, len);
+        }
+    }
+
+    fclose(fp);
+}

EnhancedSolver/2D-mpi/src/discretization.h

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#ifndef __DISCRETIZATION_H_
+#define __DISCRETIZATION_H_
+#include "comm.h"
+#include "grid.h"
+#include "parameter.h"
+#include<unistd.h>
+
+enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
+
+enum OBJECTBOUNDARY {
+    FLUID = 0,
+    TOP,
+    BOTTOM,
+    LEFT,
+    RIGHT,
+    TOPLEFT,
+    BOTTOMLEFT,
+    TOPRIGHT,
+    BOTTOMRIGHT,
+    OBSTACLE
+};
+
+enum SHAPE { NOSHAPE = 0, RECT, CIRCLE };
+
+typedef struct {
+    /* geometry and grid information */
+    Grid grid;
+    /* arrays */
+    double *p, *rhs;
+    double *f, *g;
+    double *u, *v;
+    /* parameters */
+    double re, tau, gamma;
+    double gx, gy;
+    /* time stepping */
+    double dt, te;
+    double dtBound;
+    char* problem;
+
+    double xLocal, yLocal, xOffset, yOffset, xOffsetEnd, yOffsetEnd;
+
+
+    int bcLeft, bcRight, bcBottom, bcTop;
+    /* communication */
+    Comm comm;
+} Discretization;
+
+extern void initDiscretiztion(Discretization*, Parameter*);
+extern void computeRHS(Discretization*);
+extern void normalizePressure(Discretization*);
+extern void computeTimestep(Discretization*);
+extern void setBoundaryConditions(Discretization*);
+extern void setSpecialBoundaryCondition(Discretization*);
+extern void setObjectBoundaryCondition(Discretization*);
+extern void computeFG(Discretization*);
+extern void adaptUV(Discretization*);
+extern void writeResult(Discretization* s, double* u, double* v, double* p);
+extern double sumOffset(double* , int , int , int );
+extern void print(Discretization* , double* );
+#endif

EnhancedSolver/2D-mpi/src/grid.h

@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#ifndef __GRID_H_
+#define __GRID_H_
+
+typedef struct {
+    double dx, dy;
+    int imax, jmax;
+    double xlength, ylength;
+    double* s;
+} Grid;
+
+#endif // __GRID_H_

EnhancedSolver/2D-mpi/src/likwid-marker.h

@@ -0,0 +1,54 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
+ *      Copyright (c) 2020 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+#ifndef LIKWID_MARKERS_H
+#define LIKWID_MARKERS_H
+
+#ifdef LIKWID_PERFMON
+#include <likwid.h>
+#define LIKWID_MARKER_INIT                likwid_markerInit()
+#define LIKWID_MARKER_THREADINIT          likwid_markerThreadInit()
+#define LIKWID_MARKER_SWITCH              likwid_markerNextGroup()
+#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag)
+#define LIKWID_MARKER_START(regionTag)    likwid_markerStartRegion(regionTag)
+#define LIKWID_MARKER_STOP(regionTag)     likwid_markerStopRegion(regionTag)
+#define LIKWID_MARKER_CLOSE               likwid_markerClose()
+#define LIKWID_MARKER_RESET(regionTag)    likwid_markerResetRegion(regionTag)
+#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count)                       \
+    likwid_markerGetRegion(regionTag, nevents, events, time, count)
+#else /* LIKWID_PERFMON */
+#define LIKWID_MARKER_INIT
+#define LIKWID_MARKER_THREADINIT
+#define LIKWID_MARKER_SWITCH
+#define LIKWID_MARKER_REGISTER(regionTag)
+#define LIKWID_MARKER_START(regionTag)
+#define LIKWID_MARKER_STOP(regionTag)
+#define LIKWID_MARKER_CLOSE
+#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count)
+#define LIKWID_MARKER_RESET(regionTag)
+#endif /* LIKWID_PERFMON */
+
+#endif /*LIKWID_MARKERS_H*/

EnhancedSolver/2D-mpi/src/main.c

@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "allocate.h"
+#include "comm.h"
+#include "discretization.h"
+#include "grid.h"
+#include "parameter.h"
+#include "particletracing.h"
+#include "progress.h"
+#include "timing.h"
+
+static void writeResults(Discretization* s)
+{
+#ifdef _MPI
+    size_t bytesize = (s->grid.imax + 2) * (s->grid.jmax + 2) * sizeof(double);
+
+    double* ug = allocate(64, bytesize);
+    double* vg = allocate(64, bytesize);
+    double* pg = allocate(64, bytesize);
+
+    commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->grid.imax, s->grid.jmax);
+    if (commIsMaster(&s->comm)) {
+        writeResult(s, ug, vg, pg);
+    }
+
+    free(ug);
+    free(vg);
+    free(pg);
+#else
+    writeResult(s, s->u, s->v, s->p);
+#endif
+}
+
+int main(int argc, char** argv)
+{
+    int rank;
+    double timeStart, timeStop;
+    Parameter p;
+    Discretization d;
+    Solver s;
+    ParticleTracer particletracer;
+
+    commInit(&d.comm, argc, argv);
+    initParameter(&p);
+    FILE* fp;
+    fp = initResidualWriter();
+
+    if (argc != 2) {
+        printf("Usage: %s <configFile>\n", argv[0]);
+        exit(EXIT_SUCCESS);
+    }
+
+    readParameter(&p, argv[1]);
+    commPartition(&d.comm, p.jmax, p.imax);
+    if (commIsMaster(&d.comm)) {
+        printParameter(&p);
+    }
+
+    initDiscretiztion(&d, &p);
+    initSolver(&s, &d, &p);
+    initParticleTracer(&particletracer, &d, &p);
+
+#ifdef TEST
+    commPrintConfig(&d.comm);
+    commTestInit(&d.comm, d.p, d.f, d.g);
+    commExchange(&d.comm, d.p);
+    commShift(&d.comm, d.f, d.g);
+    commTestWrite(&d.comm, d.p, d.f, d.g);
+    writeResults(&d);
+    commFinalize(&d.comm);
+    exit(EXIT_SUCCESS);
+#endif
+#ifndef VERBOSE
+    initProgress(d.te);
+#endif
+
+    double tau = d.tau;
+    double te  = d.te;
+    double t   = 0.0;
+    double res = 0.0;
+    timeStart = getTimeStamp();
+    while (t <= te) {
+        if (tau > 0.0) computeTimestep(&d);
+        setBoundaryConditions(&d);
+        setSpecialBoundaryCondition(&d);
+        setObjectBoundaryCondition(&d);
+        computeFG(&d);
+        computeRHS(&d);
+        res = solve(&s, d.p, d.rhs);
+        adaptUV(&d);
+        trace(&particletracer, &d, t);
+
+        writeResidual(fp, t, res);
+
+        t += d.dt;
+
+#ifdef VERBOSE
+        if (commIsMaster(s.comm)) {
+            printf("TIME %f , TIMESTEP %f\n", t, d.dt);
+        }
+#else
+        printProgress(t);
+#endif
+    }
+    timeStop = getTimeStamp();
+#ifndef VERBOSE
+    stopProgress();
+#endif
+    if (commIsMaster(s.comm)) {
+        printf("Solution took %.2fs\n", timeStop - timeStart);
+    }
+    fclose(fp);
+    freeParticles(&particletracer);
+    writeResults(&d);
+    commFinalize(s.comm);
+    return EXIT_SUCCESS;
+}

EnhancedSolver/2D-mpi/src/parameter.c

@@ -0,0 +1,143 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "parameter.h"
+#include "util.h"
+#define MAXLINE 4096
+
+void initParameter(Parameter* param)
+{
+    param->xlength    = 1.0;
+    param->ylength    = 1.0;
+    param->imax       = 100;
+    param->jmax       = 100;
+    param->itermax    = 1000;
+    param->eps        = 0.0001;
+    param->omg        = 1.8;
+    param->levels     = 5;
+    param->presmooth  = 5;
+    param->postsmooth = 5;
+}
+
+void readParameter(Parameter* param, const char* filename)
+{
+    FILE* fp = fopen(filename, "r");
+    char line[MAXLINE];
+    int i;
+
+    if (!fp) {
+        fprintf(stderr, "Could not open parameter file: %s\n", filename);
+        exit(EXIT_FAILURE);
+    }
+
+    while (!feof(fp)) {
+        line[0] = '\0';
+        fgets(line, MAXLINE, fp);
+        for (i = 0; line[i] != '\0' && line[i] != '#'; i++)
+            ;
+        line[i] = '\0';
+
+        char* tok = strtok(line, " ");
+        char* val = strtok(NULL, " ");
+
+#define PARSE_PARAM(p, f)                                                                \
+    if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) {                         \
+        param->p = f(val);                                                               \
+    }
+#define PARSE_STRING(p) PARSE_PARAM(p, strdup)
+#define PARSE_INT(p)    PARSE_PARAM(p, atoi)
+#define PARSE_REAL(p)   PARSE_PARAM(p, atof)
+
+        if (tok != NULL && val != NULL) {
+            PARSE_REAL(xlength);
+            PARSE_REAL(ylength);
+            PARSE_INT(imax);
+            PARSE_INT(jmax);
+            PARSE_INT(itermax);
+            PARSE_REAL(eps);
+            PARSE_REAL(omg);
+            PARSE_REAL(re);
+            PARSE_REAL(tau);
+            PARSE_REAL(gamma);
+            PARSE_REAL(dt);
+            PARSE_REAL(te);
+            PARSE_REAL(gx);
+            PARSE_REAL(gy);
+            PARSE_STRING(name);
+            PARSE_INT(bcLeft);
+            PARSE_INT(bcRight);
+            PARSE_INT(bcBottom);
+            PARSE_INT(bcTop);
+            PARSE_INT(levels);
+            PARSE_INT(presmooth);
+            PARSE_INT(postsmooth);
+            PARSE_REAL(u_init);
+            PARSE_REAL(v_init);
+            PARSE_REAL(p_init);
+
+            /* Added new particle tracing parameters */
+            PARSE_INT(numberOfParticles);
+            PARSE_REAL(startTime);
+            PARSE_REAL(injectTimePeriod);
+            PARSE_REAL(writeTimePeriod);
+            PARSE_REAL(x1);
+            PARSE_REAL(y1);
+            PARSE_REAL(x2);
+            PARSE_REAL(y2);
+
+            /* Added obstacle geometry parameters */
+            PARSE_INT(shape);
+            PARSE_REAL(xCenter);
+            PARSE_REAL(yCenter);
+            PARSE_REAL(xRectLength);
+            PARSE_REAL(yRectLength);
+            PARSE_REAL(circleRadius);
+        }
+    }
+
+    fclose(fp);
+}
+
+void printParameter(Parameter* param)
+{
+    printf("Parameters for %s\n", param->name);
+    printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n",
+        param->bcLeft,
+        param->bcRight,
+        param->bcBottom,
+        param->bcTop);
+    printf("\tReynolds number: %.2f\n", param->re);
+    printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n",
+        param->u_init,
+        param->v_init,
+        param->p_init);
+    printf("Geometry data:\n");
+    printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength);
+    printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax);
+    printf("Timestep parameters:\n");
+    printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te);
+    printf("\tTau factor: %.2f\n", param->tau);
+    printf("Iterative solver parameters:\n");
+    printf("\tMax iterations: %d\n", param->itermax);
+    printf("\tepsilon (stopping tolerance) : %f\n", param->eps);
+    printf("\tgamma (stopping tolerance) : %f\n", param->gamma);
+    printf("\tomega (SOR relaxation): %f\n", param->omg);
+    printf("Particle tracer parameters:\n");
+    printf("\tNumber of particles: %d\n", param->numberOfParticles);
+    printf("\tstartTime : %f\n", param->startTime);
+    printf("\tinjecTimePeriod : %f\n", param->injectTimePeriod);
+    printf("\twriteTimePeriod: %f\n", param->writeTimePeriod);
+    printf("\tx1 : %f, x2 : %f, y1 : %f, y2 : %f\n",
+        param->x1,
+        param->x2,
+        param->y1,
+        param->y2);
+    printf("\tShape : %d\n", param->shape);
+}

EnhancedSolver/2D-mpi/src/parameter.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#ifndef __PARAMETER_H_
+#define __PARAMETER_H_
+
+typedef struct {
+    double xlength, ylength;
+    int imax, jmax;
+    int itermax;
+    double eps, omg;
+    double re, tau, gamma;
+    double te, dt;
+    double gx, gy;
+    char* name;
+    int bcLeft, bcRight, bcBottom, bcTop;
+    double u_init, v_init, p_init;
+    
+    int levels, presmooth, postsmooth;
+
+    int numberOfParticles;
+    double startTime, injectTimePeriod, writeTimePeriod;
+
+    double x1, y1, x2, y2;
+
+    int shape;
+    double xCenter, yCenter, xRectLength, yRectLength, circleRadius;
+
+
+} Parameter;
+
+void initParameter(Parameter*);
+void readParameter(Parameter*, const char*);
+void printParameter(Parameter*);
+#endif

EnhancedSolver/2D-mpi/src/particletracing.c

@@ -0,0 +1,675 @@
+/*
+ * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "particletracing.h"
+
+#define U(i, j) u[(j) * (imaxLocal + 2) + (i)]
+#define V(i, j) v[(j) * (imaxLocal + 2) + (i)]
+#define S(i, j) s[(j) * (imaxLocal + 2) + (i)]
+
+static int ts = 0;
+
+#define IDIM 0
+#define JDIM 1
+
+#define XOFFSET    0
+#define YOFFSET    1
+#define XOFFSETEND 2
+#define YOFFSETEND 3
+
+static double sum(int* sizes, int size)
+{
+    double sum = 0;
+
+    for (int i = 0; i < size; ++i) {
+        sum += sizes[i];
+    }
+
+    return sum;
+}
+
+void printUV(ParticleTracer* particletracer, double* u, double* v)
+{
+    int imaxLocal = particletracer->imaxLocal;
+
+    for (int i = 0; i < particletracer->size; i++) {
+        if (i == particletracer->rank) {
+            printf(
+                "\n### RANK %d #######################################################\n",
+                particletracer->rank);
+            printf("\nGrid U : \n");
+            for (int j = 0; j < particletracer->jmaxLocal + 2; j++) {
+                printf("%02d: ", j);
+                for (int i = 0; i < particletracer->imaxLocal + 2; i++) {
+                    printf("%4.2f  ", u[j * (imaxLocal + 2) + i]);
+                }
+                printf("\n");
+            }
+            fflush(stdout);
+            printf("\nGrid V : \n");
+            for (int j = 0; j < particletracer->jmaxLocal + 2; j++) {
+                printf("%02d: ", j);
+                for (int i = 0; i < particletracer->imaxLocal + 2; i++) {
+                    printf("%4.2f  ", v[j * (imaxLocal + 2) + i]);
+                }
+                printf("\n");
+            }
+            fflush(stdout);
+        }
+
+#ifdef _MPI
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+    }
+}
+
+void initParticleTracer(
+    ParticleTracer* particletracer, Discretization* d, Parameter* params)
+{
+    int dims[NDIMS]    = { 0, 0 };
+    int periods[NDIMS] = { 0, 0 };
+
+    /* initializing local properties from params */
+    particletracer->rank = d->comm.rank;
+    particletracer->size = d->comm.size;
+
+    particletracer->numberOfParticles = params->numberOfParticles;
+    particletracer->startTime         = params->startTime;
+    particletracer->injectTimePeriod  = params->injectTimePeriod;
+    particletracer->writeTimePeriod   = params->writeTimePeriod;
+
+    particletracer->dt = params->dt;
+    particletracer->dx = params->xlength / params->imax;
+    particletracer->dy = params->ylength / params->jmax;
+
+    particletracer->xlength = params->xlength;
+    particletracer->ylength = params->ylength;
+
+    particletracer->x1 = params->x1;
+    particletracer->y1 = params->y1;
+    particletracer->x2 = params->x2;
+    particletracer->y2 = params->y2;
+
+    particletracer->lastInjectTime = params->startTime;
+    particletracer->lastUpdateTime = params->startTime;
+    particletracer->lastWriteTime  = params->startTime;
+
+    particletracer->pointer          = 0;
+    particletracer->totalParticles   = 0;
+    particletracer->removedParticles = 0;
+
+    particletracer->imax = params->imax;
+    particletracer->jmax = params->jmax;
+
+    particletracer->imaxLocal        = d->comm.imaxLocal;
+    particletracer->jmaxLocal        = d->comm.jmaxLocal;
+    particletracer->removedParticles = 0;
+
+    // Estimating the number of particles over the number of timesteps that could be
+    // required.
+    particletracer->estimatedNumParticles = (particletracer->imaxLocal *
+                                             particletracer->jmaxLocal);
+
+    // Allocating memory for the estimated particles over the timesteps.
+    particletracer->particlePool = malloc(
+        sizeof(Particle) * particletracer->estimatedNumParticles);
+
+    // Initializing the number of particles to 0 and turning OFF all of the particles.
+    // Contain information in x and y length metric, not i and j discretization metrics.
+    for (int i = 0; i < particletracer->estimatedNumParticles; ++i) {
+        particletracer->particlePool[i].x    = 0.0;
+        particletracer->particlePool[i].y    = 0.0;
+        particletracer->particlePool[i].flag = false;
+    }
+
+    // Creating a linearly spaced particle line.
+    particletracer->linSpaceLine = malloc(
+        sizeof(Particle) * particletracer->numberOfParticles);
+
+    // Creating an array for each rank that will hold information about
+    // offsets from other ranks. Holds each ranks x and y length metrics.
+    particletracer->offset = (double*)malloc(sizeof(double) * 4 * particletracer->size);
+
+    // Calculating each ranks local x and y length metrics.
+
+    double offset[4][particletracer->size];
+
+    // Calculating each ranks x and y local lengths.
+    particletracer->xLocal = d->xLocal;
+    particletracer->yLocal = d->yLocal;
+
+    double xLocal[particletracer->size];
+    double yLocal[particletracer->size];
+
+    // Calculate own x and y length metric offset based on other ranks offset data.
+    particletracer->xOffset    = d->xOffset;
+    particletracer->yOffset    = d->yOffset;
+    particletracer->xOffsetEnd = d->xOffsetEnd;
+    particletracer->yOffsetEnd = d->yOffsetEnd;
+
+    printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : "
+           "%.2f\n",
+        particletracer->rank,
+        particletracer->xOffset,
+        particletracer->yOffset,
+        particletracer->xOffsetEnd,
+        particletracer->yOffsetEnd);
+
+#ifdef _MPI
+    // Gather each ranks x and y length metric that marks each ranks own territory.
+    // Once the boundary leaves local domain, then it needs to know which ranks to send.
+    // And to know whos boundary it is, we need to know the rank.
+    MPI_Allgather(&particletracer->xOffset,
+        1,
+        MPI_DOUBLE,
+        offset[0],
+        1,
+        MPI_DOUBLE,
+        d->comm.comm);
+    MPI_Allgather(&particletracer->yOffset,
+        1,
+        MPI_DOUBLE,
+        offset[1],
+        1,
+        MPI_DOUBLE,
+        d->comm.comm);
+    MPI_Allgather(&particletracer->xOffsetEnd,
+        1,
+        MPI_DOUBLE,
+        offset[2],
+        1,
+        MPI_DOUBLE,
+        d->comm.comm);
+    MPI_Allgather(&particletracer->yOffsetEnd,
+        1,
+        MPI_DOUBLE,
+        offset[3],
+        1,
+        MPI_DOUBLE,
+        d->comm.comm);
+#endif
+
+    memcpy(particletracer->offset, offset, sizeof(offset));
+
+    particleRandomizer(particletracer);
+
+#ifdef _MPI
+    // Create the mpi_particle datatype
+    MPI_Datatype mpi_particle;
+    int lengths[3] = { 1, 1, 1 };
+
+    MPI_Aint displacements[3];
+    Particle dummy_particle;
+    MPI_Aint base_address;
+    MPI_Get_address(&dummy_particle, &base_address);
+    MPI_Get_address(&dummy_particle.x, &displacements[0]);
+    MPI_Get_address(&dummy_particle.y, &displacements[1]);
+    MPI_Get_address(&dummy_particle.flag, &displacements[2]);
+    displacements[0] = MPI_Aint_diff(displacements[0], base_address);
+    displacements[1] = MPI_Aint_diff(displacements[1], base_address);
+    displacements[2] = MPI_Aint_diff(displacements[2], base_address);
+
+    MPI_Datatype types[3] = { MPI_DOUBLE, MPI_DOUBLE, MPI_C_BOOL };
+    MPI_Type_create_struct(3,
+        lengths,
+        displacements,
+        types,
+        &particletracer->mpi_particle);
+    MPI_Type_commit(&particletracer->mpi_particle);
+#endif
+}
+
+void printParticles(ParticleTracer* particletracer)
+{
+    for (int i = 0; i < particletracer->totalParticles; ++i) {
+        printf("Rank : %d Particle position X : %.2f, Y : %.2f, flag : %d, total pt : "
+               "%d, pointer : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, "
+               "yOffsetEnd : %.2f\n",
+            particletracer->rank,
+            particletracer->particlePool[i].x,
+            particletracer->particlePool[i].y,
+            particletracer->particlePool[i].flag,
+            particletracer->totalParticles,
+            particletracer->pointer,
+            particletracer->xOffset,
+            particletracer->yOffset,
+            particletracer->xOffsetEnd,
+            particletracer->yOffsetEnd);
+    }
+}
+void injectParticles(ParticleTracer* particletracer, double* s)
+{
+    double x, y;
+    compress(particletracer);
+    particleRandomizer(particletracer);
+
+    int imaxLocal = particletracer->imaxLocal;
+    int jmaxLocal = particletracer->jmaxLocal;
+
+    for (int i = 0; i < particletracer->numberOfParticles; ++i) {
+        x = particletracer->linSpaceLine[i].x;
+        y = particletracer->linSpaceLine[i].y;
+        if (x >= particletracer->xOffset && y >= particletracer->yOffset &&
+            x <= particletracer->xOffsetEnd && y <= particletracer->yOffsetEnd) {
+
+            particletracer->particlePool[particletracer->pointer].x = x;
+            particletracer->particlePool[particletracer->pointer].y = y;
+
+            int i = particletracer->particlePool[particletracer->pointer].x /
+                    particletracer->dx;
+            int j = particletracer->particlePool[particletracer->pointer].y /
+                    particletracer->dy;
+
+            int iOffset = particletracer->xOffset / particletracer->dx,
+                jOffset = particletracer->yOffset / particletracer->dy;
+
+            if (S(i - iOffset, j - jOffset) == FLUID) {
+                particletracer->particlePool[particletracer->pointer].flag = true;
+                ++(particletracer->pointer);
+                ++(particletracer->totalParticles);
+            }
+        }
+    }
+}
+
+void advanceParticles(ParticleTracer* particletracer,
+    double* restrict u,
+    double* restrict v,
+    double* restrict s,
+    Comm* comm,
+    double time)
+{
+
+    int imax      = particletracer->imax;
+    int jmax      = particletracer->jmax;
+    int imaxLocal = particletracer->imaxLocal;
+    int jmaxLocal = particletracer->jmaxLocal;
+
+    double dx = particletracer->dx;
+    double dy = particletracer->dy;
+
+    double xlength = particletracer->xlength;
+    double ylength = particletracer->ylength;
+
+    Particle buff[particletracer->size][(particletracer->estimatedNumParticles)];
+    memset(buff, 0, sizeof(buff));
+    Particle recvbuff[particletracer->size][(particletracer->estimatedNumParticles)];
+    memset(buff, 0, sizeof(recvbuff));
+
+    int particleBufIndex[particletracer->size],
+        recvparticleBufIndex[particletracer->size];
+
+    memset(particleBufIndex, 0, sizeof(particleBufIndex));
+    memset(recvparticleBufIndex, 0, sizeof(recvparticleBufIndex));
+
+    for (int i = 0; i < particletracer->totalParticles; ++i) {
+        if (particletracer->particlePool[i].flag == true) {
+            double xTemp = particletracer->particlePool[i].x;
+            double yTemp = particletracer->particlePool[i].y;
+
+            double x = xTemp - particletracer->xOffset;
+            double y = yTemp - particletracer->yOffset;
+
+            int iCoord = (int)(x / dx) + 1;
+            int jCoord = (int)((y + 0.5 * dy) / dy) + 1;
+
+            double x1 = (double)(iCoord - 1) * dx;
+            double y1 = ((double)(jCoord - 1) - 0.5) * dy;
+            double x2 = (double)iCoord * dx;
+            double y2 = ((double)jCoord - 0.5) * dy;
+
+            double u_n = (1.0 / (dx * dy)) *
+                         ((x2 - x) * (y2 - y) * U(iCoord - 1, jCoord - 1) +
+                             (x - x1) * (y2 - y) * U(iCoord, jCoord - 1) +
+                             (x2 - x) * (y - y1) * U(iCoord - 1, jCoord) +
+                             (x - x1) * (y - y1) * U(iCoord, jCoord));
+
+            double new_x = (x + particletracer->xOffset) + particletracer->dt * u_n;
+            particletracer->particlePool[i].x = new_x;
+
+            iCoord = (int)((x + 0.5 * dx) / dx) + 1;
+            jCoord = (int)(y / dy) + 1;
+
+            x1 = ((double)(iCoord - 1) - 0.5) * dx;
+            y1 = (double)(jCoord - 1) * dy;
+            x2 = ((double)iCoord - 0.5) * dx;
+            y2 = (double)jCoord * dy;
+
+            double v_n = (1.0 / (dx * dy)) *
+                         ((x2 - x) * (y2 - y) * V(iCoord - 1, jCoord - 1) +
+                             (x - x1) * (y2 - y) * V(iCoord, jCoord - 1) +
+                             (x2 - x) * (y - y1) * V(iCoord - 1, jCoord) +
+                             (x - x1) * (y - y1) * V(iCoord, jCoord));
+
+            double new_y = (y + particletracer->yOffset) + particletracer->dt * v_n;
+            particletracer->particlePool[i].y = new_y;
+
+            if (((new_x < particletracer->xOffset) ||
+                    (new_x > particletracer->xOffsetEnd) ||
+                    (new_y < particletracer->yOffset) ||
+                    (new_y > particletracer->yOffsetEnd))) {
+                // New logic to transfer particles to neighbouring ranks or discard the
+                // particle.
+#ifdef _MPI
+                for (int i = 0; i < particletracer->size; ++i) {
+                    if ((new_x >=
+                            particletracer->offset[i + particletracer->size * XOFFSET]) &&
+                        (new_x <= particletracer
+                                      ->offset[i + particletracer->size * XOFFSETEND]) &&
+                        (new_y >=
+                            particletracer->offset[i + particletracer->size * YOFFSET]) &&
+                        (new_y <= particletracer
+                                      ->offset[i + particletracer->size * YOFFSETEND]) &&
+                        i != particletracer->rank) {
+                        buff[i][particleBufIndex[i]].x    = new_x;
+                        buff[i][particleBufIndex[i]].y    = new_y;
+                        buff[i][particleBufIndex[i]].flag = true;
+                        ++particleBufIndex[i];
+                    }
+                }
+#endif
+                particletracer->particlePool[i].flag = false;
+                particletracer->removedParticles++;
+            }
+
+            int i_new = new_x / dx, j_new = new_y / dy;
+            int iOffset = particletracer->xOffset / dx,
+                jOffset = particletracer->yOffset / dy;
+
+            if (S(i_new - iOffset, j_new - jOffset) != FLUID) {
+                particletracer->particlePool[i].flag = false;
+                particletracer->removedParticles++;
+            }
+        }
+    }
+
+#ifdef _MPI
+    for (int i = 0; i < particletracer->size; ++i) {
+        if (i != comm->rank) {
+            MPI_Send(&particleBufIndex[i], 1, MPI_INT, i, 0, comm->comm);
+        }
+    }
+    for (int i = 0; i < particletracer->size; ++i) {
+        if (i != particletracer->rank) {
+            MPI_Recv(&recvparticleBufIndex[i],
+                1,
+                MPI_INT,
+                i,
+                0,
+                comm->comm,
+                MPI_STATUS_IGNORE);
+
+            // if (0 !=recvparticleBufIndex[i]) {
+            //     printf("Rank %d will receive %d particles from rank %d\n",
+            //         particletracer->rank,
+            //         recvparticleBufIndex[i],
+            //         i);
+            // }
+        }
+    }
+
+    for (int i = 0; i < particletracer->size; ++i) {
+        if (i != particletracer->rank) {
+            MPI_Send(buff[i],
+                particleBufIndex[i],
+                particletracer->mpi_particle,
+                i,
+                0,
+                comm->comm);
+        }
+    }
+    for (int i = 0; i < particletracer->size; ++i) {
+        if (i != particletracer->rank) {
+            MPI_Recv(recvbuff[i],
+                recvparticleBufIndex[i],
+                particletracer->mpi_particle,
+                i,
+                0,
+                comm->comm,
+                MPI_STATUS_IGNORE);
+        }
+    }
+    for (int i = 0; i < particletracer->size; ++i) {
+        if (i != particletracer->rank) {
+            for (int j = 0; j < recvparticleBufIndex[i]; ++j) {
+                particletracer->particlePool[particletracer->pointer].x = recvbuff[i][j]
+                                                                              .x;
+                particletracer->particlePool[particletracer->pointer].y = recvbuff[i][j]
+                                                                              .y;
+                particletracer->particlePool[particletracer->pointer].flag = true;
+                ++(particletracer->pointer);
+                ++(particletracer->totalParticles);
+            }
+        }
+    }
+#endif
+}
+
+void freeParticles(ParticleTracer* particletracer)
+{
+    free(particletracer->particlePool);
+    free(particletracer->linSpaceLine);
+    free(particletracer->offset);
+}
+
+void writeParticles(ParticleTracer* particletracer, Comm* comm)
+{
+    int collectedBuffIndex[particletracer->size];
+    compress(particletracer);
+#ifdef _MPI
+
+    MPI_Gather(&particletracer->totalParticles,
+        1,
+        MPI_INT,
+        collectedBuffIndex,
+        1,
+        MPI_INT,
+        0,
+        comm->comm);
+
+    if (particletracer->rank != 0) {
+        Particle buff[particletracer->totalParticles];
+        for (int i = 0; i < particletracer->totalParticles; ++i) {
+            buff[i].x    = particletracer->particlePool[i].x;
+            buff[i].y    = particletracer->particlePool[i].y;
+            buff[i].flag = particletracer->particlePool[i].flag;
+            // printf("Rank : %d sending to rank 0 X : %.2f, Y : %.2f with totalpt :
+            // %d\n", particletracer->rank, buff[i].x, buff[i].y,
+            // particletracer->totalParticles);
+        }
+        MPI_Send(buff,
+            particletracer->totalParticles,
+            particletracer->mpi_particle,
+            0,
+            1,
+            comm->comm);
+    }
+#endif
+
+    if (particletracer->rank == 0) {
+        char filename[50];
+        FILE* fp;
+
+        snprintf(filename, 50, "vis_files/particles_%d.dat", ts);
+        fp = fopen(filename, "w");
+
+        if (fp == NULL) {
+            printf("Error!\n");
+            exit(EXIT_FAILURE);
+        }
+        // fprintf(fp, "# vtk DataFile Version 3.0\n");
+        // fprintf(fp, "PAMPI cfd solver particle tracing file\n");
+        // fprintf(fp, "ASCII\n");
+
+        // fprintf(fp, "DATASET UNSTRUCTURED_GRID\n");
+        // fprintf(fp, "FIELD FieldData 2\n");
+        // fprintf(fp, "TIME 1 1 double\n");
+        // fprintf(fp, "%d\n", ts);
+        // fprintf(fp, "CYCLE 1 1 int\n");
+        // fprintf(fp, "1\n");
+
+#ifdef _MPI
+        int overallTotalParticles = sum(collectedBuffIndex, particletracer->size);
+
+        // fprintf(fp, "POINTS %d float\n", overallTotalParticles);
+
+        // printf("Total particles : %d\n", overallTotalParticles);
+        for (int i = 1; i < particletracer->size; ++i) {
+            Particle recvBuff[collectedBuffIndex[i]];
+            MPI_Recv(&recvBuff,
+                collectedBuffIndex[i],
+                particletracer->mpi_particle,
+                i,
+                1,
+                comm->comm,
+                MPI_STATUS_IGNORE);
+
+            for (int j = 0; j < collectedBuffIndex[i]; ++j) {
+                double x = recvBuff[j].x;
+                double y = recvBuff[j].y;
+                fprintf(fp, "%f %f\n", x, y);
+
+                // printf("Rank : 0 receiving from rank %d X : %.2f, Y : %.2f with totalpt
+                // : %d\n", i, x, y, particletracer->totalParticles);
+            }
+        }
+#else
+        int overallTotalParticles = particletracer->totalParticles;
+
+        // fprintf(fp, "POINTS %d float\n", overallTotalParticles);
+
+        // printf("Total particles : %d\n", overallTotalParticles);
+#endif
+        for (int i = 0; i < particletracer->totalParticles; ++i) {
+            double x = particletracer->particlePool[i].x;
+            double y = particletracer->particlePool[i].y;
+            fprintf(fp, "%f %f\n", x, y);
+        }
+
+        // fprintf(fp, "CELLS %d %d\n", overallTotalParticles, 2 * overallTotalParticles);
+
+        // for (int i = 0; i < overallTotalParticles; ++i)
+        // {
+        //     fprintf(fp, "1 %d\n", i);
+        // }
+
+        // fprintf(fp, "CELL_TYPES %d\n", overallTotalParticles);
+
+        // for (int i = 0; i < overallTotalParticles; ++i)
+        // {
+        //     fprintf(fp, "1\n");
+        // }
+
+        fclose(fp);
+    }
+
+    ++ts;
+}
+
+void printParticleTracerParameters(ParticleTracer* particletracer)
+{
+    printf("Particle Tracing data:\n");
+    printf("Rank : %d\n", particletracer->rank);
+    printf("\tNumber of particles : %d being injected for every period of %.2f\n",
+        particletracer->numberOfParticles,
+        particletracer->injectTimePeriod);
+    printf("\tstartTime : %.2f\n", particletracer->startTime);
+    printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : "
+           "%.2f, x2 : %.2f, y2 : %.2f\n",
+        particletracer->x1,
+        particletracer->y1,
+        particletracer->x2,
+        particletracer->y2);
+    printf("\tPointer : %d, TotalParticles : %d\n",
+        particletracer->pointer,
+        particletracer->totalParticles);
+    printf("\tdt : %.2f, dx : %.2f, dy : %.2f\n",
+        particletracer->dt,
+        particletracer->dx,
+        particletracer->dy);
+    printf("\txOffset : %.2f, yOffset : %.2f\n",
+        particletracer->xOffset,
+        particletracer->yOffset);
+    printf("\txOffsetEnd : %.2f, yOffsetEnd : %.2f\n",
+        particletracer->xOffsetEnd,
+        particletracer->yOffsetEnd);
+    printf("\txLocal : %.2f, yLocal : %.2f\n",
+        particletracer->xLocal,
+        particletracer->yLocal);
+}
+
+void trace(ParticleTracer* particletracer, Discretization* d, double time)
+{
+    if (time >= particletracer->startTime) {
+        if ((time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod) {
+            injectParticles(particletracer, d->grid.s);
+            particletracer->lastInjectTime = time;
+        }
+        if ((time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod) {
+            writeParticles(particletracer, &d->comm);
+            particletracer->lastWriteTime = time;
+        }
+        advanceParticles(particletracer, d->u, d->v, d->grid.s, &d->comm, time);
+
+        if (particletracer->removedParticles > (particletracer->totalParticles * 0.2)) {
+            compress(particletracer);
+        }
+
+        particletracer->lastUpdateTime = time;
+    }
+}
+
+void compress(ParticleTracer* particletracer)
+{
+    Particle* memPool = particletracer->particlePool;
+    Particle tempPool[particletracer->totalParticles];
+    int totalParticles = 0;
+
+    // printf("\nPerforming compression ...");
+
+    for (int i = 0; i < particletracer->totalParticles; ++i) {
+        if (memPool[i].flag == true) {
+            tempPool[totalParticles].x    = memPool[i].x;
+            tempPool[totalParticles].y    = memPool[i].y;
+            tempPool[totalParticles].flag = memPool[i].flag;
+            ++totalParticles;
+        }
+    }
+
+    // printf(" remove %d particles\n", particletracer->totalParticles - totalParticles);
+
+    particletracer->totalParticles   = totalParticles;
+    particletracer->removedParticles = 0;
+    particletracer->pointer          = totalParticles;
+
+    memcpy(particletracer->particlePool, tempPool, totalParticles * sizeof(Particle));
+}
+
+void particleRandomizer(ParticleTracer* particletracer)
+{
+    memset(particletracer->linSpaceLine,
+        0,
+        sizeof(Particle) * particletracer->numberOfParticles);
+
+    for (int i = 0; i < particletracer->numberOfParticles; ++i) {
+
+        particletracer->linSpaceLine[i].x = (((double)rand() / RAND_MAX) *
+                                                (particletracer->x2 -
+                                                    particletracer->x1)) +
+                                            particletracer->x1;
+        particletracer->linSpaceLine[i].y = (((double)rand() / RAND_MAX) *
+                                                (particletracer->y2 -
+                                                    particletracer->y1)) +
+                                            particletracer->y1;
+        particletracer->linSpaceLine[i].flag = true;
+    }
+}

EnhancedSolver/2D-mpi/src/particletracing.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#ifndef __PARTICLETRACING_H_
+#define __PARTICLETRACING_H_
+#include "allocate.h"
+#include "parameter.h"
+
+#include "solver.h"
+#include <mpi.h>
+#include <stdbool.h>
+
+#define NDIMS 2
+
+typedef enum COORD { X = 0, Y, NCOORD } COORD;
+
+typedef struct {
+    double x, y;
+    bool flag;
+} Particle;
+
+typedef struct {
+    int numberOfParticles, removedParticles, totalParticles;
+    double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime,
+        lastWriteTime;
+
+    int estimatedNumParticles;
+
+    double dx, dy, dt;
+    Particle* linSpaceLine;
+    Particle* particlePool;
+
+    int pointer;
+
+    double imax, jmax, xlength, ylength, imaxLocal, jmaxLocal;
+
+    double x1, y1, x2, y2;
+
+    int size, rank;
+    
+#ifdef _MPI
+    MPI_Datatype mpi_particle;
+#endif
+
+    double xLocal, yLocal, xOffset, yOffset, xOffsetEnd, yOffsetEnd;
+
+    double* offset;
+
+} ParticleTracer;
+
+extern void initParticleTracer(ParticleTracer*, Discretization*, Parameter*);
+extern void injectParticles(ParticleTracer*, double*);
+extern void advanceParticles(ParticleTracer*, double*, double*, double*, Comm*, double);
+extern void freeParticles(ParticleTracer*);
+extern void writeParticles(ParticleTracer*, Comm*);
+extern void printParticleTracerParameters(ParticleTracer*);
+extern void printParticles(ParticleTracer*);
+extern void trace(ParticleTracer*, Discretization*, double);
+extern void compress(ParticleTracer*);
+extern void particleRandomizer(ParticleTracer*);
+
+#endif

EnhancedSolver/2D-mpi/src/progress.c

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <math.h>
+#include <mpi.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "progress.h"
+
+static double _end;
+static int _current;
+
+void initProgress(double end)
+{
+    _end     = end;
+    _current = 0;
+
+    printf("[          ]");
+    fflush(stdout);
+}
+
+void printProgress(double current)
+{
+    int new = (int)rint((current / _end) * 10.0);
+
+    if (new > _current) {
+        char progress[11];
+        _current    = new;
+        progress[0] = 0;
+
+        for (int i = 0; i < 10; i++) {
+            if (i < _current) {
+                sprintf(progress + strlen(progress), "#");
+            } else {
+                sprintf(progress + strlen(progress), " ");
+            }
+        }
+        printf("\r[%s]", progress);
+    }
+    fflush(stdout);
+}
+
+void stopProgress()
+{
+    printf("\n");
+    fflush(stdout);
+}
+
+FILE* initResidualWriter()
+{
+    FILE* fp;
+    fp = fopen("residual.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(-1);
+    }
+
+    return fp;
+
+}
+
+void writeResidual(FILE* fp, double ts, double res)
+{
+    fprintf(fp, "%f, %f\n", ts, res);
+}