Fully working BasicSolver

BasicSolver/2D-mpi/Makefile

@@ -39,9 +39,20 @@ $(BUILD_DIR)/%.s:  %.c
 	$(info ===>  GENERATE ASM  $@)
 	$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
 
-.PHONY: clean distclean tags info asm format
+.PHONY: clean distclean vis vis_clean tags info asm format
 
-clean:
+vis:
+	$(info ===>  GENERATE VISUALIZATION)
+	@gnuplot -e "filename='pressure.dat'" ./surface.plot
+	@gnuplot -e "filename='velocity.dat'" ./vector.plot
+	@gnuplot -e "filename='residual.dat'" ./residual.plot
+
+vis_clean:
+	$(info ===>  CLEAN VISUALIZATION)
+	@rm -f *.dat
+	@rm -f *.png
+
+clean: vis_clean
 	$(info ===>  CLEAN)
 	@rm -rf $(BUILD_DIR)
 	@rm -f tags

BasicSolver/2D-mpi/config.mk

@@ -1,10 +1,11 @@
 # Supported: GCC, CLANG, ICC
 TAG ?= ICC
+# Supported: true, false
 ENABLE_MPI ?= true
 ENABLE_OPENMP ?= false
 # Supported: rb, mg
 SOLVER ?= mg
-# Run in debug settings ?= mg
+# Supported: v1, v2, v3
 COMM_TYPE ?= v3
 
 #Feature options

BasicSolver/2D-mpi/dcavity.par

@@ -33,14 +33,14 @@ jmax       128		# number of interior cells in y-direction
 # ---------
 
 te      10.0		# final time
-dt     0.02	    # time stepsize
+dt      0.02	    # time stepsize
-tau     0.5		# safety factor for time stepsize control (<0 constant delt)
+tau     0.5		    # safety factor for time stepsize control (<0 constant delt)
 
 # Multigrid data:
 # ---------
 
 levels        2         # Multigrid levels
-presmooth     20         # Pre-smoothning iterations
+presmooth     20        # Pre-smoothning iterations
 postsmooth    5         # Post-smoothning iterations
 
 # Pressure Iteration Data:
@@ -48,6 +48,6 @@ postsmooth    5         # Post-smoothning iterations
 
 itermax  1000		# maximal number of pressure iteration in one time step
 eps      0.001		# stopping tolerance for pressure iteration
-omg      1.9		# relaxation parameter for SOR iteration
+omg      1.7		# relaxation parameter for SOR iteration
 gamma    0.9		# upwind differencing factor gamma
 #===============================================================================

BasicSolver/2D-mpi/residual.plot

@@ -0,0 +1,9 @@
+set terminal png size 1800,768 enhanced font ,12
+set output 'residual.png'
+set datafile separator whitespace
+set xlabel "Timestep"
+set ylabel "Residual"
+
+set logscale y 2
+
+plot 'residual.dat' using 1:2 title "Residual"

BasicSolver/2D-mpi/src/comm-v1.c

@@ -14,7 +14,7 @@ static int sum(int* sizes, int position)
 {
     int sum = 0;
 
-    for (int i = 0; i < position; i++) {
+    for (int i = 0; i < position; i += position) {
         sum += sizes[i];
     }
 
@@ -67,6 +67,7 @@ int commIsBoundary(Comm* c, int direction)
 
 void commExchange(Comm* c, double* grid)
 {
+    // printf("Rank : %d In exchange \n", c->rank);
 #ifdef _MPI
     MPI_Request requests[4] = { MPI_REQUEST_NULL,
         MPI_REQUEST_NULL,
@@ -174,6 +175,17 @@ void commPartition(Comm* c, int jmax, int imax)
 #ifdef _MPI
     c->imaxLocal = imax;
     c->jmaxLocal = sizeOfRank(c->coords[JDIM], c->size, jmax);
+
+    c->neighbours[BOTTOM] = c->rank == 0 ? -1 : c->rank - 1;
+    c->neighbours[TOP]    = c->rank == (c->size - 1) ? -1 : c->rank + 1;
+    c->neighbours[LEFT]   = -1;
+    c->neighbours[RIGHT]  = -1;
+
+    c->coords[IDIM] = 0;
+    c->coords[JDIM] = c->rank;
+
+    c->dims[IDIM] = 1;
+    c->dims[JDIM] = c->size;
 #else
     c->imaxLocal = imax;
     c->jmaxLocal = jmax;
@@ -182,15 +194,33 @@ void commPartition(Comm* c, int jmax, int imax)
 
 void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal)
 {
-    Comm* newcomm;
 
 #if defined _MPI
     newcomm->comm = MPI_COMM_NULL;
-    int result    = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
+    int result    = MPI_Comm_dup(MPI_COMM_WORLD, &newcomm->comm);
 
     if (result == MPI_ERR_COMM) {
         printf("\nNull communicator. Duplication failed !!\n");
     }
+
+    newcomm->rank = oldcomm->rank;
+    newcomm->size = oldcomm->size;
+
+    newcomm->imaxLocal = imaxLocal / 2;
+    newcomm->jmaxLocal = jmaxLocal / 2;
+
+    newcomm->neighbours[BOTTOM] = newcomm->rank == 0 ? -1 : newcomm->rank - 1;
+    newcomm->neighbours[TOP]    = newcomm->rank == (newcomm->size - 1) ? -1 : newcomm->rank + 1;
+    newcomm->neighbours[LEFT]   = -1;
+    newcomm->neighbours[RIGHT]  = -1;
+
+    newcomm->coords[IDIM] = 0;
+    newcomm->coords[JDIM] = newcomm->rank;
+
+    newcomm->dims[IDIM] = 1;
+    newcomm->dims[JDIM] = newcomm->size;
+
+
 #endif
     newcomm->imaxLocal = imaxLocal;
     newcomm->jmaxLocal = jmaxLocal;

BasicSolver/2D-mpi/src/comm-v2.c

@@ -4,18 +4,18 @@
  * Use of this source code is governed by a MIT style
  * license that can be found in the LICENSE file.
  */
+#include "comm.h"
 #include <stdio.h>
 #include <stdlib.h>
-
+#include <string.h>
-#include "comm.h"
 
 #ifdef _MPI
 // subroutines local to this module
-static int sum(int* sizes, int position)
+static int sum(int* sizes, int init, int offset, int coord)
 {
     int sum = 0;
 
-    for (int i = 0; i < position; i++) {
+    for (int i = init - offset; coord > 0; i -= offset, --coord) {
         sum += sizes[i];
     }
 
@@ -79,8 +79,8 @@ static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax
             int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
             int coords[NDIMS];
             MPI_Cart_coords(c->comm, i, NDIMS, coords);
-            int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
+            int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]),
-                sum(newSizesI, coords[IDIM]) };
+                sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) };
             printf(
                 "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
                 "Starts(i,j): %d %d\n",
@@ -290,14 +290,21 @@ void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLo
 {
 #if defined _MPI
     newcomm->comm = MPI_COMM_NULL;
-    int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
+    int result    = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
 
     if (result == MPI_ERR_COMM) {
         printf("\nNull communicator. Duplication failed !!\n");
     }
 
-    newcomm->imaxLocal = imaxLocal;
+    newcomm->rank = oldcomm->rank;
-    newcomm->jmaxLocal = jmaxLocal;
+    newcomm->size = oldcomm->size;
+
+    memcpy(&newcomm->neighbours, &oldcomm->neighbours, sizeof(oldcomm->neighbours));
+    memcpy(&newcomm->coords, &oldcomm->coords, sizeof(oldcomm->coords));
+    memcpy(&newcomm->dims, &oldcomm->dims, sizeof(oldcomm->dims));
+
+    newcomm->imaxLocal = imaxLocal/2;
+    newcomm->jmaxLocal = jmaxLocal/2;
 
     MPI_Datatype jBufferType;
     MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
@@ -329,7 +336,7 @@ void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLo
 
 void commFreeCommunicator(Comm* comm)
 {
-    #ifdef _MPI
+#ifdef _MPI
-        MPI_Comm_free(&comm->comm);
+    MPI_Comm_free(&comm->comm);
-    #endif
+#endif
 }

BasicSolver/2D-mpi/src/comm-v3.c

@@ -11,11 +11,11 @@
 
 #ifdef _MPI
 // subroutines local to this module
-static int sum(int* sizes, int position)
+static int sum(int* sizes, int init, int offset, int coord)
 {
     int sum = 0;
 
-    for (int i = 0; i < position; i++) {
+    for (int i = init - offset; coord > 0; i -= offset, --coord) {
         sum += sizes[i];
     }
 
@@ -79,8 +79,8 @@ static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax
             int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
             int coords[NDIMS];
             MPI_Cart_coords(c->comm, i, NDIMS, coords);
-            int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
+            int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]),
-                sum(newSizesI, coords[IDIM]) };
+                sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) };
             printf(
                 "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
                 "Starts(i,j): %d %d\n",
@@ -271,14 +271,18 @@ void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLo
 {
 #if defined _MPI
 
-    int result    = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
+    int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm);
 
     if (result == MPI_ERR_COMM) {
         printf("\nNull communicator. Duplication failed !!\n");
     }
 
-    newcomm->imaxLocal = imaxLocal;
+    newcomm->rank = oldcomm->rank;
-    newcomm->jmaxLocal = jmaxLocal;
+    newcomm->size = oldcomm->size;
+    
+
+    newcomm->imaxLocal = imaxLocal / 2;
+    newcomm->jmaxLocal = jmaxLocal / 2;
 
     MPI_Datatype jBufferType;
     MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
@@ -310,7 +314,7 @@ void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLo
 
 void commFreeCommunicator(Comm* comm)
 {
-    #ifdef _MPI
+#ifdef _MPI
-        MPI_Comm_free(&comm->comm);
+    MPI_Comm_free(&comm->comm);
-    #endif
+#endif
 }

BasicSolver/2D-mpi/src/main.c

@@ -49,6 +49,9 @@ int main(int argc, char** argv)
     commInit(&d.comm, argc, argv);
     initParameter(&p);
 
+    FILE* fp;
+    if (commIsMaster(&d.comm)) fp = initResidualWriter();
+
     if (argc != 2) {
         printf("Usage: %s <configFile>\n", argv[0]);
         exit(EXIT_SUCCESS);
@@ -79,16 +82,21 @@ int main(int argc, char** argv)
     double tau = d.tau;
     double te  = d.te;
     double t   = 0.0;
+    double res = 0.0;
 
     timeStart = getTimeStamp();
     while (t <= te) {
+        
         if (tau > 0.0) computeTimestep(&d);
         setBoundaryConditions(&d);
         setSpecialBoundaryCondition(&d);
         computeFG(&d);
         computeRHS(&d);
-        solve(&s, d.p, d.rhs);
+        res = solve(&s, d.p, d.rhs);
         adaptUV(&d);
+
+        if (commIsMaster(&d.comm)) writeResidual(fp, t, res);
+
         t += d.dt;
 
 #ifdef VERBOSE
@@ -106,7 +114,7 @@ int main(int argc, char** argv)
     if (commIsMaster(s.comm)) {
         printf("Solution took %.2fs\n", timeStop - timeStart);
     }
-
+    if (commIsMaster(&d.comm)) fclose(fp);
     writeResults(&d);
     commFinalize(s.comm);
     return EXIT_SUCCESS;

BasicSolver/2D-mpi/src/progress.c

@@ -8,7 +8,7 @@
 #include <mpi.h>
 #include <stdio.h>
 #include <string.h>
-
+#include <stdlib.h>
 #include "progress.h"
 
 static double _end;
@@ -49,3 +49,22 @@ void stopProgress()
     printf("\n");
     fflush(stdout);
 }
+
+FILE* initResidualWriter()
+{
+    FILE* fp;
+    fp = fopen("residual.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return fp;
+
+}
+
+void writeResidual(FILE* fp, double ts, double res)
+{
+    fprintf(fp, "%f, %f\n", ts, res);
+}

BasicSolver/2D-mpi/src/progress.h

@@ -10,5 +10,6 @@
 extern void initProgress(double);
 extern void printProgress(double);
 extern void stopProgress();
-
+extern FILE* initResidualWriter(void);
+extern void writeResidual(FILE*, double, double);
 #endif

BasicSolver/2D-mpi/src/solver-mg.c

@@ -14,9 +14,9 @@
 #define FINEST_LEVEL   0
 #define COARSEST_LEVEL (s->levels - 1)
 // #define S(i, j)        s[(j) * (imaxLocal + 2) + (i)]
-#define E(i, j)        e[(j) * (imaxLocal + 2) + (i)]
+#define E(i, j)   e[(j) * (imaxLocal + 2) + (i)]
-#define R(i, j)        r[(j) * (imaxLocal + 2) + (i)]
+#define R(i, j)   r[(j) * (imaxLocal + 2) + (i)]
-#define OLD(i, j)      old[(j) * (imaxLocal + 2) + (i)]
+#define OLD(i, j) old[(j) * (imaxLocal + 2) + (i)]
 
 static void restrictMG(Solver* s, int level, Comm* comm)
 {
@@ -109,7 +109,6 @@ static void setBoundaryCondition(Solver* s, double* p, int imaxLocal, int jmaxLo
 #endif
 }
 
-
 static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm)
 {
     int imaxLocal = comm->imaxLocal;
@@ -144,8 +143,7 @@ static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm)
                 P(i, j) -= factor *
                            (RHS(i, j) -
                                ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
-                                   (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) *
+                                   (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2));
-                                   idy2));
             }
             isw = 3 - isw;
         }
@@ -231,10 +229,10 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm
     restrictMG(s, level, comm);
 
     Comm newcomm;
-    commUpdateDatatypes(s->comm, &newcomm, comm->imaxLocal / 2, comm->jmaxLocal / 2);
+    commUpdateDatatypes(s->comm, &newcomm, comm->imaxLocal, comm->jmaxLocal);
+
 
     // MGSolver on residual and error.
-    // TODO: What if there is a rest?
     multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm);
 
     commFreeCommunicator(&newcomm);
@@ -290,7 +288,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     }
 }
 
-void solve(Solver* s, double* p, double* rhs)
+double solve(Solver* s, double* p, double* rhs)
 {
     double res = multiGrid(s, p, rhs, 0, s->comm);
 
@@ -299,4 +297,6 @@ void solve(Solver* s, double* p, double* rhs)
         printf("Residuum: %.6f\n", res);
     }
 #endif
+
+    return res;
 }

BasicSolver/2D-mpi/src/solver-rb.c

@@ -23,7 +23,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     s->comm    = &d->comm;
 }
 
-void solve(Solver* s, double* p, double* rhs)
+double solve(Solver* s, double* p, double* rhs)
 {
     int imax      = s->grid->imax;
     int jmax      = s->grid->jmax;
@@ -101,4 +101,6 @@ void solve(Solver* s, double* p, double* rhs)
         printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
     }
 #endif
+
+    return res;
 }

BasicSolver/2D-mpi/src/solver.h

@@ -25,5 +25,5 @@ typedef struct {
 } Solver;
 
 void initSolver(Solver*, Discretization*, Parameter*);
-void solve(Solver*, double*, double*);
+double solve(Solver*, double*, double*);
 #endif

BasicSolver/2D-seq/Makefile

@@ -38,9 +38,22 @@ $(BUILD_DIR)/%.s:  %.c
 	$(info ===>  GENERATE ASM  $@)
 	$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
 
-.PHONY: clean distclean tags info asm format
+.PHONY: clean distclean vis vis_clean tags info asm format
 
-clean:
+vis:
+	$(info ===>  GENERATE VISUALIZATION)
+	@gnuplot -e "filename='pressure.dat'" ./surface.plot
+	@gnuplot -e "filename='velocity.dat'" ./vector.plot
+	@gnuplot -e "filename='residual.dat'" ./residual.plot
+
+vis_clean:
+	$(info ===>  CLEAN VISUALIZATION)
+	@rm -f *.dat
+	@rm -f *.png
+	@rm -f ./vis_files/*.dat
+	@rm -f ./vis_files/*.gif
+
+clean: vis_clean
 	$(info ===>  CLEAN)
 	@rm -rf $(BUILD_DIR)
 	@rm -f tags

BasicSolver/2D-seq/residual.plot

@@ -0,0 +1,9 @@
+set terminal png size 1800,768 enhanced font ,12
+set output 'residual.png'
+set datafile separator whitespace
+set xlabel "Timestep"
+set ylabel "Residual"
+
+set logscale y 2
+
+plot 'residual.dat' using 1:2 title "Residual"

BasicSolver/2D-seq/src/main.c

@@ -22,6 +22,8 @@ int main(int argc, char** argv)
     Solver s;
 
     initParameter(&p);
+    FILE* fp;
+    fp = initResidualWriter();
 
     if (argc != 2) {
         printf("Usage: %s <configFile>\n", argv[0]);
@@ -41,8 +43,9 @@ int main(int argc, char** argv)
     double te  = d.te;
     double t   = 0.0;
     int nt     = 0;
+    double res = 0.0;
 
-    timeStart = getTimeStamp();
+    timeStart  = getTimeStamp();
     while (t <= te) {
         if (tau > 0.0) computeTimestep(&d);
         setBoundaryConditions(&d);
@@ -50,8 +53,11 @@ int main(int argc, char** argv)
         computeFG(&d);
         computeRHS(&d);
         if (nt % 100 == 0) normalizePressure(&d);
-        solve(&s, d.p, d.rhs);
+        res = solve(&s, d.p, d.rhs);
         adaptUV(&d);
+
+        writeResidual(fp, t, res);
+
         t += d.dt;
         nt++;
 
@@ -61,6 +67,7 @@ int main(int argc, char** argv)
         printProgress(t);
 #endif
     }
+    fclose(fp);
     timeStop = getTimeStamp();
     stopProgress();
     printf("Solution took %.2fs\n", timeStop - timeStart);

BasicSolver/2D-seq/src/progress.c

@@ -49,3 +49,22 @@ void stopProgress()
     printf("\n");
     fflush(stdout);
 }
+
+FILE* initResidualWriter()
+{
+    FILE* fp;
+    fp = fopen("residual.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return fp;
+
+}
+
+void writeResidual(FILE* fp, double ts, double res)
+{
+    fprintf(fp, "%f, %f\n", ts, res);
+}

BasicSolver/2D-seq/src/progress.h

@@ -10,5 +10,7 @@
 extern void initProgress(double);
 extern void printProgress(double);
 extern void stopProgress(void);
+extern FILE* initResidualWriter(void);
+extern void writeResidual(FILE*, double, double);
 
 #endif

BasicSolver/2D-seq/src/solver-mg.c

@@ -209,11 +209,13 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     }
 }
 
-void solve(Solver* s, double* p, double* rhs)
+double solve(Solver* s, double* p, double* rhs)
 {
     double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax);
 
 #ifdef VERBOSE
     printf("Residuum: %.6f\n", res);
 #endif
+
+return res;
 }

BasicSolver/2D-seq/src/solver-rb.c

@@ -15,7 +15,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     s->omega   = p->omg;
 }
 
-void solve(Solver* solver, double* p, double* rhs)
+double solve(Solver* solver, double* p, double* rhs)
 {
     int imax      = solver->grid->imax;
     int jmax      = solver->grid->jmax;
@@ -73,4 +73,6 @@ void solve(Solver* solver, double* p, double* rhs)
 #ifdef VERBOSE
     printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
 #endif
+
+    return res;
 }

BasicSolver/2D-seq/src/solver-sor.c

@@ -15,7 +15,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     s->omega   = p->omg;
 }
 
-void solve(Solver* solver, double* p, double* rhs)
+double solve(Solver* solver, double* p, double* rhs)
 {
     int imax      = solver->grid->imax;
     int jmax      = solver->grid->jmax;
@@ -65,4 +65,6 @@ void solve(Solver* solver, double* p, double* rhs)
 #ifdef VERBOSE
     printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
 #endif
+
+    return res;
 }

BasicSolver/2D-seq/src/solver.h

@@ -22,6 +22,6 @@ typedef struct {
 } Solver;
 
 extern void initSolver(Solver*, Discretization*, Parameter*);
-extern void solve(Solver*, double*, double*);
+extern double solve(Solver*, double*, double*);
 
 #endif

BasicSolver/3D-mpi/Makefile

@@ -43,9 +43,19 @@ $(BUILD_DIR)/%.s:  %.c
 	$(info ===>  GENERATE ASM  $@)
 	$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
 
-.PHONY: clean distclean tags info asm format
+.PHONY: clean distclean vis vis_clean tags info asm format
 
-clean:
+vis:
+	$(info ===>  GENERATE VISUALIZATION)
+	@gnuplot -e "filename='residual.dat'" ./residual.plot
+
+vis_clean:
+	$(info ===>  CLEAN VISUALIZATION)
+	@rm -f *.dat
+	@rm -f *.vtk
+	@rm -f *.png
+
+clean: vis_clean
 	$(info ===>  CLEAN)
 	@rm -rf $(BUILD_DIR)
 	@rm -f tags

BasicSolver/3D-mpi/canal.par

@@ -32,8 +32,8 @@ xlength       30.0     # domain size in x-direction
 ylength       4.0	   # domain size in y-direction
 zlength       4.0	   # domain size in z-direction
 imax          200      # number of interior cells in x-direction
-jmax          50	   # number of interior cells in y-direction
+jmax          40	   # number of interior cells in y-direction
-kmax          50	   # number of interior cells in z-direction
+kmax          40	   # number of interior cells in z-direction
 
 # Time Data:
 # ---------
@@ -45,8 +45,8 @@ tau      0.5     # safety factor for time stepsize control (<0 constant delt)
 # Multigrid data:
 # ---------
 
-levels        3         # Multigrid levels
+levels        2         # Multigrid levels
-presmooth     5         # Pre-smoothning iterations
+presmooth     20         # Pre-smoothning iterations
 postsmooth    5         # Post-smoothning iterations
 
 # Pressure Iteration Data:

BasicSolver/3D-mpi/config.mk

@@ -1,5 +1,6 @@
 # Supported: GCC, CLANG, ICC
 TAG ?= ICC
+# Supported: true, false
 ENABLE_MPI ?= true
 ENABLE_OPENMP ?= false
 # Supported: rb, mg

BasicSolver/3D-mpi/dcavity.par

@@ -18,7 +18,7 @@ gx    0.0			# Body forces (e.g. gravity)
 gy    0.0			#
 gz    0.0			#
 
-re    1000.0		    # Reynolds number
+re    1000.0		# Reynolds number
 
 u_init    0.0		# initial value for velocity in x-direction
 v_init    0.0		# initial value for velocity in y-direction
@@ -46,7 +46,7 @@ tau      0.5		# safety factor for time stepsize control (<0 constant delt)
 # ---------
 
 levels        3         # Multigrid levels
-presmooth     10         # Pre-smoothning iterations
+presmooth     20        # Pre-smoothning iterations
 postsmooth    5         # Post-smoothning iterations
 
 # Pressure Iteration Data:

BasicSolver/3D-mpi/residual.plot

@@ -0,0 +1,9 @@
+set terminal png size 1800,768 enhanced font ,12
+set output 'residual.png'
+set datafile separator whitespace
+set xlabel "Timestep"
+set ylabel "Residual"
+
+set logscale y 2
+
+plot 'residual.dat' using 1:2 title "Residual"

BasicSolver/3D-mpi/src/comm.c

@@ -167,11 +167,12 @@ static void assembleResult(Comm* c,
     MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
 }
 
-static int sum(int* sizes, int position)
+// subroutines local to this module
+static int sum(int* sizes, int init, int offset, int coord)
 {
     int sum = 0;
 
-    for (int i = 0; i < position; i++) {
+    for (int i = init - offset; coord > 0; i -= offset, --coord) {
         sum += sizes[i];
     }
 
@@ -350,9 +351,13 @@ void commCollectResult(Comm* c,
         for (int i = 0; i < c->size; i++) {
             int coords[NCORDS];
             MPI_Cart_coords(c->comm, i, NDIMS, coords);
-            offset[i * NDIMS + IDIM] = sum(imaxLocalAll, coords[ICORD]);
+            offset[i * NDIMS + IDIM] = sum(imaxLocalAll,
-            offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, coords[JCORD]);
+                i,
-            offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, coords[KCORD]);
+                c->dims[IDIM] * c->dims[JDIM],
+                coords[ICORD]);
+            offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, i, c->dims[IDIM], coords[JCORD]);
+            offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, i, 1, coords[KCORD]);
+
             printf("Rank: %d, Coords(k,j,i): %d %d %d, Size(k,j,i): %d %d %d, "
                    "Offset(k,j,i): %d %d %d\n",
                 i,
@@ -561,9 +566,9 @@ void commPartition(Comm* c, int kmax, int jmax, int imax)
     MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]);
     MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords);
 
-    c->imaxLocal = sizeOfRank(c->coords[IDIM], dims[ICORD], imax);
+    c->imaxLocal = sizeOfRank(c->coords[KDIM], dims[ICORD], imax);
     c->jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JCORD], jmax);
-    c->kmaxLocal = sizeOfRank(c->coords[KDIM], dims[KCORD], kmax);
+    c->kmaxLocal = sizeOfRank(c->coords[IDIM], dims[KCORD], kmax);
 
     // setup buffer types for communication
     setupCommunication(c, LEFT, BULK);
@@ -608,9 +613,12 @@ void commUpdateDatatypes(
         printf("\nNull communicator. Duplication failed !!\n");
     }
 
-    newcomm->imaxLocal = imaxLocal;
+    newcomm->rank = oldcomm->rank;
-    newcomm->jmaxLocal = jmaxLocal;
+    newcomm->size = oldcomm->size;
-    newcomm->kmaxLocal = kmaxLocal;
+
+    newcomm->imaxLocal = imaxLocal / 2;
+    newcomm->jmaxLocal = jmaxLocal / 2;
+    newcomm->kmaxLocal = kmaxLocal / 2;
 
     setupCommunication(newcomm, LEFT, BULK);
     setupCommunication(newcomm, LEFT, HALO);

BasicSolver/3D-mpi/src/main.c

@@ -25,6 +25,8 @@ int main(int argc, char** argv)
 
     commInit(&d.comm, argc, argv);
     initParameter(&p);
+    FILE* fp;
+    if (commIsMaster(&d.comm)) fp = initResidualWriter();
 
     if (argc != 2) {
         printf("Usage: %s <configFile>\n", argv[0]);
@@ -37,8 +39,8 @@ int main(int argc, char** argv)
     if (commIsMaster(&d.comm)) {
         printParameter(&p);
     }
-    initDiscretization(&d, &p);
     
+    initDiscretization(&d, &p);
     initSolver(&s, &d, &p);
 
 #ifndef VERBOSE
@@ -49,6 +51,7 @@ int main(int argc, char** argv)
     double te  = d.te;
     double t   = 0.0;
     int nt     = 0;
+    double res = 0.0;
 
     timeStart = getTimeStamp();
     while (t <= te) {
@@ -58,8 +61,11 @@ int main(int argc, char** argv)
         computeFG(&d);
         computeRHS(&d);
         if (nt % 100 == 0) normalizePressure(&d);
-        solve(&s, d.p, d.rhs);
+        res = solve(&s, d.p, d.rhs);
         adaptUV(&d);
+
+        if (commIsMaster(&d.comm)) writeResidual(fp, t, res);
+
         t += d.dt;
         nt++;
 
@@ -87,6 +93,8 @@ int main(int argc, char** argv)
     vtkVector(&opts, "velocity", (VtkVector) { d.u, d.v, d.w });
     vtkClose(&opts);
 #else
+    if (commIsMaster(&d.comm)) fclose(fp);
+
     double *pg, *ug, *vg, *wg;
 
     if (commIsMaster(s.comm)) {

BasicSolver/3D-mpi/src/parameter.c

@@ -69,6 +69,9 @@ void readParameter(Parameter* param, const char* filename)
             PARSE_INT(jmax);
             PARSE_INT(kmax);
             PARSE_INT(itermax);
+            PARSE_INT(levels);
+            PARSE_INT(presmooth);
+            PARSE_INT(postsmooth);
             PARSE_REAL(eps);
             PARSE_REAL(omg);
             PARSE_REAL(re);

BasicSolver/3D-mpi/src/progress.c

@@ -4,12 +4,12 @@
  * Use of this source code is governed by a MIT style
  * license that can be found in the LICENSE file.
  */
+#include "progress.h"
 #include <math.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
-#include "progress.h"
-
 static double _end;
 static int _current;
 
@@ -48,3 +48,18 @@ void stopProgress()
     printf("\n");
     fflush(stdout);
 }
+
+FILE* initResidualWriter()
+{
+    FILE* fp;
+    fp = fopen("residual.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return fp;
+}
+
+void writeResidual(FILE* fp, double ts, double res) { fprintf(fp, "%f, %f\n", ts, res); }

BasicSolver/3D-mpi/src/progress.h

@@ -4,11 +4,14 @@
  * Use of this source code is governed by a MIT-style
  * license that can be found in the LICENSE file.
  */
+
+#include <stdio.h>
+
 #ifndef __PROGRESS_H_
 #define __PROGRESS_H_
-
 extern void initProgress(double);
 extern void printProgress(double);
 extern void stopProgress(void);
-
+extern FILE* initResidualWriter(void);
+extern void writeResidual(FILE*, double, double);
 #endif

BasicSolver/3D-mpi/src/solver-mg.c

@@ -328,9 +328,9 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm
     Comm newcomm;
     commUpdateDatatypes(s->comm,
         &newcomm,
-        comm->imaxLocal / 2,
+        imaxLocal,
-        comm->jmaxLocal / 2,
+        jmaxLocal,
-        comm->kmaxLocal / 2);
+        kmaxLocal);
 
     // MGSolver on residual and error.
     multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm);
@@ -371,7 +371,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     int jmax   = s->grid->jmax;
     int kmax   = s->grid->kmax;
     int levels = s->levels;
-    printf("Using Multigrid solver with %d levels\n", levels);
+    if (commIsMaster(s->comm)) printf("Using Multigrid solver with %d levels\n", levels);
 
     s->r = malloc(levels * sizeof(double*));
     s->e = malloc(levels * sizeof(double*));
@@ -389,7 +389,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     }
 }
 
-void solve(Solver* s, double* p, double* rhs)
+double solve(Solver* s, double* p, double* rhs)
 {
     double res = multiGrid(s, p, rhs, 0, s->comm);
 
@@ -398,4 +398,6 @@ void solve(Solver* s, double* p, double* rhs)
         printf("Residuum: %.6f\n", res);
     }
 #endif
+
+    return res;
 }

BasicSolver/3D-mpi/src/solver-rb.c

@@ -16,7 +16,20 @@
 #include "solver.h"
 #include "util.h"
 
-void solve(Solver* s, double* p, double* rhs)
+void initSolver(Solver* s, Discretization* d, Parameter* p)
+{
+    s->eps        = p->eps;
+    s->omega      = p->omg;
+    s->itermax    = p->itermax;
+    s->levels     = p->levels;
+    s->grid       = &d->grid;
+    s->presmooth  = p->presmooth;
+    s->postsmooth = p->postsmooth;
+    s->comm       = &d->comm;
+    s->problem    = p->name;
+}
+
+double solve(Solver* s, double* p, double* rhs)
 {
     int imaxLocal = s->comm->imaxLocal;
     int jmaxLocal = s->comm->jmaxLocal;
@@ -157,17 +170,6 @@ void solve(Solver* s, double* p, double* rhs)
         printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
     }
 #endif
-}
 
-void initSolver(Solver* s, Discretization* d, Parameter* p)
+return res;
-{
-    s->eps        = p->eps;
-    s->omega      = p->omg;
-    s->itermax    = p->itermax;
-    s->levels     = p->levels;
-    s->grid       = &d->grid;
-    s->presmooth  = p->presmooth;
-    s->postsmooth = p->postsmooth;
-    s->comm       = &d->comm;
-    s->problem    = p->name;
 }

BasicSolver/3D-mpi/src/solver.h

@@ -34,6 +34,6 @@ typedef struct {
     Comm* comm;
 } Solver;
 
-extern void solve(Solver* , double* , double* );
+extern double solve(Solver* , double* , double* );
 extern void initSolver(Solver*, Discretization*, Parameter*);
 #endif

BasicSolver/3D-seq/Makefile

@@ -40,6 +40,18 @@ $(BUILD_DIR)/%.s:  %.c
 
 .PHONY: clean distclean tags info asm format
 
+vis:
+	$(info ===>  GENERATE VISUALIZATION)
+	@gnuplot -e "filename='residual.dat'" ./residual.plot
+
+vis_clean:
+	$(info ===>  CLEAN VISUALIZATION)
+	@rm -f *.dat
+	@rm -f *.vtk
+	@rm -f *.png
+
+clean: vis_clean
+
 clean:
 	$(info ===>  CLEAN)
 	@rm -rf $(BUILD_DIR)

BasicSolver/3D-seq/canal.par

@@ -38,7 +38,7 @@ kmax          50	   # number of interior cells in z-direction
 # Time Data:
 # ---------
 
-te       100.0   # final time
+te       60.0   # final time
 dt       0.02    # time stepsize
 tau      0.5     # safety factor for time stepsize control (<0 constant delt)
 

BasicSolver/3D-seq/dcavity.par

@@ -38,7 +38,7 @@ kmax       128		# number of interior cells in z-direction
 # Time Data:
 # ---------
 
-te       2.0		# final time
+te       10.0		# final time
 dt       0.02	    # time stepsize
 tau      0.5		# safety factor for time stepsize control (<0 constant delt)
 
@@ -46,7 +46,7 @@ tau      0.5		# safety factor for time stepsize control (<0 constant delt)
 # ---------
 
 levels        3         # Multigrid levels
-presmooth     10         # Pre-smoothning iterations
+presmooth     20         # Pre-smoothning iterations
 postsmooth    5         # Post-smoothning iterations
 
 # Pressure Iteration Data:

BasicSolver/3D-seq/residual.plot

@@ -0,0 +1,9 @@
+set terminal png size 1800,768 enhanced font ,12
+set output 'residual.png'
+set datafile separator whitespace
+set xlabel "Timestep"
+set ylabel "Residual"
+
+set logscale y 2
+
+plot 'residual.dat' using 1:2 title "Residual"

BasicSolver/3D-seq/src/main.c

@@ -73,6 +73,9 @@ int main(int argc, char** argv)
     Solver s;
     initParameter(&p);
 
+    FILE* fp;
+    fp = initResidualWriter();
+
     if (argc != 2) {
         printf("Usage: %s <configFile>\n", argv[0]);
         exit(EXIT_SUCCESS);
@@ -90,6 +93,7 @@ int main(int argc, char** argv)
     double te  = d.te;
     double t   = 0.0;
     int nt     = 0;
+    double res = 0.0;
 
     timeStart = getTimeStamp();
     while (t <= te) {
@@ -99,8 +103,11 @@ int main(int argc, char** argv)
         computeFG(&d);
         computeRHS(&d);
         if (nt % 100 == 0) normalizePressure(&d);
-        solve(&s, d.p, d.rhs);
+        res = solve(&s, d.p, d.rhs);
         adaptUV(&d);
+
+        writeResidual(fp, t, res);
+
         t += d.dt;
         nt++;
 
@@ -124,6 +131,7 @@ int main(int argc, char** argv)
     vg = allocate(64, bytesize);
     wg = allocate(64, bytesize);
     
+    fclose(fp);
     createBulkArrays(&d, pg, ug, vg, wg);
     VtkOptions opts = { .grid = d.grid };
     vtkOpen(&opts, d.problem);

BasicSolver/3D-seq/src/progress.c

@@ -49,3 +49,22 @@ void stopProgress()
     printf("\n");
     fflush(stdout);
 }
+
+FILE* initResidualWriter()
+{
+    FILE* fp;
+    fp = fopen("residual.dat", "w");
+
+    if (fp == NULL) {
+        printf("Error!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return fp;
+
+}
+
+void writeResidual(FILE* fp, double ts, double res)
+{
+    fprintf(fp, "%f, %f\n", ts, res);
+}

BasicSolver/3D-seq/src/progress.h

@@ -10,5 +10,6 @@
 extern void initProgress(double);
 extern void printProgress(double);
 extern void stopProgress(void);
-
+extern FILE* initResidualWriter(void);
+extern void writeResidual(FILE*, double, double);
 #endif

BasicSolver/3D-seq/src/solver-mg.c

@@ -292,11 +292,13 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     }
 }
 
-void solve(Solver* s, double* p, double* rhs)
+double solve(Solver* s, double* p, double* rhs)
 {
     double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax, s->grid->kmax);
 
 #ifdef VERBOSE
     printf("Residuum: %.6f\n", res);
 #endif
+
+return res;
 }

BasicSolver/3D-seq/src/solver-rb.c

@@ -15,7 +15,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p)
     s->omega   = p->omg;
 }
 
-void solve(Solver* s, double* p, double* rhs)
+double solve(Solver* s, double* p, double* rhs)
 {
     int imax      = s->grid->imax;
     int jmax      = s->grid->jmax;
@@ -96,4 +96,6 @@ void solve(Solver* s, double* p, double* rhs)
 #ifdef VERBOSE
     printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
 #endif
+
+return res;
 }

BasicSolver/3D-seq/src/solver.h

@@ -22,6 +22,6 @@ typedef struct {
 } Solver;
 
 extern void initSolver(Solver*, Discretization*, Parameter*);
-extern void solve(Solver*, double*, double*);
+extern double solve(Solver*, double*, double*);
 
 #endif