Synchronize and Update variants. Prepare Assigment codes.

BasicSolver/3D-mpi/config.mk

@@ -1,5 +1,6 @@
 # Supported: GCC, CLANG, ICC
 TAG ?= CLANG
+ENABLE_MPI ?= true
 ENABLE_OPENMP ?= false
 
 #Feature options

BasicSolver/3D-mpi/include_CLANG.mk

@@ -1,4 +1,10 @@
+ifeq ($(ENABLE_MPI),true)
 CC   = mpicc
+DEFINES  = -D_MPI
+else
+CC = cc
+endif
+
 GCC  = cc
 LINKER = $(CC)
 
@@ -9,9 +15,7 @@ LIBS     = # -lomp
 endif
 
 VERSION  = --version
-# CFLAGS   = -O3 -std=c17 $(OPENMP)
 CFLAGS   = -Ofast -std=c17
-#CFLAGS   = -Ofast -fnt-store=aggressive  -std=c99 $(OPENMP) #AMD CLANG
 LFLAGS   = $(OPENMP) -lm
-DEFINES  = -D_GNU_SOURCE# -DDEBUG
+DEFINES  += -D_GNU_SOURCE# -DDEBUG
 INCLUDES =

BasicSolver/3D-mpi/src/allocate.c

@@ -5,10 +5,13 @@
  * license that can be found in the LICENSE file.
  */
 #include <errno.h>
+#include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
 
-void* allocate(int alignment, size_t bytesize)
+#include "allocate.h"
+
+void* allocate(size_t alignment, size_t bytesize)
 {
     int errorCode;
     void* ptr;

BasicSolver/3D-mpi/src/allocate.h

@@ -8,6 +8,6 @@
 #define __ALLOCATE_H_
 #include <stdlib.h>
 
-extern void* allocate(int alignment, size_t bytesize);
+extern void* allocate(size_t alignment, size_t bytesize);
 
 #endif

BasicSolver/3D-mpi/src/comm.c

@@ -4,7 +4,9 @@
  * Use of this source code is governed by a MIT style
  * license that can be found in the LICENSE file.
  */
+#if defined(_MPI)
 #include <mpi.h>
+#endif
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -12,6 +14,7 @@
 #include "allocate.h"
 #include "comm.h"
 
+#if defined(_MPI)
 // subroutines local to this module
 static int sizeOfRank(int rank, int size, int N)
 {
@@ -177,19 +180,23 @@ static int sum(int* sizes, int position)
 
     return sum;
 }
+#endif
 
 // exported subroutines
 void commReduction(double* v, int op)
 {
+#if defined(_MPI)
     if (op == MAX) {
         MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
     } else if (op == SUM) {
         MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
     }
+#endif
 }
 
 int commIsBoundary(Comm* c, Direction direction)
 {
+#if defined(_MPI)
     switch (direction) {
     case LEFT:
         return c->coords[ICORD] == 0;
@@ -213,12 +220,14 @@ int commIsBoundary(Comm* c, Direction direction)
         printf("ERROR!\n");
         break;
     }
+#endif
 
-    return 0;
+    return 1;
 }
 
 void commExchange(Comm* c, double* grid)
 {
+#if defined(_MPI)
     int counts[6]      = { 1, 1, 1, 1, 1, 1 };
     MPI_Aint displs[6] = { 0, 0, 0, 0, 0, 0 };
 
@@ -231,10 +240,12 @@ void commExchange(Comm* c, double* grid)
         displs,
         c->rbufferTypes,
         c->comm);
+#endif
 }
 
 void commShift(Comm* c, double* f, double* g, double* h)
 {
+#if defined(_MPI)
     MPI_Request requests[6] = { MPI_REQUEST_NULL,
         MPI_REQUEST_NULL,
         MPI_REQUEST_NULL,
@@ -282,6 +293,7 @@ void commShift(Comm* c, double* f, double* g, double* h)
     MPI_Isend(h, 1, c->sbufferTypes[BACK], c->neighbours[BACK], 2, c->comm, &requests[5]);
 
     MPI_Waitall(6, requests, MPI_STATUSES_IGNORE);
+#endif
 }
 
 #define G(v, i, j, k)                                                                    \
@@ -300,6 +312,7 @@ void commCollectResult(Comm* c,
     int jmax,
     int imax)
 {
+#if defined(_MPI)
     int imaxLocal = c->imaxLocal;
     int jmaxLocal = c->jmaxLocal;
     int kmaxLocal = c->kmaxLocal;
@@ -426,10 +439,12 @@ void commCollectResult(Comm* c,
         imax);
 
     free(tmp);
+#endif
 }
 
 void commPrintConfig(Comm* c)
 {
+#if defined(_MPI)
     fflush(stdout);
     MPI_Barrier(MPI_COMM_WORLD);
     if (commIsMaster(c)) {
@@ -459,13 +474,24 @@ void commPrintConfig(Comm* c)
         }
     }
     MPI_Barrier(MPI_COMM_WORLD);
+#endif
 }
 
-void commInit(Comm* c, int kmax, int jmax, int imax)
+void commInit(Comm* c, int argc, char** argv)
 {
-    /* setup communication */
+#if defined(_MPI)
+    MPI_Init(&argc, &argv);
     MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
     MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
+#else
+    c->rank      = 0;
+    c->size      = 1;
+#endif
+}
+
+void commPartition(Comm* c, int kmax, int jmax, int imax)
+{
+#if defined(_MPI)
     int dims[NDIMS]    = { 0, 0, 0 };
     int periods[NDIMS] = { 0, 0, 0 };
     MPI_Dims_create(c->size, NDIMS, dims);
@@ -492,12 +518,21 @@ void commInit(Comm* c, int kmax, int jmax, int imax)
     setupCommunication(c, FRONT, HALO);
     setupCommunication(c, BACK, BULK);
     setupCommunication(c, BACK, HALO);
+#else
+    c->imaxLocal = imax;
+    c->jmaxLocal = jmax;
+    c->kmaxLocal = kmax;
+#endif
 }
 
-void commFree(Comm* c)
+void commFinalize(Comm* c)
 {
+#if defined(_MPI)
     for (int i = 0; i < NDIRS; i++) {
         MPI_Type_free(&c->sbufferTypes[i]);
         MPI_Type_free(&c->rbufferTypes[i]);
     }
+
+    MPI_Finalize();
+#endif
 }

BasicSolver/3D-mpi/src/comm.h

@@ -1,12 +1,14 @@
 /*
- * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+ * Copyright (C) 2024 NHR@FAU, University Erlangen-Nuremberg.
  * All rights reserved. This file is part of nusif-solver.
  * Use of this source code is governed by a MIT style
  * license that can be found in the LICENSE file.
  */
 #ifndef __COMM_H_
 #define __COMM_H_
+#if defined(_MPI)
 #include <mpi.h>
+#endif
 /*
  * Spatial directions:
  * ICORD (0) from 0 (LEFT) to imax (RIGHT)
@@ -24,17 +26,19 @@ enum op { MAX = 0, SUM };
 typedef struct {
     int rank;
     int size;
+#if defined(_MPI)
     MPI_Comm comm;
     MPI_Datatype sbufferTypes[NDIRS];
     MPI_Datatype rbufferTypes[NDIRS];
+#endif
     int neighbours[NDIRS];
     int coords[NDIMS], dims[NDIMS];
     int imaxLocal, jmaxLocal, kmaxLocal;
-    MPI_File fh;
 } Comm;
 
-extern void commInit(Comm* comm, int kmax, int jmax, int imax);
-extern void commFree(Comm* comm);
+extern void commInit(Comm* c, int argc, char** argv);
+extern void commPartition(Comm* c, int kmax, int jmax, int imax);
+extern void commFinalize(Comm* comm);
 extern void commPrintConfig(Comm*);
 extern void commExchange(Comm*, double*);
 extern void commShift(Comm* c, double* f, double* g, double* h);

BasicSolver/3D-mpi/src/main.c

@@ -1,90 +1,82 @@
 /*
- * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+ * Copyright (C) 2024 NHR@FAU, University Erlangen-Nuremberg.
  * All rights reserved.
  * Use of this source code is governed by a MIT-style
  * license that can be found in the LICENSE file.
  */
 #include <float.h>
 #include <limits.h>
-#include <mpi.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 #include "allocate.h"
+#include "comm.h"
 #include "parameter.h"
 #include "progress.h"
 #include "solver.h"
-#include "test.h"
 #include "timing.h"
 #include "vtkWriter.h"
 
 int main(int argc, char** argv)
 {
-    int rank;
     double timeStart, timeStop;
-    Parameter params;
-    Solver solver;
+    Parameter p;
+    Solver s;
 
-    MPI_Init(&argc, &argv);
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    initParameter(&params);
+    commInit(&s.comm, argc, argv);
+    initParameter(&p);
 
     if (argc != 2) {
         printf("Usage: %s <configFile>\n", argv[0]);
         exit(EXIT_SUCCESS);
     }
 
-    readParameter(&params, argv[1]);
-    if (commIsMaster(&solver.comm)) {
-        printParameter(&params);
+    readParameter(&p, argv[1]);
+    commPartition(&s.comm, p.kmax, p.jmax, p.imax);
+    if (commIsMaster(&s.comm)) {
+        printParameter(&p);
     }
-    initSolver(&solver, &params);
+    initSolver(&s, &p);
 #ifndef VERBOSE
-    initProgress(solver.te);
+    initProgress(s.te);
 #endif
 
-    double tau = solver.tau;
-    double te  = solver.te;
+    double tau = s.tau;
+    double te  = s.te;
     double t   = 0.0;
-    int nt     = 0;
 
     timeStart = getTimeStamp();
     while (t <= te) {
-        if (tau > 0.0) computeTimestep(&solver);
-        setBoundaryConditions(&solver);
-        setSpecialBoundaryCondition(&solver);
-        computeFG(&solver);
-        computeRHS(&solver);
-        // if (nt % 100 == 0) normalizePressure(&solver);
-        solve(&solver);
-        adaptUV(&solver);
-        t += solver.dt;
-        nt++;
+        if (tau > 0.0) computeTimestep(&s);
+        setBoundaryConditions(&s);
+        setSpecialBoundaryCondition(&s);
+        computeFG(&s);
+        computeRHS(&s);
+        solve(&s);
+        adaptUV(&s);
+        t += s.dt;
 
 #ifdef VERBOSE
-        if (commIsMaster(&solver.comm)) {
-            printf("TIME %f , TIMESTEP %f\n", t, solver.dt);
+        if (commIsMaster(&s.comm)) {
+            printf("TIME %f , TIMESTEP %f\n", t, s.dt);
         }
 #else
         printProgress(t);
 #endif
     }
     timeStop = getTimeStamp();
+#ifndef VERBOSE
     stopProgress();
-    if (commIsMaster(&solver.comm)) {
+#endif
+    if (commIsMaster(&s.comm)) {
         printf("Solution took %.2fs\n", timeStop - timeStart);
     }
 
-    // testInit(&solver);
-    // commExchange(&solver.comm, solver.p);
-    // testPrintHalo(&solver, solver.p);
-
     double *pg, *ug, *vg, *wg;
 
-    if (commIsMaster(&solver.comm)) {
-        size_t bytesize = solver.grid.imax * solver.grid.jmax * solver.grid.kmax *
-                          sizeof(double);
+    if (commIsMaster(&s.comm)) {
+        size_t bytesize = s.grid.imax * s.grid.jmax * s.grid.kmax * sizeof(double);
 
         pg = allocate(64, bytesize);
         ug = allocate(64, bytesize);
@@ -92,26 +84,25 @@ int main(int argc, char** argv)
         wg = allocate(64, bytesize);
     }
 
-    commCollectResult(&solver.comm,
+    commCollectResult(&s.comm,
         ug,
         vg,
         wg,
         pg,
-        solver.u,
-        solver.v,
-        solver.w,
-        solver.p,
-        solver.grid.kmax,
-        solver.grid.jmax,
-        solver.grid.imax);
+        s.u,
+        s.v,
+        s.w,
+        s.p,
+        s.grid.kmax,
+        s.grid.jmax,
+        s.grid.imax);
 
-    VtkOptions opts = { .grid = solver.grid, .comm = solver.comm };
-    vtkOpen(&opts, solver.problem);
+    VtkOptions opts = { .grid = s.grid, .comm = s.comm };
+    vtkOpen(&opts, s.problem);
     vtkScalar(&opts, "pressure", pg);
     vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg });
     vtkClose(&opts);
 
-    commFree(&solver.comm);
-    MPI_Finalize();
+    commFinalize(&s.comm);
     return EXIT_SUCCESS;
 }

BasicSolver/3D-mpi/src/progress.h

@@ -9,6 +9,6 @@
 
 extern void initProgress(double);
 extern void printProgress(double);
-extern void stopProgress();
+extern void stopProgress(void);
 
 #endif

BasicSolver/3D-mpi/src/solver.c

@@ -104,7 +104,6 @@ void initSolver(Solver* s, Parameter* params)
     s->tau     = params->tau;
     s->gamma   = params->gamma;
 
-    commInit(&s->comm, s->grid.kmax, s->grid.jmax, s->grid.imax);
     /* allocate arrays */
     int imaxLocal = s->comm.imaxLocal;
     int jmaxLocal = s->comm.jmaxLocal;
@@ -199,27 +198,36 @@ void solve(Solver* s)
     double epssq = eps * eps;
     int it       = 0;
     double res   = 1.0;
-    commExchange(&s->comm, p);
+    int pass, ksw, jsw, isw;
 
     while ((res >= epssq) && (it < itermax)) {
-        res = 0.0;
+        ksw = 1;
 
-        for (int k = 1; k < kmaxLocal + 1; k++) {
-            for (int j = 1; j < jmaxLocal + 1; j++) {
-                for (int i = 1; i < imaxLocal + 1; i++) {
+        for (pass = 0; pass < 2; pass++) {
+            jsw = ksw;
+            commExchange(&s->comm, p);
 
-                    double r = RHS(i, j, k) -
-                               ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) *
-                                       idx2 +
-                                   (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
-                                       idy2 +
-                                   (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
-                                       idz2);
+            for (int k = 1; k < kmaxLocal + 1; k++) {
+                isw = jsw;
+                for (int j = 1; j < jmaxLocal + 1; j++) {
+                    for (int i = isw; i < imaxLocal + 1; i += 2) {
 
-                    P(i, j, k) -= (factor * r);
-                    res += (r * r);
+                        double r =
+                            RHS(i, j, k) -
+                            ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
+                                (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
+                                    idy2 +
+                                (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
+                                    idz2);
+
+                        P(i, j, k) -= (factor * r);
+                        res += (r * r);
+                    }
+                    isw = 3 - isw;
                 }
+                jsw = 3 - jsw;
             }
+            ksw = 3 - ksw;
         }
 
         if (commIsBoundary(&s->comm, FRONT)) {

BasicSolver/3D-mpi/src/solver.h

@@ -33,13 +33,13 @@ typedef struct {
     Comm comm;
 } Solver;
 
-void initSolver(Solver*, Parameter*);
-void computeRHS(Solver*);
-void solve(Solver*);
-void normalizePressure(Solver*);
-void computeTimestep(Solver*);
-void setBoundaryConditions(Solver*);
-void setSpecialBoundaryCondition(Solver*);
-void computeFG(Solver*);
-void adaptUV(Solver*);
+extern void initSolver(Solver*, Parameter*);
+extern void computeRHS(Solver*);
+extern void solve(Solver*);
+extern void normalizePressure(Solver*);
+extern void computeTimestep(Solver*);
+extern void setBoundaryConditions(Solver*);
+extern void setSpecialBoundaryCondition(Solver*);
+extern void computeFG(Solver*);
+extern void adaptUV(Solver*);
 #endif

BasicSolver/3D-mpi/src/timing.c

@@ -7,18 +7,16 @@
 #include <stdlib.h>
 #include <time.h>
 
-double getTimeStamp()
+double getTimeStamp(void)
 {
     struct timespec ts;
     clock_gettime(CLOCK_MONOTONIC, &ts);
     return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9;
 }
 
-double getTimeResolution()
+double getTimeResolution(void)
 {
     struct timespec ts;
     clock_getres(CLOCK_MONOTONIC, &ts);
     return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9;
 }
-
-double getTimeStamp_() { return getTimeStamp(); }

BasicSolver/3D-mpi/src/timing.h

@@ -7,8 +7,7 @@
 #ifndef __TIMING_H_
 #define __TIMING_H_
 
-extern double getTimeStamp();
-extern double getTimeResolution();
-extern double getTimeStamp_();
+extern double getTimeStamp(void);
+extern double getTimeResolution(void);
 
 #endif // __TIMING_H_