Completed migrating PT to 3D MPI w/o MPI IO

BasicSolver/3D-mpi/src/comm.c

@@ -475,9 +475,17 @@ void commInit(Comm* c, int kmax, int jmax, int imax)
     MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]);
     MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords);
 
-    c->imaxLocal = sizeOfRank(c->coords[IDIM], dims[ICORD], imax);
+    c->imaxLocal = sizeOfRank(c->coords[KDIM], dims[ICORD], imax);
+    //printf("\nRank : %d\nimaxLocal : %d -> c->coords[IDIM] : %d , dims[ICORD] : %d, imax : %d\n", c->rank, c->imaxLocal, c->coords[IDIM], dims[ICORD], imax);
     c->jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JCORD], jmax);
-    c->kmaxLocal = sizeOfRank(c->coords[KDIM], dims[KCORD], kmax);
+    //printf("jmaxLocal : %d -> c->coords[JDIM] : %d , dims[JCORD] : %d, imax : %d\n", c->jmaxLocal, c->coords[JDIM], dims[JCORD], jmax);
+    c->kmaxLocal = sizeOfRank(c->coords[IDIM], dims[KCORD], kmax);
+    //printf("kmaxLocal : %d -> c->coords[KDIM] : %d , dims[KCORD] : %d, imax : %d\n", c->kmaxLocal, c->coords[KDIM], dims[KCORD], kmax);
+
+    
+    // sizeOfRank(int rank, int size, int N)
+    // { return N / size + ((N % size > rank) ? 1 : 0); }
+
 
     // setup buffer types for communication
     setupCommunication(c, LEFT, BULK);

BasicSolver/3D-mpi/src/main.c

@@ -17,6 +17,7 @@
 #include "solver.h"
 #include "test.h"
 #include "timing.h"
+#include "particletracing.h"
 #include "vtkWriter.h"
 
 enum VARIANT { SOR = 1, RB, RBA };
@@ -28,6 +29,7 @@ int main(int argc, char** argv)
 
     double timeStart, timeStop;
     Parameter params;
+    ParticleTracer particletracer;
     Solver solver;
 
     MPI_Init(&argc, &argv);
@@ -48,17 +50,20 @@ int main(int argc, char** argv)
         printParameter(&params);
     }
     initSolver(&solver, &params);
+    initParticleTracer(&particletracer, &params, &solver);
+
 #ifndef VERBOSE
     initProgress(solver.te);
 #endif
 
+    printParticleTracerParameters(&particletracer);
+
     double tau = solver.tau;
     double te  = solver.te;
     double t   = 0.0;
     int nt     = 0;
 
-
-    void (*solver_generic[])(solver) = {solve, solveRB, solveRBA};
+    void (*solver_generic[])() = {solve, solveRB, solveRBA};
     
     timeStart = getTimeStamp();
 
@@ -69,8 +74,11 @@ int main(int argc, char** argv)
         computeFG(&solver);
         computeRHS(&solver);
         // if (nt % 100 == 0) normalizePressure(&solver);
-        (*solver_generic[variant - 1])(&solver);
+        solver_generic[variant - 1](&solver);
         adaptUV(&solver);
+
+        trace(&particletracer, solver.u, solver.v, solver.w, t);  
+
         t += solver.dt;
         nt++;
 

BasicSolver/3D-mpi/src/parameter.c

@@ -90,6 +90,18 @@ void readParameter(Parameter* param, const char* filename)
             PARSE_REAL(p_init);
             PARSE_REAL(rho);
 
+            /* Added new particle tracing parameters */
+            PARSE_INT(numberOfParticles);
+            PARSE_REAL(startTime);
+            PARSE_REAL(injectTimePeriod);
+            PARSE_REAL(writeTimePeriod);
+            PARSE_REAL(x1);
+            PARSE_REAL(y1);
+            PARSE_REAL(z1);
+            PARSE_REAL(x2);
+            PARSE_REAL(y2);
+            PARSE_REAL(z2);
+
         }
     }
 

BasicSolver/3D-mpi/src/parameter.h

@@ -18,6 +18,11 @@ typedef struct {
     char* name;
     int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
     double u_init, v_init, w_init, p_init;
+
+    int numberOfParticles;
+    double startTime, injectTimePeriod, writeTimePeriod;
+
+    double x1, y1, z1, x2, y2, z2;
 } Parameter;
 
 void initParameter(Parameter*);

BasicSolver/3D-mpi/src/particletracing.c

@@ -0,0 +1,613 @@
+/*
+ * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "particletracing.h"
+#include "solver.h"
+
+#define U(i, j, k) u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define V(i, j, k) v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+#define W(i, j, k) w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
+
+static int ts = 0;
+
+#define XOFFSET 0
+#define YOFFSET 1
+#define ZOFFSET 2
+#define XOFFSETEND 3
+#define YOFFSETEND 4
+#define ZOFFSETEND 5
+
+static double sum(int* sizes, int size)
+{
+    double sum = 0;
+
+    for (int i = 0; i < size; ++i) 
+    {
+        sum += sizes[i];
+    }
+
+    return sum;
+}
+
+static double sumOffset(double* sizes, int init, int offset, int coord)
+{
+    double sum = 0;
+
+    for (int i = init - offset; coord > 0; i-=offset, --coord) 
+    {
+        sum += sizes[i];
+    }
+
+    return sum;
+}
+
+
+void printParticles(ParticleTracer* particletracer)
+{
+    for(int i = 0; i < particletracer->totalParticles; ++i)
+    {   
+        printf("Rank : %d Particle position X : %.2f, Y : %.2f, flag : %d, total pt : %d, pointer : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : %.2f\n", 
+            particletracer->rank, 
+            particletracer->particlePool[i].x,
+            particletracer->particlePool[i].y,
+            particletracer->particlePool[i].flag, 
+            particletracer->totalParticles, 
+            particletracer->pointer, 
+            particletracer->xOffset, particletracer->yOffset, 
+            particletracer->xOffsetEnd, particletracer->yOffsetEnd);
+    }
+}
+void injectParticles(ParticleTracer* particletracer)
+{
+    double x, y, z;
+    for(int i = 0; i < particletracer->numberOfParticles; ++i)
+    {
+        x = particletracer->linSpaceLine[i].x;
+        y = particletracer->linSpaceLine[i].y;
+        z = particletracer->linSpaceLine[i].z;
+        if(x >= particletracer->xOffset && y >= particletracer->yOffset && z >= particletracer->zOffset &&  x <= particletracer->xOffsetEnd && y <= particletracer->yOffsetEnd && y <= particletracer->zOffsetEnd )
+        {
+            // printf("\nRank : %d\n", particletracer->rank);
+            // printf("\t%.2f >= %.2f && %.2f >= %.2f && %.2f <= %.2f && %.2f <= %.2f\n",x , particletracer->xOffset ,y , particletracer->yOffset, x , particletracer->xOffsetEnd ,y , particletracer->yOffsetEnd);
+        
+            particletracer->particlePool[particletracer->pointer].x = x;
+            particletracer->particlePool[particletracer->pointer].y = y;
+            particletracer->particlePool[particletracer->pointer].z = z;
+            particletracer->particlePool[particletracer->pointer].flag = true;
+            ++(particletracer->pointer);
+            ++(particletracer->totalParticles);
+        }
+    }
+}
+
+void advanceParticles(ParticleTracer* particletracer, double* restrict u, double* restrict v, double* restrict w,double time)
+{
+    int imax       = particletracer->imax;
+    int jmax       = particletracer->jmax;
+    int kmax       = particletracer->kmax;
+    int imaxLocal  = particletracer->imaxLocal;
+    int jmaxLocal  = particletracer->jmaxLocal;
+    int kmaxLocal  = particletracer->kmaxLocal;
+
+
+    double dx      = particletracer->dx;
+    double dy      = particletracer->dy;
+    double dz      = particletracer->dz;
+
+    double xlength = particletracer->xlength;
+    double ylength = particletracer->ylength;
+    double zlength = particletracer->zlength;
+
+
+    Particle buff[particletracer->size][30];
+
+    for(int i = 0; i < particletracer->size; ++i)
+    {   
+        for(int j = 0; j < 30; ++j)
+        { 
+            buff[i][j].x = 0.0;
+            buff[i][j].y = 0.0;
+            buff[i][j].z = 0.0;
+            buff[i][j].flag = false;
+        }
+    }
+    int particleBufIndex[particletracer->size];
+
+    memset(particleBufIndex, 0, sizeof(particleBufIndex));
+
+
+    for(int i = 0; i < particletracer->totalParticles; ++i)
+    {
+        if(particletracer->particlePool[i].flag == true)
+        {
+            double xTemp = particletracer->particlePool[i].x;
+            double yTemp = particletracer->particlePool[i].y;
+            double zTemp = particletracer->particlePool[i].z;
+
+
+            double x   = xTemp - particletracer->xOffset;
+            double y   = yTemp - particletracer->yOffset;
+            double z   = zTemp - particletracer->zOffset;
+
+            int iCoord = (int)(x / dx) + 1;
+            int jCoord = (int)((y + 0.5 * dy) / dy) + 1;
+            int kCoord = (int)((z + 0.5 * dz) / dz) + 1;
+
+
+            double x1 = (double)(iCoord - 1) * dx;
+            double y1 = ((double)(jCoord - 1) - 0.5) * dy;
+            double z1 = ((double)(kCoord - 1) - 0.5) * dz;
+
+            double x2 = (double)iCoord * dx;
+            double y2 = ((double)jCoord - 0.5) * dy;
+            double z2 = ((double)kCoord - 0.5) * dz;
+
+
+            double u_n = (1.0 / (dx * dy * dz)) *
+                    (
+                    (x2 - x) * (y2 - y) * (z2 - z) * U(iCoord - 1, jCoord - 1, kCoord - 1) +
+                    (x - x1) * (y2 - y) * (z2 - z) * U(iCoord, jCoord - 1, kCoord - 1) +
+                    (x2 - x) * (y - y1) * (z2 - z) * U(iCoord - 1, jCoord, kCoord - 1) +
+                    (x - x1) * (y - y1) * (z2 - z) * U(iCoord, jCoord, kCoord - 1) + 
+                    (x2 - x) * (y2 - y) * (z - z1) * U(iCoord - 1, jCoord - 1, kCoord) +
+                    (x - x1) * (y2 - y) * (z - z1) * U(iCoord, jCoord - 1, kCoord) +
+                    (x2 - x) * (y - y1) * (z - z1) * U(iCoord - 1, jCoord, kCoord) +
+                    (x - x1) * (y - y1) * (z - z1) * U(iCoord, jCoord, kCoord)
+                    );
+
+            double new_x = (x + particletracer->xOffset) + particletracer->dt * u_n;
+            particletracer->particlePool[i].x = new_x;
+
+            iCoord = (int)((x + 0.5 * dx) / dx) + 1;
+            jCoord = (int)(y / dy) + 1;
+            kCoord = (int)((z + 0.5 * dz) / dz) + 1;
+
+            x1 = ((double)(iCoord - 1) - 0.5) * dx;
+            y1 = (double)(jCoord - 1) * dy;
+            z1 = ((double)(kCoord - 1) - 0.5) * dz;
+
+            x2 = ((double)iCoord - 0.5) * dx;
+            y2 = (double)jCoord * dy;
+            z2 = ((double)kCoord - 0.5) * dz;
+            
+            double v_n = (1.0 / (dx * dy * dz)) *
+                        (
+                        (x2 - x) * (y2 - y) * (z2 - z) * V(iCoord - 1, jCoord - 1, kCoord - 1) +
+                        (x - x1) * (y2 - y) * (z2 - z) * V(iCoord, jCoord - 1, kCoord - 1) +
+                        (x2 - x) * (y - y1) * (z2 - z) * V(iCoord - 1, jCoord, kCoord - 1) +
+                        (x - x1) * (y - y1) * (z2 - z) * V(iCoord, jCoord, kCoord - 1) +
+                        (x2 - x) * (y2 - y) * (z - z1) * V(iCoord - 1, jCoord - 1, kCoord) +
+                        (x - x1) * (y2 - y) * (z - z1) * V(iCoord, jCoord - 1, kCoord) +
+                        (x2 - x) * (y - y1) * (z - z1) * V(iCoord - 1, jCoord, kCoord) +
+                        (x - x1) * (y - y1) * (z - z1) * V(iCoord, jCoord, kCoord)
+                        );
+
+            double new_y = (y + particletracer->yOffset) + particletracer->dt * v_n;
+            particletracer->particlePool[i].y = new_y;
+
+            iCoord = (int)((x + 0.5 * dx) / dx) + 1;
+            jCoord = (int)((y + 0.5 * dy) / dy) + 1;
+            kCoord = (int)(z / dz) + 1;
+
+            x1 = ((double)(iCoord - 1) - 0.5) * dx;
+            y1 = ((double)(jCoord - 1) - 0.5) * dy;
+            z1 = (double)(kCoord - 1) * dz;
+
+            x2 = ((double)iCoord - 0.5) * dx;
+            y2 = ((double)jCoord - 0.5) * dy;
+            z2 = (double)kCoord * dz;
+            
+            double w_n = (1.0 / (dx * dy * dz)) *
+                        (
+                        (x2 - x) * (y2 - y) * (z2 - z) * W(iCoord - 1, jCoord - 1, kCoord - 1) +
+                        (x - x1) * (y2 - y) * (z2 - z) * W(iCoord, jCoord - 1, kCoord - 1) +
+                        (x2 - x) * (y - y1) * (z2 - z) * W(iCoord - 1, jCoord, kCoord - 1) +
+                        (x - x1) * (y - y1) * (z2 - z) * W(iCoord, jCoord, kCoord - 1) +
+                        (x2 - x) * (y2 - y) * (z - z1) * W(iCoord - 1, jCoord - 1, kCoord) +
+                        (x - x1) * (y2 - y) * (z - z1) * W(iCoord, jCoord - 1, kCoord) +
+                        (x2 - x) * (y - y1) * (z - z1) * W(iCoord - 1, jCoord, kCoord) +
+                        (x - x1) * (y - y1) * (z - z1) * W(iCoord, jCoord, kCoord)
+                        );
+
+            double new_z = (z + particletracer->zOffset) + particletracer->dt * w_n;
+            particletracer->particlePool[i].z = new_z;
+
+            // printf("Rank : %d\n", particletracer->rank);
+            // printf("\tOld X : %.2f, translated X : %.2f, xOffset : %.2f, New X : %.2f, iCoord : %d\n\tOld Y : %.2f, translated X : %.2f, yOffset : %.2f, New Y : %.2f, jCoord : %d\n\n",xTemp, x, particletracer->xOffset, new_x, iCoord, yTemp, y, particletracer->yOffset , new_y, jCoord);
+            // printf("\tU(iCoord - 1, jCoord - 1) : %.2f, U(iCoord, jCoord - 1) : %.2f, U(iCoord - 1, jCoord) : %.2f, U(iCoord, jCoord) : %.2f\n", U(iCoord - 1, jCoord - 1), U(iCoord, jCoord - 1), U(iCoord - 1, jCoord), U(iCoord, jCoord));
+            // printf("\tV(iCoord - 1, jCoord - 1) : %.2f, V(iCoord, jCoord - 1) : %.2f, V(iCoord - 1, jCoord) : %.2f, V(iCoord, jCoord) : %.2f\n\n", V(iCoord - 1, jCoord - 1), V(iCoord, jCoord - 1), V(iCoord - 1, jCoord), V(iCoord, jCoord));
+            //printf("\t U N : %.2f, V N : %.2f\n\n", u_n, v_n);
+
+            if (((new_x < particletracer->xOffset) || (new_x >= particletracer->xOffsetEnd) ||
+                 (new_y < particletracer->yOffset) || (new_y >= particletracer->yOffsetEnd) ||
+                 (new_z < particletracer->zOffset) || (new_z >= particletracer->zOffsetEnd)))
+            {
+                //New logic to transfer particles to neighbouring ranks or discard the particle.
+
+                for(int i = 0; i < particletracer->size; ++i)
+                {
+                    if((new_x >= particletracer->offset[i + particletracer->size * XOFFSET]) && 
+                       (new_x <= particletracer->offset[i + particletracer->size * XOFFSETEND]) && 
+                       (new_y >= particletracer->offset[i + particletracer->size * YOFFSET]) && 
+                       (new_y <= particletracer->offset[i + particletracer->size * YOFFSETEND]) &&
+                       (new_z >= particletracer->offset[i + particletracer->size * ZOFFSET]) && 
+                       (new_z <= particletracer->offset[i + particletracer->size * ZOFFSETEND]) &&
+                       i != particletracer->rank)
+                    {
+                        buff[i][particleBufIndex[i]].x = new_x;
+                        buff[i][particleBufIndex[i]].y = new_y;
+                        buff[i][particleBufIndex[i]].z = new_z;
+                        buff[i][particleBufIndex[i]].flag = true;
+                        ++particleBufIndex[i];
+                    }
+                }
+                particletracer->particlePool[i].flag = false;
+            }
+        }
+    }
+
+    for(int i = 0; i < particletracer->size; ++i)
+    {   
+        if(i != particletracer->rank)
+        {
+            MPI_Send(buff[i], 30, particletracer->mpi_particle, i, 0, particletracer->comm);
+        }
+    }
+    for(int i = 0; i < particletracer->size; ++i)
+    {   
+        if(i != particletracer->rank)
+        {
+            MPI_Recv(buff[i], 30, particletracer->mpi_particle, i, 0, particletracer->comm, MPI_STATUS_IGNORE);
+        }
+    }
+    for(int i = 0; i < particletracer->size; ++i)
+    {   
+        if(i != particletracer->rank)
+        {
+            for(int j = 0; j < 30; ++j)
+            {
+                if(buff[i][j].flag == true)
+                {
+                    particletracer->particlePool[particletracer->pointer].x = buff[i][j].x;
+                    particletracer->particlePool[particletracer->pointer].y = buff[i][j].y;
+                    particletracer->particlePool[particletracer->pointer].z = buff[i][j].z;
+                    particletracer->particlePool[particletracer->pointer].flag = true;
+                    ++(particletracer->pointer);
+                    ++(particletracer->totalParticles);
+                }
+            }
+        }
+    }
+}
+
+void freeParticles(ParticleTracer* particletracer)
+{
+    free(particletracer->particlePool);
+    free(particletracer->linSpaceLine);
+    free(particletracer->offset);
+}
+
+void writeParticles(ParticleTracer* particletracer)
+{
+    int collectedBuffIndex[particletracer->size];
+
+    MPI_Gather(&particletracer->totalParticles, 1, MPI_INT, collectedBuffIndex, 1, MPI_INT, 0, particletracer->comm);
+
+    if(particletracer->rank != 0)
+    {
+        Particle buff[particletracer->totalParticles];
+        for(int i = 0; i < particletracer->totalParticles; ++i)
+        {   
+            buff[i].x = particletracer->particlePool[i].x;
+            buff[i].y = particletracer->particlePool[i].y;
+            buff[i].z = particletracer->particlePool[i].z;
+            buff[i].flag =  particletracer->particlePool[i].flag;
+            //printf("Rank : %d sending to rank 0 X : %.2f, Y : %.2f with totalpt : %d\n", particletracer->rank, buff[i].x, buff[i].y, particletracer->totalParticles);
+        }
+        MPI_Send(buff, particletracer->totalParticles, particletracer->mpi_particle, 0, 1, particletracer->comm);
+    }
+    if(particletracer->rank == 0)
+    {
+        char filename[50];
+        FILE* fp;
+
+        snprintf(filename, 50, "vtk_files/particles_%d.vtk", ts);
+        fp = fopen(filename, "w");
+
+        if (fp == NULL) {
+            printf("Error!\n");
+            exit(EXIT_FAILURE);
+        }
+        fprintf(fp, "# vtk DataFile Version 3.0\n");
+        fprintf(fp, "PAMPI cfd solver particle tracing file\n");
+        fprintf(fp, "ASCII\n");
+
+
+        fprintf(fp, "DATASET UNSTRUCTURED_GRID\n");
+        fprintf(fp, "FIELD FieldData 2\n");
+        fprintf(fp, "TIME 1 1 double\n");
+        fprintf(fp, "%d\n", ts);
+        fprintf(fp, "CYCLE 1 1 int\n");
+        fprintf(fp, "1\n");
+
+        int overallTotalParticles = sum(collectedBuffIndex, particletracer->size);
+
+        fprintf(fp, "POINTS %d float\n", overallTotalParticles);
+
+        printf("Total particles : %d\n", overallTotalParticles);
+
+        for (int i = 1; i < particletracer->size; ++i)
+        {
+            Particle recvBuff[collectedBuffIndex[i]];
+            MPI_Recv(&recvBuff, collectedBuffIndex[i], particletracer->mpi_particle, i, 1, particletracer->comm, MPI_STATUS_IGNORE);
+
+            for (int j = 0; j < collectedBuffIndex[i]; ++j)
+            {
+                double x = recvBuff[j].x;
+                double y = recvBuff[j].y;
+                double z = recvBuff[j].z;
+                fprintf(fp, "%f %f %f\n", x, y, z);
+                //printf("Rank : 0 receiving from rank %d X : %.2f, Y : %.2f with totalpt : %d\n", i, x, y, particletracer->totalParticles);
+
+            }
+        
+        }
+        for (int i = 0; i < particletracer->totalParticles; ++i)
+        {
+            double x = particletracer->particlePool[i].x;
+            double y = particletracer->particlePool[i].y;
+            double z = particletracer->particlePool[i].z;
+            fprintf(fp, "%f %f %f\n", x, y, z);
+        }
+
+    fprintf(fp, "CELLS %d %d\n", overallTotalParticles, 2 * overallTotalParticles);
+
+
+    for (int i = 0; i < overallTotalParticles; ++i)
+    {
+        fprintf(fp, "1 %d\n", i);
+    }
+
+    fprintf(fp, "CELL_TYPES %d\n", overallTotalParticles);
+
+
+    for (int i = 0; i < overallTotalParticles; ++i)
+    {
+        fprintf(fp, "1\n");
+    }
+
+        fclose(fp);
+    }
+
+    ++ts;
+}
+
+void initParticleTracer(ParticleTracer* particletracer, Parameter* params, Solver* solver)
+{
+    /* initializing local properties from params */
+
+    particletracer->numberOfParticles = params->numberOfParticles;
+    particletracer->startTime         = params->startTime;
+    particletracer->injectTimePeriod  = params->injectTimePeriod;
+    particletracer->writeTimePeriod   = params->writeTimePeriod;
+
+    particletracer->dt                = params->dt;
+    particletracer->dx                = params->xlength / params->imax;
+    particletracer->dy                = params->ylength / params->jmax;
+    particletracer->dz                = params->zlength / params->kmax;
+
+    particletracer->xlength           = params->xlength;
+    particletracer->ylength           = params->ylength;
+    particletracer->zlength           = params->zlength;
+    
+    particletracer->x1                = params->x1;
+    particletracer->y1                = params->y1;
+    particletracer->z1                = params->z1;
+    particletracer->x2                = params->x2;
+    particletracer->y2                = params->y2;
+    particletracer->z2                = params->z2;
+
+    particletracer->lastInjectTime    = params->startTime;
+    particletracer->lastUpdateTime    = params->startTime;
+    particletracer->lastWriteTime     = params->startTime;
+
+    particletracer->pointer           = 0;
+    particletracer->totalParticles    = 0;
+
+    particletracer->imax              = params->imax;
+    particletracer->jmax              = params->jmax;
+    particletracer->kmax              = params->kmax;
+
+    particletracer->imaxLocal         = solver->comm.imaxLocal;
+    particletracer->jmaxLocal         = solver->comm.jmaxLocal;
+    particletracer->kmaxLocal         = solver->comm.kmaxLocal;
+
+    particletracer->estimatedNumParticles = (particletracer->imaxLocal * particletracer->jmaxLocal * particletracer->kmaxLocal);
+
+    particletracer->particlePool      = malloc(sizeof(Particle) * particletracer->estimatedNumParticles);
+
+    for(int i = 0; i < particletracer->estimatedNumParticles; ++i)
+    {   
+        particletracer->particlePool[i].x = 0.0;
+        particletracer->particlePool[i].y = 0.0;
+        particletracer->particlePool[i].z = 0.0;
+        particletracer->particlePool[i].flag = false;
+    }
+
+    particletracer->linSpaceLine      = malloc(sizeof(Particle) * particletracer->numberOfParticles);
+
+    /* duplicating communication from solver */
+    MPI_Comm_dup(solver->comm.comm, &particletracer->comm);
+
+    particletracer->rank              = solver->comm.rank;
+    particletracer->size              = solver->comm.size;
+
+    particletracer->offset            = (double *)malloc(sizeof(double) * 4 * particletracer->size);
+
+
+    memcpy(particletracer->dims, solver->comm.dims, sizeof(solver->comm.dims));
+
+    memcpy(particletracer->coords, solver->comm.coords, sizeof(solver->comm.coords));
+
+    particletracer->xLocal           = particletracer->imaxLocal * particletracer->dx;
+    particletracer->yLocal           = particletracer->jmaxLocal * particletracer->dy;
+    particletracer->zLocal           = particletracer->kmaxLocal * particletracer->dz;
+
+
+    double xLocal[particletracer->size];
+    double yLocal[particletracer->size];
+    double zLocal[particletracer->size];
+
+    double offset[6][particletracer->size];
+    
+    MPI_Allgather(&particletracer->xLocal, 1, MPI_DOUBLE, xLocal, 1, MPI_DOUBLE, particletracer->comm);
+    MPI_Allgather(&particletracer->yLocal, 1, MPI_DOUBLE, yLocal, 1, MPI_DOUBLE, particletracer->comm);
+    MPI_Allgather(&particletracer->zLocal, 1, MPI_DOUBLE, zLocal, 1, MPI_DOUBLE, particletracer->comm);
+
+    particletracer->xOffset     = sumOffset(xLocal, particletracer->rank, (particletracer->dims[1] * particletracer->dims[2]), particletracer->coords[0]);
+    particletracer->yOffset     = sumOffset(yLocal, particletracer->rank, particletracer->dims[2], particletracer->coords[1]);
+    particletracer->zOffset     = sumOffset(zLocal, particletracer->rank, 1, particletracer->coords[2]);
+    particletracer->xOffsetEnd  = particletracer->xOffset + particletracer->xLocal;
+    particletracer->yOffsetEnd  = particletracer->yOffset + particletracer->yLocal;
+    particletracer->zOffsetEnd  = particletracer->zOffset + particletracer->zLocal;
+
+    //printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, zOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : %.2f, zOffsetEnd : %.2f\n", particletracer->rank, particletracer->xOffset, particletracer->yOffset, particletracer->zOffset, particletracer->xOffsetEnd, particletracer->yOffsetEnd, particletracer->zOffsetEnd);
+
+    MPI_Allgather(&particletracer->xOffset, 1, MPI_DOUBLE, offset[0], 1, MPI_DOUBLE, particletracer->comm);
+    MPI_Allgather(&particletracer->yOffset, 1, MPI_DOUBLE, offset[1], 1, MPI_DOUBLE, particletracer->comm);
+    MPI_Allgather(&particletracer->zOffset, 1, MPI_DOUBLE, offset[2], 1, MPI_DOUBLE, particletracer->comm);
+    MPI_Allgather(&particletracer->xOffsetEnd, 1, MPI_DOUBLE, offset[3], 1, MPI_DOUBLE, particletracer->comm);
+    MPI_Allgather(&particletracer->yOffsetEnd, 1, MPI_DOUBLE, offset[4], 1, MPI_DOUBLE, particletracer->comm);
+    MPI_Allgather(&particletracer->zOffsetEnd, 1, MPI_DOUBLE, offset[5], 1, MPI_DOUBLE, particletracer->comm);
+
+    memcpy(particletracer->offset, offset, sizeof(offset));
+
+    
+    // if(particletracer->rank == 0)
+    // {   
+    //     for(int i = 0;i < particletracer->size; ++i)
+    //     {
+    //         printf("Rank : %d, xLocal : %.2f, yLocal : %.2f, zLocal : %.2f\n", i, xLocal[i], yLocal[i], zLocal[i]);
+    //     }
+    //     for(int i = 0;i < particletracer->size; ++i)
+    //     {
+    //         printf("Rank : %d and its xOffset : %.2f, yOffset : %.2f, zOffset : %.2f,xOffsetEnd : %.2f, yOffsetEnd : %.2f, zOffsetEnd : %.2f\n", 
+    //         i, particletracer->offset[i + particletracer->size * XOFFSET],
+    //         particletracer->offset[i + particletracer->size * YOFFSET], particletracer->offset[i + particletracer->size * ZOFFSET]
+    //         , particletracer->offset[i + particletracer->size * XOFFSETEND], 
+    //         particletracer->offset[i + particletracer->size * YOFFSETEND], particletracer->offset[i + particletracer->size * ZOFFSETEND]);
+    //     }
+    // }
+    
+    for (int i = 0; i < particletracer->numberOfParticles; ++i) 
+    {
+        // double spacing                      = (double)i / (double)(particletracer->numberOfParticles - 1);
+        // particletracer->linSpaceLine[i].x   = spacing * particletracer->x1 + (1.0 - spacing) * particletracer->x2;
+        // particletracer->linSpaceLine[i].y   = spacing * particletracer->y1 + (1.0 - spacing) * particletracer->y2;
+        // particletracer->linSpaceLine[i].z   = spacing * particletracer->z1 + (1.0 - spacing) * particletracer->z2;
+
+        particletracer->linSpaceLine[i].x   = particletracer->x1;
+        particletracer->linSpaceLine[i].y   = (double) rand() / RAND_MAX  * particletracer->ylength;
+        particletracer->linSpaceLine[i].z   = (double) rand() / RAND_MAX  * particletracer->zlength;
+
+        particletracer->linSpaceLine[i].flag = true;
+    }
+
+    
+    // Create the mpi_particle datatype
+    MPI_Datatype mpi_particle;
+    int lengths[4] = { 1, 1, 1, 1 };
+ 
+    MPI_Aint displacements[4];
+    Particle dummy_particle;
+    MPI_Aint base_address;
+    MPI_Get_address(&dummy_particle, &base_address);
+    MPI_Get_address(&dummy_particle.x, &displacements[0]);
+    MPI_Get_address(&dummy_particle.y, &displacements[1]);
+    MPI_Get_address(&dummy_particle.z, &displacements[2]);
+    MPI_Get_address(&dummy_particle.flag, &displacements[3]);
+    displacements[0] = MPI_Aint_diff(displacements[0], base_address);
+    displacements[1] = MPI_Aint_diff(displacements[1], base_address);
+    displacements[2] = MPI_Aint_diff(displacements[2], base_address);
+    displacements[3] = MPI_Aint_diff(displacements[3], base_address);
+    MPI_Datatype types[4] = { MPI_DOUBLE, MPI_DOUBLE, MPI_DOUBLE, MPI_C_BOOL };
+    MPI_Type_create_struct(4, lengths, displacements, types, &particletracer->mpi_particle);
+    MPI_Type_commit(&particletracer->mpi_particle);
+
+}
+
+void printParticleTracerParameters(ParticleTracer* particletracer)
+{
+    printf("Particle Tracing data:\n");
+    printf("Rank : %d\n", particletracer->rank);
+    printf("\tNumber of particles : %d being injected for every period of %.2f\n", particletracer->numberOfParticles, particletracer->injectTimePeriod);
+    printf("\tstartTime : %.2f\n", particletracer->startTime);
+    printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : %.2f, z1 : %.2f, x2 : %.2f, y2 : %.2f,  z2 : %.2f\n", particletracer->x1, particletracer->y1, particletracer->z1, particletracer->x2, particletracer->y2, particletracer->z2);
+    printf("\tPointer : %d, TotalParticles : %d\n", particletracer->pointer, particletracer->totalParticles);
+    printf("\tdt : %.2f, dx : %.2f, dy : %.2f, dz : %.2f\n", particletracer->dt, particletracer->dx, particletracer->dy, particletracer->dz);
+    printf("\tcoord[0] : %d, coord[1] : %d, coord[2] : %d\n", particletracer->coords[IDIM], particletracer->coords[JDIM], particletracer->coords[KDIM]);
+    printf("\txOffset : %.2f, yOffset : %.2f, zOffset : %.2f\n", particletracer->xOffset, particletracer->yOffset, particletracer->zOffset);
+    printf("\txOffsetEnd : %.2f, yOffsetEnd : %.2f, zOffsetEnd : %.2f\n", particletracer->xOffsetEnd, particletracer->yOffsetEnd, particletracer->zOffsetEnd);
+    printf("\txLocal : %.2f, yLocal : %.2f, zLocal : %.2f\n", particletracer->xLocal, particletracer->yLocal, particletracer->zLocal);
+}   
+
+void trace(ParticleTracer* particletracer, double* restrict u, double* restrict v, double* restrict w, double time)
+{
+    if (time >= particletracer->startTime)
+    {
+        if( (time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod)
+        {
+            injectParticles(particletracer);
+            particletracer->lastInjectTime = time;
+        }
+        if( (time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod)
+        {
+            writeParticles(particletracer);
+            particletracer->lastWriteTime = time;
+        }
+        advanceParticles(particletracer, u, v, w, time);
+        compress(particletracer);
+        particletracer->lastUpdateTime = time;   
+    }
+}
+
+void compress(ParticleTracer* particletracer)
+{
+    Particle* memPool = particletracer->particlePool;
+    Particle tempPool[particletracer->totalParticles];
+
+    for(int i = 0; i < particletracer->totalParticles; ++i)
+    {   
+        tempPool[i].x = 0.0;
+        tempPool[i].y = 0.0;
+        tempPool[i].z = 0.0;
+        tempPool[i].flag = false;
+    }
+    int totalParticles = 0;
+
+    for(int i=0; i < particletracer->totalParticles; ++i)
+    {
+        if(memPool[i].flag == true)
+        {
+            tempPool[totalParticles].x    = memPool[i].x;
+            tempPool[totalParticles].y    = memPool[i].y;
+            tempPool[totalParticles].z    = memPool[i].z;
+            tempPool[totalParticles].flag = memPool[i].flag;
+            ++totalParticles;
+        }
+    }
+
+    particletracer->totalParticles = totalParticles;
+    particletracer->pointer        = totalParticles;
+
+    memcpy(particletracer->particlePool, tempPool, totalParticles * sizeof(Particle));
+}

BasicSolver/3D-mpi/src/particletracing.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+ * All rights reserved. This file is part of nusif-solver.
+ * Use of this source code is governed by a MIT style
+ * license that can be found in the LICENSE file.
+ */
+#ifndef __PARTICLETRACING_H_
+#define __PARTICLETRACING_H_
+#include "allocate.h"
+#include "parameter.h"
+#include "solver.h"
+#include <stdbool.h>
+#include <mpi.h>
+
+#define NDIMS 3
+
+typedef enum COORD { X = 0, Y, NCOORD } COORD;
+typedef struct
+{
+    double x, y, z;
+    bool flag;
+} Particle;
+
+typedef struct {
+    int numberOfParticles, totalParticles;
+    double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime, lastWriteTime;
+
+    int estimatedNumParticles;
+
+    double dx, dy,dz, dt;
+    Particle* linSpaceLine;
+    Particle* particlePool;
+
+    int pointer;
+    
+    double imax, jmax, kmax, xlength, ylength, zlength, imaxLocal, jmaxLocal, kmaxLocal;
+
+    double x1, y1, z1, x2, y2, z2;
+
+    MPI_Comm comm;
+    MPI_Datatype mpi_particle;
+    
+    int rank, size;
+    int coords[NDIMS], dims[NDIMS];
+
+    double xLocal, yLocal, zLocal, xOffset, yOffset, zOffset, xOffsetEnd, yOffsetEnd, zOffsetEnd;
+
+    double* offset;
+
+} ParticleTracer;
+
+void initParticleTracer(ParticleTracer*, Parameter*, Solver* );
+void injectParticles(ParticleTracer*);
+void advanceParticles(ParticleTracer*, double* , double* , double* , double);
+void freeParticles(ParticleTracer*);
+void writeParticles(ParticleTracer*);
+void printParticleTracerParameters(ParticleTracer*);
+void printParticles(ParticleTracer*);
+void trace(ParticleTracer*, double* , double* , double* , double );
+void compress(ParticleTracer* );
+#endif

BasicSolver/3D-mpi/src/solver.c

@@ -141,7 +141,7 @@ void initSolver(Solver* s, Parameter* params)
     s->dtBound       = 0.5 * s->re * 1.0 / invSqrSum;
 
 #ifdef VERBOSE
-    printConfig(s);
+    //printConfig(s);
 #endif /* VERBOSE */
 }