moebiusband/NuSiF-Solver
2
0
Fork 2
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Working 2D-mpi for enhanced solver

Browse Source
This commit is contained in:
Aditya Ujeniya
2023-10-09 16:56:43 +02:00
parent fcff353999
commit 86f7677f34
31 changed files with 39012 additions and 171322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
BasicSolver/2D-mpi-v3/src/main.c
View File

@@ -11,18 +11,14 @@
#include <unistd.h>
#include "parameter.h"
#include "progress.h"
#include "solver.h"
#include "timing.h"
#include "particletracing.h"
#include "vtkWriter.h"
#include "progress.h"
#include "timing.h"
#include <mpi.h>
enum VARIANT { SOR = 1, RB, RBA };
int main (int argc, char** argv)
int main(int argc, char** argv)
{
int rank;
int variant = RB;
@@ -32,7 +28,6 @@ int main (int argc, char** argv)
Solver solver;
ParticleTracer particletracer;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
initParameter(&params);
@@ -43,18 +38,16 @@ int main (int argc, char** argv)
}
readParameter(&params, argv[1]);
if (argc == 3)
{
if (argc == 3) {
variant = atoi(argv[2]);
}
initSolver(&solver, &params);
initParticleTracer(&particletracer, &params, &solver);
if (rank == 0)
{
if (rank == 0) {
printParameter(&params);
}
printParticleTracerParameters(&particletracer);
initProgress(solver.te);
@@ -62,28 +55,26 @@ int main (int argc, char** argv)
double te = solver.te;
double t = 0.0;
int (*solver_generic[])() = {solve, solveRB, solveRBA};
int (*solver_generic[])() = { solve, solveRB, solveRBA };
S = getTimeStamp();
while (t <= te)
{
while (t <= te) {
if (tau > 0.0) {
computeTimestep(&solver);
}
setBoundaryConditions(&solver);
setSpecialBoundaryCondition(&solver);
setObjectBoundaryCondition(&solver);
computeFG(&solver);
computeRHS(&solver);
solver_generic[variant - 1](&solver);
adaptUV(&solver);
//trace(&particletracer, solver.u, solver.v, t);
trace(&particletracer, solver.u, solver.v, solver.s, t);
t += solver.dt;
#ifdef VERBOSE
if (rank == 0) {
@@ -102,7 +93,6 @@ int main (int argc, char** argv)
collectResult(&solver);
freeParticles(&particletracer);
MPI_Finalize();
return EXIT_SUCCESS;
}

10
BasicSolver/2D-mpi-v3/src/parameter.c
View File

@@ -25,7 +25,6 @@ void initParameter(Parameter* param)
param->gamma = 0.9;
param->tau = 0.5;
param->rho = 0.99;
}
void readParameter(Parameter* param, const char* filename)
@@ -82,7 +81,6 @@ void readParameter(Parameter* param, const char* filename)
PARSE_REAL(p_init);
PARSE_REAL(rho);
/* Added new particle tracing parameters */
PARSE_INT(numberOfParticles);
PARSE_REAL(startTime);
@@ -92,6 +90,14 @@ void readParameter(Parameter* param, const char* filename)
PARSE_REAL(y1);
PARSE_REAL(x2);
PARSE_REAL(y2);
/* Added obstacle geometry parameters */
PARSE_INT(shape);
PARSE_REAL(xCenter);
PARSE_REAL(yCenter);
PARSE_REAL(xRectLength);
PARSE_REAL(yRectLength);
PARSE_REAL(circleRadius);
}
}

3
BasicSolver/2D-mpi-v3/src/parameter.h
View File

@@ -23,6 +23,9 @@ typedef struct {
double startTime, injectTimePeriod, writeTimePeriod;
double x1, y1, x2, y2;
int shape;
double xCenter, yCenter, xRectLength, yRectLength, circleRadius;
} Parameter;
void initParameter(Parameter*);

583
BasicSolver/2D-mpi-v3/src/particletracing.c
View File

@@ -10,18 +10,19 @@
#include <stdlib.h>
#include <string.h>
#include "vtkWriter.h"
#include "particletracing.h"
#define U(i, j) u[(j) * (imaxLocal + 2) + (i)]
#define V(i, j) v[(j) * (imaxLocal + 2) + (i)]
#define U(i, j) u[(j) * (imaxLocal + 2) + (i)]
#define V(i, j) v[(j) * (imaxLocal + 2) + (i)]
#define S(i, j) s[(j) * (imaxLocal + 2) + (i)]
static int ts = 0;
#define IDIM 0
#define JDIM 1
#define XOFFSET 0
#define YOFFSET 1
#define XOFFSET 0
#define YOFFSET 1
#define XOFFSETEND 2
#define YOFFSETEND 3
@@ -29,32 +30,7 @@ static double sum(int* sizes, int size)
{
double sum = 0;
for (int i = 0; i < size; ++i)
{
sum += sizes[i];
}
return sum;
}
static double sumX(double* sizes, int size, int offset, int coord, int init)
{
double sum = 0;
for (int i = init; i < size && coord > 0; i+=offset, --coord)
{
sum += sizes[i];
}
return sum;
}
static double sumY(double* sizes, int size, int offset, int coord, int init)
{
double sum = 0;
for (int i = init * offset; i < size && coord > 0; ++i, --coord)
{
for (int i = 0; i < size; ++i) {
sum += sizes[i];
}
@@ -70,7 +46,7 @@ void printUV(ParticleTracer* particletracer, double* u, double* v)
printf(
"\n### RANK %d #######################################################\n",
particletracer->rank);
printf("\nGrid U : \n");
printf("\nGrid U : \n");
for (int j = 0; j < particletracer->jmaxLocal + 2; j++) {
printf("%02d: ", j);
for (int i = 0; i < particletracer->imaxLocal + 2; i++) {
@@ -95,33 +71,37 @@ void printUV(ParticleTracer* particletracer, double* u, double* v)
void printParticles(ParticleTracer* particletracer)
{
for(int i = 0; i < particletracer->totalParticles; ++i)
{
printf("Rank : %d Particle position X : %.2f, Y : %.2f, flag : %d, total pt : %d, pointer : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : %.2f\n",
particletracer->rank,
for (int i = 0; i < particletracer->totalParticles; ++i) {
printf("Rank : %d Particle position X : %.2f, Y : %.2f, flag : %d, total pt : "
"%d, pointer : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, "
"yOffsetEnd : %.2f\n",
particletracer->rank,
particletracer->particlePool[i].x,
particletracer->particlePool[i].y,
particletracer->particlePool[i].flag,
particletracer->totalParticles,
particletracer->pointer,
particletracer->xOffset, particletracer->yOffset,
particletracer->xOffsetEnd, particletracer->yOffsetEnd);
particletracer->particlePool[i].flag,
particletracer->totalParticles,
particletracer->pointer,
particletracer->xOffset,
particletracer->yOffset,
particletracer->xOffsetEnd,
particletracer->yOffsetEnd);
}
}
void injectParticles(ParticleTracer* particletracer)
{
double x, y;
for(int i = 0; i < particletracer->numberOfParticles; ++i)
{
for (int i = 0; i < particletracer->numberOfParticles; ++i) {
x = particletracer->linSpaceLine[i].x;
y = particletracer->linSpaceLine[i].y;
if(x >= particletracer->xOffset && y >= particletracer->yOffset && x <= particletracer->xOffsetEnd && y <= particletracer->yOffsetEnd )
{
if (x >= particletracer->xOffset && y >= particletracer->yOffset &&
x <= particletracer->xOffsetEnd && y <= particletracer->yOffsetEnd) {
// printf("\nRank : %d\n", particletracer->rank);
// printf("\t%.2f >= %.2f && %.2f >= %.2f && %.2f <= %.2f && %.2f <= %.2f\n",x , particletracer->xOffset ,y , particletracer->yOffset, x , particletracer->xOffsetEnd ,y , particletracer->yOffsetEnd);
particletracer->particlePool[particletracer->pointer].x = x;
particletracer->particlePool[particletracer->pointer].y = y;
// printf("\t%.2f >= %.2f && %.2f >= %.2f && %.2f <= %.2f && %.2f <= %.2f\n",x
// , particletracer->xOffset ,y , particletracer->yOffset, x ,
// particletracer->xOffsetEnd ,y , particletracer->yOffsetEnd);
particletracer->particlePool[particletracer->pointer].x = x;
particletracer->particlePool[particletracer->pointer].y = y;
particletracer->particlePool[particletracer->pointer].flag = true;
++(particletracer->pointer);
++(particletracer->totalParticles);
@@ -129,27 +109,29 @@ void injectParticles(ParticleTracer* particletracer)
}
}
void advanceParticles(ParticleTracer* particletracer, double* restrict u, double* restrict v, double time)
void advanceParticles(ParticleTracer* particletracer,
double* restrict u,
double* restrict v,
int* restrict s,
double time)
{
int imax = particletracer->imax;
int jmax = particletracer->jmax;
int imaxLocal = particletracer->imaxLocal;
int jmaxLocal = particletracer->jmaxLocal;
int imax = particletracer->imax;
int jmax = particletracer->jmax;
int imaxLocal = particletracer->imaxLocal;
int jmaxLocal = particletracer->jmaxLocal;
double dx = particletracer->dx;
double dy = particletracer->dy;
double dx = particletracer->dx;
double dy = particletracer->dy;
double xlength = particletracer->xlength;
double ylength = particletracer->ylength;
Particle buff[particletracer->size][30];
Particle buff[particletracer->size][100];
for(int i = 0; i < particletracer->size; ++i)
{
for(int j = 0; j < 30; ++j)
{
buff[i][j].x = 0.0;
buff[i][j].y = 0.0;
for (int i = 0; i < particletracer->size; ++i) {
for (int j = 0; j < 100; ++j) {
buff[i][j].x = 0.0;
buff[i][j].y = 0.0;
buff[i][j].flag = false;
}
}
@@ -157,16 +139,13 @@ void advanceParticles(ParticleTracer* particletracer, double* restrict u, double
memset(particleBufIndex, 0, sizeof(particleBufIndex));
for(int i = 0; i < particletracer->totalParticles; ++i)
{
if(particletracer->particlePool[i].flag == true)
{
for (int i = 0; i < particletracer->totalParticles; ++i) {
if (particletracer->particlePool[i].flag == true) {
double xTemp = particletracer->particlePool[i].x;
double yTemp = particletracer->particlePool[i].y;
double x = xTemp - particletracer->xOffset;
double y = yTemp - particletracer->yOffset;
double x = xTemp - particletracer->xOffset;
double y = yTemp - particletracer->yOffset;
int iCoord = (int)(x / dx) + 1;
int jCoord = (int)((y + 0.5 * dy) / dy) + 1;
@@ -177,10 +156,10 @@ void advanceParticles(ParticleTracer* particletracer, double* restrict u, double
double y2 = ((double)jCoord - 0.5) * dy;
double u_n = (1.0 / (dx * dy)) *
((x2 - x) * (y2 - y) * U(iCoord - 1, jCoord - 1) +
(x - x1) * (y2 - y) * U(iCoord, jCoord - 1) +
(x2 - x) * (y - y1) * U(iCoord - 1, jCoord) +
(x - x1) * (y - y1) * U(iCoord, jCoord));
((x2 - x) * (y2 - y) * U(iCoord - 1, jCoord - 1) +
(x - x1) * (y2 - y) * U(iCoord, jCoord - 1) +
(x2 - x) * (y - y1) * U(iCoord - 1, jCoord) +
(x - x1) * (y - y1) * U(iCoord, jCoord));
double new_x = (x + particletracer->xOffset) + particletracer->dt * u_n;
particletracer->particlePool[i].x = new_x;
@@ -194,68 +173,92 @@ void advanceParticles(ParticleTracer* particletracer, double* restrict u, double
y2 = (double)jCoord * dy;
double v_n = (1.0 / (dx * dy)) *
((x2 - x) * (y2 - y) * V(iCoord - 1, jCoord - 1) +
(x - x1) * (y2 - y) * V(iCoord, jCoord - 1) +
(x2 - x) * (y - y1) * V(iCoord - 1, jCoord) +
(x - x1) * (y - y1) * V(iCoord, jCoord));
((x2 - x) * (y2 - y) * V(iCoord - 1, jCoord - 1) +
(x - x1) * (y2 - y) * V(iCoord, jCoord - 1) +
(x2 - x) * (y - y1) * V(iCoord - 1, jCoord) +
(x - x1) * (y - y1) * V(iCoord, jCoord));
double new_y = (y + particletracer->yOffset) + particletracer->dt * v_n;
particletracer->particlePool[i].y = new_y;
// printf("Rank : %d\n", particletracer->rank);
// printf("\tOld X : %.2f, translated X : %.2f, xOffset : %.2f, New X : %.2f, iCoord : %d\n\tOld Y : %.2f, translated X : %.2f, yOffset : %.2f, New Y : %.2f, jCoord : %d\n\n",xTemp, x, particletracer->xOffset, new_x, iCoord, yTemp, y, particletracer->yOffset , new_y, jCoord);
// printf("\tU(iCoord - 1, jCoord - 1) : %.2f, U(iCoord, jCoord - 1) : %.2f, U(iCoord - 1, jCoord) : %.2f, U(iCoord, jCoord) : %.2f\n", U(iCoord - 1, jCoord - 1), U(iCoord, jCoord - 1), U(iCoord - 1, jCoord), U(iCoord, jCoord));
// printf("\tV(iCoord - 1, jCoord - 1) : %.2f, V(iCoord, jCoord - 1) : %.2f, V(iCoord - 1, jCoord) : %.2f, V(iCoord, jCoord) : %.2f\n\n", V(iCoord - 1, jCoord - 1), V(iCoord, jCoord - 1), V(iCoord - 1, jCoord), V(iCoord, jCoord));
//printf("\t U N : %.2f, V N : %.2f\n\n", u_n, v_n);
// printf("\tOld X : %.2f, translated X : %.2f, xOffset : %.2f, New X : %.2f,
// iCoord : %d\n\tOld Y : %.2f, translated X : %.2f, yOffset : %.2f, New Y :
// %.2f, jCoord : %d\n\n",xTemp, x, particletracer->xOffset, new_x, iCoord,
// yTemp, y, particletracer->yOffset , new_y, jCoord); printf("\tU(iCoord - 1,
// jCoord - 1) : %.2f, U(iCoord, jCoord - 1) : %.2f, U(iCoord - 1, jCoord) :
// %.2f, U(iCoord, jCoord) : %.2f\n", U(iCoord - 1, jCoord - 1), U(iCoord,
// jCoord - 1), U(iCoord - 1, jCoord), U(iCoord, jCoord)); printf("\tV(iCoord
// - 1, jCoord - 1) : %.2f, V(iCoord, jCoord - 1) : %.2f, V(iCoord - 1,
// jCoord) : %.2f, V(iCoord, jCoord) : %.2f\n\n", V(iCoord - 1, jCoord - 1),
// V(iCoord, jCoord - 1), V(iCoord - 1, jCoord), V(iCoord, jCoord));
// printf("\t U N : %.2f, V N : %.2f\n\n", u_n, v_n);
if (((new_x < particletracer->xOffset) || (new_x >= particletracer->xOffsetEnd) ||
(new_y < particletracer->yOffset) || (new_y >= particletracer->yOffsetEnd)))
{
//New logic to transfer particles to neighbouring ranks or discard the particle.
if (((new_x < particletracer->xOffset) ||
(new_x >= particletracer->xOffsetEnd) ||
(new_y < particletracer->yOffset) ||
(new_y >= particletracer->yOffsetEnd))) {
// New logic to transfer particles to neighbouring ranks or discard the
// particle.
for(int i = 0; i < particletracer->size; ++i)
{
if((new_x >= particletracer->offset[i + particletracer->size * XOFFSET]) &&
(new_x <= particletracer->offset[i + particletracer->size * XOFFSETEND]) &&
(new_y >= particletracer->offset[i + particletracer->size * YOFFSET]) &&
(new_y <= particletracer->offset[i + particletracer->size * YOFFSETEND]) &&
i != particletracer->rank)
{
buff[i][particleBufIndex[i]].x = new_x;
buff[i][particleBufIndex[i]].y = new_y;
for (int i = 0; i < particletracer->size; ++i) {
if ((new_x >=
particletracer->offset[i + particletracer->size * XOFFSET]) &&
(new_x <= particletracer
->offset[i + particletracer->size * XOFFSETEND]) &&
(new_y >=
particletracer->offset[i + particletracer->size * YOFFSET]) &&
(new_y <= particletracer
->offset[i + particletracer->size * YOFFSETEND]) &&
i != particletracer->rank) {
buff[i][particleBufIndex[i]].x = new_x;
buff[i][particleBufIndex[i]].y = new_y;
buff[i][particleBufIndex[i]].flag = true;
++particleBufIndex[i];
}
}
particletracer->particlePool[i].flag = false;
}
int i_new = new_x / dx, j_new = new_y / dy;
int iOffset = particletracer->xOffset / dx,
jOffset = particletracer->yOffset / dy;
if (S(i_new - iOffset, j_new - jOffset) != NONE) {
particletracer->particlePool[i].flag = false;
}
}
}
for(int i = 0; i < particletracer->size; ++i)
{
if(i != particletracer->rank)
{
MPI_Send(buff[i], 30, particletracer->mpi_particle, i, 0, particletracer->comm);
for (int i = 0; i < particletracer->size; ++i) {
if (i != particletracer->rank) {
MPI_Send(buff[i],
100,
particletracer->mpi_particle,
i,
0,
particletracer->comm);
}
}
for(int i = 0; i < particletracer->size; ++i)
{
if(i != particletracer->rank)
{
MPI_Recv(buff[i], 30, particletracer->mpi_particle, i, 0, particletracer->comm, MPI_STATUS_IGNORE);
for (int i = 0; i < particletracer->size; ++i) {
if (i != particletracer->rank) {
MPI_Recv(buff[i],
100,
particletracer->mpi_particle,
i,
0,
particletracer->comm,
MPI_STATUS_IGNORE);
}
}
for(int i = 0; i < particletracer->size; ++i)
{
if(i != particletracer->rank)
{
for(int j = 0; j < 30; ++j)
{
if(buff[i][j].flag == true)
{
particletracer->particlePool[particletracer->pointer].x = buff[i][j].x;
particletracer->particlePool[particletracer->pointer].y = buff[i][j].y;
for (int i = 0; i < particletracer->size; ++i) {
if (i != particletracer->rank) {
for (int j = 0; j < 100; ++j) {
if (buff[i][j].flag == true) {
particletracer->particlePool[particletracer->pointer].x = buff[i][j]
.x;
particletracer->particlePool[particletracer->pointer].y = buff[i][j]
.y;
particletracer->particlePool[particletracer->pointer].flag = true;
++(particletracer->pointer);
++(particletracer->totalParticles);
@@ -276,87 +279,98 @@ void writeParticles(ParticleTracer* particletracer)
{
int collectedBuffIndex[particletracer->size];
MPI_Gather(&particletracer->totalParticles, 1, MPI_INT, collectedBuffIndex, 1, MPI_INT, 0, particletracer->comm);
MPI_Gather(&particletracer->totalParticles,
1,
MPI_INT,
collectedBuffIndex,
1,
MPI_INT,
0,
particletracer->comm);
if(particletracer->rank != 0)
{
if (particletracer->rank != 0) {
Particle buff[particletracer->totalParticles];
for(int i = 0; i < particletracer->totalParticles; ++i)
{
buff[i].x = particletracer->particlePool[i].x;
buff[i].y = particletracer->particlePool[i].y;
buff[i].flag = particletracer->particlePool[i].flag;
//printf("Rank : %d sending to rank 0 X : %.2f, Y : %.2f with totalpt : %d\n", particletracer->rank, buff[i].x, buff[i].y, particletracer->totalParticles);
for (int i = 0; i < particletracer->totalParticles; ++i) {
buff[i].x = particletracer->particlePool[i].x;
buff[i].y = particletracer->particlePool[i].y;
buff[i].flag = particletracer->particlePool[i].flag;
// printf("Rank : %d sending to rank 0 X : %.2f, Y : %.2f with totalpt :
// %d\n", particletracer->rank, buff[i].x, buff[i].y,
// particletracer->totalParticles);
}
MPI_Send(buff, particletracer->totalParticles, particletracer->mpi_particle, 0, 1, particletracer->comm);
MPI_Send(buff,
particletracer->totalParticles,
particletracer->mpi_particle,
0,
1,
particletracer->comm);
}
if(particletracer->rank == 0)
{
if (particletracer->rank == 0) {
char filename[50];
FILE* fp;
snprintf(filename, 50, "vtk_files/particles_%d.vtk", ts);
snprintf(filename, 50, "vis_files/particles_%d.dat", ts);
fp = fopen(filename, "w");
if (fp == NULL) {
printf("Error!\n");
exit(EXIT_FAILURE);
}
fprintf(fp, "# vtk DataFile Version 3.0\n");
fprintf(fp, "PAMPI cfd solver particle tracing file\n");
fprintf(fp, "ASCII\n");
// fprintf(fp, "# vtk DataFile Version 3.0\n");
// fprintf(fp, "PAMPI cfd solver particle tracing file\n");
// fprintf(fp, "ASCII\n");
fprintf(fp, "DATASET UNSTRUCTURED_GRID\n");
fprintf(fp, "FIELD FieldData 2\n");
fprintf(fp, "TIME 1 1 double\n");
fprintf(fp, "%d\n", ts);
fprintf(fp, "CYCLE 1 1 int\n");
fprintf(fp, "1\n");
// fprintf(fp, "DATASET UNSTRUCTURED_GRID\n");
// fprintf(fp, "FIELD FieldData 2\n");
// fprintf(fp, "TIME 1 1 double\n");
// fprintf(fp, "%d\n", ts);
// fprintf(fp, "CYCLE 1 1 int\n");
// fprintf(fp, "1\n");
int overallTotalParticles = sum(collectedBuffIndex, particletracer->size);
fprintf(fp, "POINTS %d float\n", overallTotalParticles);
// fprintf(fp, "POINTS %d float\n", overallTotalParticles);
printf("Total particles : %d\n", overallTotalParticles);
// printf("Total particles : %d\n", overallTotalParticles);
for (int i = 1; i < particletracer->size; ++i)
{
for (int i = 1; i < particletracer->size; ++i) {
Particle recvBuff[collectedBuffIndex[i]];
MPI_Recv(&recvBuff, collectedBuffIndex[i], particletracer->mpi_particle, i, 1, particletracer->comm, MPI_STATUS_IGNORE);
MPI_Recv(&recvBuff,
collectedBuffIndex[i],
particletracer->mpi_particle,
i,
1,
particletracer->comm,
MPI_STATUS_IGNORE);
for (int j = 0; j < collectedBuffIndex[i]; ++j)
{
for (int j = 0; j < collectedBuffIndex[i]; ++j) {
double x = recvBuff[j].x;
double y = recvBuff[j].y;
fprintf(fp, "%f %f 0\n", x, y);
//printf("Rank : 0 receiving from rank %d X : %.2f, Y : %.2f with totalpt : %d\n", i, x, y, particletracer->totalParticles);
fprintf(fp, "%f %f\n", x, y);
// printf("Rank : 0 receiving from rank %d X : %.2f, Y : %.2f with totalpt
// : %d\n", i, x, y, particletracer->totalParticles);
}
}
for (int i = 0; i < particletracer->totalParticles; ++i)
{
for (int i = 0; i < particletracer->totalParticles; ++i) {
double x = particletracer->particlePool[i].x;
double y = particletracer->particlePool[i].y;
fprintf(fp, "%f %f 0\n", x, y);
fprintf(fp, "%f %f\n", x, y);
}
fprintf(fp, "CELLS %d %d\n", overallTotalParticles, 2 * overallTotalParticles);
// fprintf(fp, "CELLS %d %d\n", overallTotalParticles, 2 * overallTotalParticles);
// for (int i = 0; i < overallTotalParticles; ++i)
// {
// fprintf(fp, "1 %d\n", i);
// }
for (int i = 0; i < overallTotalParticles; ++i)
{
fprintf(fp, "1 %d\n", i);
}
// fprintf(fp, "CELL_TYPES %d\n", overallTotalParticles);
fprintf(fp, "CELL_TYPES %d\n", overallTotalParticles);
for (int i = 0; i < overallTotalParticles; ++i)
{
fprintf(fp, "1\n");
}
// for (int i = 0; i < overallTotalParticles; ++i)
// {
// fprintf(fp, "1\n");
// }
fclose(fp);
}
@@ -376,52 +390,53 @@ void initParticleTracer(ParticleTracer* particletracer, Parameter* params, Solve
particletracer->injectTimePeriod = params->injectTimePeriod;
particletracer->writeTimePeriod = params->writeTimePeriod;
particletracer->dt = params->dt;
particletracer->dx = params->xlength / params->imax;
particletracer->dy = params->ylength / params->jmax;
particletracer->dt = params->dt;
particletracer->dx = params->xlength / params->imax;
particletracer->dy = params->ylength / params->jmax;
particletracer->xlength = params->xlength;
particletracer->ylength = params->ylength;
particletracer->x1 = params->x1;
particletracer->y1 = params->y1;
particletracer->x2 = params->x2;
particletracer->y2 = params->y2;
particletracer->xlength = params->xlength;
particletracer->ylength = params->ylength;
particletracer->lastInjectTime = params->startTime;
particletracer->lastUpdateTime = params->startTime;
particletracer->lastWriteTime = params->startTime;
particletracer->x1 = params->x1;
particletracer->y1 = params->y1;
particletracer->x2 = params->x2;
particletracer->y2 = params->y2;
particletracer->pointer = 0;
particletracer->totalParticles = 0;
particletracer->lastInjectTime = params->startTime;
particletracer->lastUpdateTime = params->startTime;
particletracer->lastWriteTime = params->startTime;
particletracer->imax = params->imax;
particletracer->jmax = params->jmax;
particletracer->pointer = 0;
particletracer->totalParticles = 0;
particletracer->imaxLocal = solver->imaxLocal;
particletracer->jmaxLocal = solver->jmaxLocal;
particletracer->imax = params->imax;
particletracer->jmax = params->jmax;
particletracer->estimatedNumParticles = (particletracer->imaxLocal * particletracer->jmaxLocal);
particletracer->imaxLocal = solver->imaxLocal;
particletracer->jmaxLocal = solver->jmaxLocal;
particletracer->particlePool = malloc(sizeof(Particle) * particletracer->estimatedNumParticles);
particletracer->estimatedNumParticles = (particletracer->imaxLocal *
particletracer->jmaxLocal);
for(int i = 0; i < particletracer->estimatedNumParticles; ++i)
{
particletracer->particlePool[i].x = 0.0;
particletracer->particlePool[i].y = 0.0;
particletracer->particlePool = malloc(
sizeof(Particle) * particletracer->estimatedNumParticles);
for (int i = 0; i < particletracer->estimatedNumParticles; ++i) {
particletracer->particlePool[i].x = 0.0;
particletracer->particlePool[i].y = 0.0;
particletracer->particlePool[i].flag = false;
}
particletracer->linSpaceLine = malloc(sizeof(Particle) * particletracer->numberOfParticles);
particletracer->linSpaceLine = malloc(
sizeof(Particle) * particletracer->numberOfParticles);
/* duplicating communication from solver */
MPI_Comm_dup(solver->comm, &particletracer->comm);
particletracer->rank = solver->rank;
particletracer->size = solver->size;
particletracer->offset = (double *)malloc(sizeof(double) * 4 * particletracer->size);
particletracer->rank = solver->rank;
particletracer->size = solver->size;
particletracer->offset = (double*)malloc(sizeof(double) * 4 * particletracer->size);
memcpy(particletracer->dims, solver->dims, sizeof(solver->dims));
@@ -431,66 +446,85 @@ void initParticleTracer(ParticleTracer* particletracer, Parameter* params, Solve
memcpy(particletracer->jNeighbours, solver->jNeighbours, sizeof(solver->jNeighbours));
particletracer->xLocal = particletracer->imaxLocal * particletracer->dx;
particletracer->yLocal = particletracer->jmaxLocal * particletracer->dy;
double xLocal[particletracer->size];
double yLocal[particletracer->size];
particletracer->xLocal = particletracer->imaxLocal * particletracer->dx;
particletracer->yLocal = particletracer->jmaxLocal * particletracer->dy;
double offset[4][particletracer->size];
MPI_Allgather(&particletracer->xLocal, 1, MPI_DOUBLE, xLocal, 1, MPI_DOUBLE, particletracer->comm);
MPI_Allgather(&particletracer->yLocal, 1, MPI_DOUBLE, yLocal, 1, MPI_DOUBLE, particletracer->comm);
particletracer->xOffset = sumX(xLocal, particletracer->size, particletracer->dims[JDIM], particletracer->coords[IDIM], particletracer->coords[JDIM]);
particletracer->yOffset = sumY(yLocal, particletracer->size, particletracer->dims[JDIM], particletracer->coords[JDIM], particletracer->coords[IDIM]);
particletracer->xOffsetEnd = particletracer->xOffset + particletracer->xLocal;
particletracer->yOffsetEnd = particletracer->yOffset + particletracer->yLocal;
particletracer->xOffset = solver->xOffset;
particletracer->yOffset = solver->yOffset;
particletracer->xOffsetEnd = particletracer->xOffset + particletracer->xLocal;
particletracer->yOffsetEnd = particletracer->yOffset + particletracer->yLocal;
printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : %.2f\n", particletracer->rank, particletracer->xOffset, particletracer->yOffset, particletracer->xOffsetEnd, particletracer->yOffsetEnd);
printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : "
"%.2f\n",
particletracer->rank,
particletracer->xOffset,
particletracer->yOffset,
particletracer->xOffsetEnd,
particletracer->yOffsetEnd);
MPI_Allgather(&particletracer->xOffset, 1, MPI_DOUBLE, offset[0], 1, MPI_DOUBLE, particletracer->comm);
MPI_Allgather(&particletracer->yOffset, 1, MPI_DOUBLE, offset[1], 1, MPI_DOUBLE, particletracer->comm);
MPI_Allgather(&particletracer->xOffsetEnd, 1, MPI_DOUBLE, offset[2], 1, MPI_DOUBLE, particletracer->comm);
MPI_Allgather(&particletracer->yOffsetEnd, 1, MPI_DOUBLE, offset[3], 1, MPI_DOUBLE, particletracer->comm);
MPI_Allgather(&particletracer->xOffset,
1,
MPI_DOUBLE,
offset[0],
1,
MPI_DOUBLE,
particletracer->comm);
MPI_Allgather(&particletracer->yOffset,
1,
MPI_DOUBLE,
offset[1],
1,
MPI_DOUBLE,
particletracer->comm);
MPI_Allgather(&particletracer->xOffsetEnd,
1,
MPI_DOUBLE,
offset[2],
1,
MPI_DOUBLE,
particletracer->comm);
MPI_Allgather(&particletracer->yOffsetEnd,
1,
MPI_DOUBLE,
offset[3],
1,
MPI_DOUBLE,
particletracer->comm);
memcpy(particletracer->offset, offset, sizeof(offset));
// if(particletracer->rank == 0)
// {
// {
// for(int i = 0;i < particletracer->size; ++i)
// {
// printf("Rank : %d and its xOffset : %.2f, yOffset : %.2f, xLocal : %.2f, yLocal : %.2f, xOffsetEnd : %.2f, yOffsetEnd : %.2f\n",
// i, particletracer->offset[i + particletracer->size * XOFFSET],
// particletracer->offset[i + particletracer->size * YOFFSET], xLocal[i], yLocal[i],
// particletracer->offset[i + particletracer->size * XOFFSETEND],
// particletracer->offset[i + particletracer->size * YOFFSETEND]);
// printf("Rank : %d and its xOffset : %.2f, yOffset : %.2f, xOffsetEnd :
// %.2f, yOffsetEnd : %.2f\n", i, particletracer->offset[i +
// particletracer->size * XOFFSET], particletracer->offset[i +
// particletracer->size * YOFFSET], particletracer->offset[i +
// particletracer->size * XOFFSETEND], particletracer->offset[i +
// particletracer->size * YOFFSETEND]);
// }
// }
for (int i = 0; i < particletracer->numberOfParticles; ++i)
{
double spacing = (double)i / (double)(particletracer->numberOfParticles - 1);
particletracer->linSpaceLine[i].x = spacing * particletracer->x1 + (1.0 - spacing) * particletracer->x2;
particletracer->linSpaceLine[i].y = spacing * particletracer->y1 + (1.0 - spacing) * particletracer->y2;
for (int i = 0; i < particletracer->numberOfParticles; ++i) {
double spacing = (double)i / (double)(particletracer->numberOfParticles - 1);
particletracer->linSpaceLine[i].x = spacing * particletracer->x1 +
(1.0 - spacing) * particletracer->x2;
particletracer->linSpaceLine[i].y = spacing * particletracer->y1 +
(1.0 - spacing) * particletracer->y2;
particletracer->linSpaceLine[i].flag = true;
//if(particletracer->rank == 1) printf("\nRank : %d, with linspace X : %.2f and linspace Y : %.2f\n", particletracer->rank, particletracer->linSpaceLine[i].x , particletracer->linSpaceLine[i].y);
// if(particletracer->rank == 1) printf("\nRank : %d, with linspace X : %.2f and
// linspace Y : %.2f\n", particletracer->rank, particletracer->linSpaceLine[i].x ,
// particletracer->linSpaceLine[i].y);
}
// Create the mpi_particle datatype
MPI_Datatype mpi_particle;
int lengths[3] = { 1, 1, 1 };
MPI_Aint displacements[3];
Particle dummy_particle;
MPI_Aint base_address;
@@ -501,45 +535,69 @@ void initParticleTracer(ParticleTracer* particletracer, Parameter* params, Solve
displacements[0] = MPI_Aint_diff(displacements[0], base_address);
displacements[1] = MPI_Aint_diff(displacements[1], base_address);
displacements[2] = MPI_Aint_diff(displacements[2], base_address);
MPI_Datatype types[3] = { MPI_DOUBLE, MPI_DOUBLE, MPI_C_BOOL };
MPI_Type_create_struct(3, lengths, displacements, types, &particletracer->mpi_particle);
MPI_Type_commit(&particletracer->mpi_particle);
MPI_Datatype types[3] = { MPI_DOUBLE, MPI_DOUBLE, MPI_C_BOOL };
MPI_Type_create_struct(3,
lengths,
displacements,
types,
&particletracer->mpi_particle);
MPI_Type_commit(&particletracer->mpi_particle);
}
void printParticleTracerParameters(ParticleTracer* particletracer)
{
printf("Particle Tracing data:\n");
printf("Rank : %d\n", particletracer->rank);
printf("\tNumber of particles : %d being injected for every period of %.2f\n", particletracer->numberOfParticles, particletracer->injectTimePeriod);
printf("\tNumber of particles : %d being injected for every period of %.2f\n",
particletracer->numberOfParticles,
particletracer->injectTimePeriod);
printf("\tstartTime : %.2f\n", particletracer->startTime);
printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : %.2f, x2 : %.2f, y2 : %.2f\n", particletracer->x1, particletracer->y1, particletracer->x2, particletracer->y2);
printf("\tPointer : %d, TotalParticles : %d\n", particletracer->pointer, particletracer->totalParticles);
printf("\tdt : %.2f, dx : %.2f, dy : %.2f\n", particletracer->dt, particletracer->dx, particletracer->dy);
printf("\tcoord[0] : %d, coord[1] : %d\n", particletracer->coords[IDIM], particletracer->coords[JDIM]);
printf("\txOffset : %.2f, yOffset : %.2f\n", particletracer->xOffset, particletracer->yOffset);
printf("\txOffsetEnd : %.2f, yOffsetEnd : %.2f\n", particletracer->xOffsetEnd, particletracer->yOffsetEnd);
printf("\txLocal : %.2f, yLocal : %.2f\n", particletracer->xLocal, particletracer->yLocal);
}
printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : "
"%.2f, x2 : %.2f, y2 : %.2f\n",
particletracer->x1,
particletracer->y1,
particletracer->x2,
particletracer->y2);
printf("\tPointer : %d, TotalParticles : %d\n",
particletracer->pointer,
particletracer->totalParticles);
printf("\tdt : %.2f, dx : %.2f, dy : %.2f\n",
particletracer->dt,
particletracer->dx,
particletracer->dy);
printf("\tcoord[0] : %d, coord[1] : %d\n",
particletracer->coords[IDIM],
particletracer->coords[JDIM]);
printf("\txOffset : %.2f, yOffset : %.2f\n",
particletracer->xOffset,
particletracer->yOffset);
printf("\txOffsetEnd : %.2f, yOffsetEnd : %.2f\n",
particletracer->xOffsetEnd,
particletracer->yOffsetEnd);
printf("\txLocal : %.2f, yLocal : %.2f\n",
particletracer->xLocal,
particletracer->yLocal);
}
void trace(ParticleTracer* particletracer, double* restrict u, double* restrict v, double time)
void trace(ParticleTracer* particletracer,
double* restrict u,
double* restrict v,
int* restrict s,
double time)
{
if (time >= particletracer->startTime)
{
if( (time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod)
{
if (time >= particletracer->startTime) {
if ((time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod) {
injectParticles(particletracer);
particletracer->lastInjectTime = time;
}
if( (time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod)
{
if ((time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod) {
writeParticles(particletracer);
particletracer->lastWriteTime = time;
}
advanceParticles(particletracer, u, v, time);
advanceParticles(particletracer, u, v, s, time);
compress(particletracer);
particletracer->lastUpdateTime = time;
particletracer->lastUpdateTime = time;
}
}
@@ -548,18 +606,15 @@ void compress(ParticleTracer* particletracer)
Particle* memPool = particletracer->particlePool;
Particle tempPool[particletracer->totalParticles];
for(int i = 0; i < particletracer->totalParticles; ++i)
{
tempPool[i].x = 0.0;
tempPool[i].y = 0.0;
for (int i = 0; i < particletracer->totalParticles; ++i) {
tempPool[i].x = 0.0;
tempPool[i].y = 0.0;
tempPool[i].flag = false;
}
int totalParticles = 0;
for(int i=0; i < particletracer->totalParticles; ++i)
{
if(memPool[i].flag == true)
{
for (int i = 0; i < particletracer->totalParticles; ++i) {
if (memPool[i].flag == true) {
tempPool[totalParticles].x = memPool[i].x;
tempPool[totalParticles].y = memPool[i].y;
tempPool[totalParticles].flag = memPool[i].flag;

34
BasicSolver/2D-mpi-v3/src/particletracing.h
View File

@@ -8,23 +8,25 @@
#define __PARTICLETRACING_H_
#include "allocate.h"
#include "parameter.h"
#include "solver.h"
#include "particletracing.h"
#include <stdbool.h>
#include <mpi.h>
#include <stdbool.h>
#define NDIMS 2
typedef enum COORD { X = 0, Y, NCOORD } COORD;
typedef struct
{
typedef struct {
double x, y;
bool flag;
} Particle;
typedef struct {
int numberOfParticles, totalParticles;
double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime, lastWriteTime;
double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime,
lastWriteTime;
int estimatedNumParticles;
@@ -33,14 +35,14 @@ typedef struct {
Particle* particlePool;
int pointer;
double imax, jmax, xlength, ylength, imaxLocal, jmaxLocal;
double x1, y1, x2, y2;
MPI_Comm comm;
MPI_Datatype mpi_particle;
int rank, size;
int iNeighbours[NDIMS], jNeighbours[NDIMS];
int coords[NDIMS], dims[NDIMS];
@@ -51,13 +53,13 @@ typedef struct {
} ParticleTracer;
void initParticleTracer(ParticleTracer*, Parameter*, Solver* );
void injectParticles(ParticleTracer*);
void advanceParticles(ParticleTracer*, double* , double*, double);
void freeParticles(ParticleTracer*);
void writeParticles(ParticleTracer*);
void printParticleTracerParameters(ParticleTracer*);
void printParticles(ParticleTracer*);
void trace(ParticleTracer*, double* , double* , double );
void compress(ParticleTracer* );
extern void initParticleTracer(ParticleTracer*, Parameter*, Solver*);
extern void injectParticles(ParticleTracer*);
extern void advanceParticles(ParticleTracer*, double*, double*, int*, double);
extern void freeParticles(ParticleTracer*);
extern void writeParticles(ParticleTracer*);
extern void printParticleTracerParameters(ParticleTracer*);
extern void printParticles(ParticleTracer*);
extern void trace(ParticleTracer*, double*, double*, int*, double);
extern void compress(ParticleTracer*);
#endif

413
BasicSolver/2D-mpi-v3/src/solver.c
View File

@@ -14,23 +14,42 @@
#include "allocate.h"
#include "parameter.h"
#include "solver.h"
#include "util.h"
#define P(i, j) p[(j) * (imaxLocal + 2) + (i)]
#define F(i, j) f[(j) * (imaxLocal + 2) + (i)]
#define G(i, j) g[(j) * (imaxLocal + 2) + (i)]
#define U(i, j) u[(j) * (imaxLocal + 2) + (i)]
#define V(i, j) v[(j) * (imaxLocal + 2) + (i)]
#define P(i, j) p[(j) * (imaxLocal + 2) + (i)]
#define F(i, j) f[(j) * (imaxLocal + 2) + (i)]
#define G(i, j) g[(j) * (imaxLocal + 2) + (i)]
#define U(i, j) u[(j) * (imaxLocal + 2) + (i)]
#define V(i, j) v[(j) * (imaxLocal + 2) + (i)]
#define S(i, j) s[(j) * (imaxLocal + 2) + (i)]
#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)]
#define IDIM 0
#define JDIM 1
static double sumOffset(double* sizes, int init, int offset, int coord)
{
double sum = 0;
for (int i = init - offset; coord > 0; i -= offset, --coord) {
sum += sizes[i];
}
return sum;
}
static int sizeOfRank(int rank, int size, int N)
{
return N / size + ((N % size > rank) ? 1 : 0);
}
static double distance(double i, double j, double iCenter, double jCenter)
{
return sqrt(pow(iCenter - i, 2) + pow(jCenter - j, 2) * 1.0);
}
void print(Solver* solver, double* grid)
{
int imaxLocal = solver->imaxLocal;
@@ -43,7 +62,29 @@ void print(Solver* solver, double* grid)
for (int j = 0; j < solver->jmaxLocal + 2; j++) {
printf("%02d: ", j);
for (int i = 0; i < solver->imaxLocal + 2; i++) {
printf("%12.8f ", grid[j * (imaxLocal + 2) + i]);
printf("%2.2f ", grid[j * (imaxLocal + 2) + i]);
}
printf("\n");
}
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
}
}
void printInt(Solver* solver, int* grid)
{
int imaxLocal = solver->imaxLocal;
for (int i = 0; i < solver->size; i++) {
if (i == solver->rank) {
printf(
"### RANK %d #######################################################\n",
solver->rank);
for (int j = 0; j < solver->jmaxLocal + 2; j++) {
printf("%02d: ", j);
for (int i = 0; i < solver->imaxLocal + 2; i++) {
printf("%2d ", grid[j * (imaxLocal + 2) + i]);
}
printf("\n");
}
@@ -68,6 +109,21 @@ static void exchange(Solver* solver, double* grid)
solver->comm);
}
static void exchangeInt(Solver* solver, int* grid)
{
int counts[4] = { 1, 1, 1, 1 };
MPI_Neighbor_alltoallw(grid,
counts,
solver->sdisplsInt,
solver->bufferIntTypes,
grid,
counts,
solver->rdisplsInt,
solver->bufferIntTypes,
solver->comm);
}
static void shift(Solver* solver)
{
MPI_Request requests[4] = { MPI_REQUEST_NULL,
@@ -315,7 +371,6 @@ void initSolver(Solver* solver, Parameter* params)
solver->gamma = params->gamma;
solver->rho = params->rho;
/* setup communication */
MPI_Comm_rank(MPI_COMM_WORLD, &(solver->rank));
MPI_Comm_size(MPI_COMM_WORLD, &(solver->size));
@@ -350,26 +405,56 @@ void initSolver(Solver* solver, Parameter* params)
&iBufferType);
MPI_Type_commit(&iBufferType);
MPI_Datatype jBufferIntType;
MPI_Type_contiguous(solver->imaxLocal, MPI_INT, &jBufferIntType);
MPI_Type_commit(&jBufferIntType);
MPI_Datatype iBufferIntType;
MPI_Type_vector(solver->jmaxLocal,
1,
solver->imaxLocal + 2,
MPI_INT,
&iBufferIntType);
MPI_Type_commit(&iBufferIntType);
// in the order of the dimensions i->0, j->1
// first negative direction, then positive direction
size_t dblsize = sizeof(double);
int imaxLocal = solver->imaxLocal;
int jmaxLocal = solver->jmaxLocal;
size_t dblsize = sizeof(double);
size_t intsize = sizeof(int);
int imaxLocal = solver->imaxLocal;
int jmaxLocal = solver->jmaxLocal;
solver->bufferTypes[0] = iBufferType; // left
solver->bufferTypes[1] = iBufferType; // right
solver->bufferTypes[2] = jBufferType; // bottom
solver->bufferTypes[3] = jBufferType; // top
solver->bufferIntTypes[0] = iBufferIntType; // left
solver->bufferIntTypes[1] = iBufferIntType; // right
solver->bufferIntTypes[2] = jBufferIntType; // bottom
solver->bufferIntTypes[3] = jBufferIntType; // top
solver->sdispls[0] = ((imaxLocal + 2) + 1) * dblsize; // send left
solver->sdispls[1] = ((imaxLocal + 2) + imaxLocal) * dblsize; // send right
solver->sdispls[2] = ((imaxLocal + 2) + 1) * dblsize; // send bottom
solver->sdispls[3] = ((jmaxLocal) * (imaxLocal + 2) + 1) * dblsize; // send top
solver->sdisplsInt[0] = ((imaxLocal + 2) + 1) * intsize; // send left
solver->sdisplsInt[1] = ((imaxLocal + 2) + imaxLocal) * intsize; // send right
solver->sdisplsInt[2] = ((imaxLocal + 2) + 1) * intsize; // send bottom
solver->sdisplsInt[3] = ((jmaxLocal) * (imaxLocal + 2) + 1) * intsize; // send top
solver->rdispls[0] = (imaxLocal + 2) * dblsize; // recv left
solver->rdispls[1] = ((imaxLocal + 2) + (imaxLocal + 1)) * dblsize; // recv right
solver->rdispls[2] = 1 * dblsize; // recv bottom
solver->rdispls[3] = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * dblsize; // recv top
solver->rdisplsInt[0] = (imaxLocal + 2) * intsize; // recv left
solver->rdisplsInt[1] = ((imaxLocal + 2) + (imaxLocal + 1)) * intsize; // recv right
solver->rdisplsInt[2] = 1 * intsize; // recv bottom
solver->rdisplsInt[3] = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * intsize; // recv top
/* allocate arrays */
size_t bytesize = (imaxLocal + 2) * (jmaxLocal + 2) * sizeof(double);
solver->u = allocate(64, bytesize);
@@ -378,6 +463,7 @@ void initSolver(Solver* solver, Parameter* params)
solver->rhs = allocate(64, bytesize);
solver->f = allocate(64, bytesize);
solver->g = allocate(64, bytesize);
solver->s = allocate(64, bytesize);
for (int i = 0; i < (imaxLocal + 2) * (jmaxLocal + 2); i++) {
solver->u[i] = params->u_init;
@@ -386,12 +472,119 @@ void initSolver(Solver* solver, Parameter* params)
solver->rhs[i] = 0.0;
solver->f[i] = 0.0;
solver->g[i] = 0.0;
solver->s[i] = NONE;
}
double dx = solver->dx;
double dy = solver->dy;
double inv_sqr_sum = 1.0 / (dx * dx) + 1.0 / (dy * dy);
solver->dtBound = 0.5 * solver->re * 1.0 / inv_sqr_sum;
solver->xLocal = solver->imaxLocal * solver->dx;
solver->yLocal = solver->jmaxLocal * solver->dy;
double xLocal[solver->size];
double yLocal[solver->size];
MPI_Allgather(&solver->xLocal, 1, MPI_DOUBLE, xLocal, 1, MPI_DOUBLE, solver->comm);
MPI_Allgather(&solver->yLocal, 1, MPI_DOUBLE, yLocal, 1, MPI_DOUBLE, solver->comm);
solver->xOffset = sumOffset(xLocal, solver->rank, solver->dims[1], solver->coords[0]);
solver->yOffset = sumOffset(yLocal, solver->rank, 1, solver->coords[1]);
solver->xOffsetEnd = solver->xOffset + solver->xLocal;
solver->yOffsetEnd = solver->yOffset + solver->yLocal;
printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : "
"%.2f\n",
solver->rank,
solver->xOffset,
solver->yOffset,
solver->xOffsetEnd,
solver->yOffsetEnd);
int* s = solver->s;
int iOffset = 0, jOffset = 0;
double xCenter = 0, yCenter = 0, radius = 0;
double x1 = 0, x2 = 0, y1 = 0, y2 = 0;
switch (params->shape) {
case NOSHAPE:
break;
case RECT:
x1 = params->xCenter - params->xRectLength / 2;
x2 = params->xCenter + params->xRectLength / 2;
y1 = params->yCenter - params->yRectLength / 2;
y2 = params->yCenter + params->yRectLength / 2;
iOffset = solver->xOffset / dx;
jOffset = solver->yOffset / dy;
for (int j = 1; j < jmaxLocal + 1; ++j) {
for (int i = 1; i < imaxLocal + 1; ++i) {
if ((x1 <= ((i + iOffset) * dx)) && (((i + iOffset) * dx) <= x2) &&
(y1 <= ((j + jOffset) * dy)) && (((j + jOffset) * dy) <= y2)) {
S(i, j) = LOCAL;
}
}
}
break;
case CIRCLE:
xCenter = params->xCenter;
yCenter = params->yCenter;
radius = params->circleRadius;
iOffset = solver->xOffset / dx;
jOffset = solver->yOffset / dy;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
if (distance(((i + iOffset) * dx),
((j + jOffset) * dy),
xCenter,
yCenter) <= radius) {
S(i, j) = LOCAL;
}
}
}
break;
default:
break;
}
exchangeInt(solver, s);
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
if (S(i, j - 1) == NONE && S(i, j + 1) == LOCAL && S(i, j) == LOCAL)
S(i, j) = BOTTOM; // BOTTOM
if (S(i - 1, j) == NONE && S(i + 1, j) == LOCAL && S(i, j) == LOCAL)
S(i, j) = LEFT; // LEFT
if (S(i + 1, j) == NONE && S(i - 1, j) == LOCAL && S(i, j) == LOCAL)
S(i, j) = RIGHT; // RIGHT
if (S(i, j + 1) == NONE && S(i, j - 1) == LOCAL && S(i, j) == LOCAL)
S(i, j) = TOP; // TOP
if (S(i - 1, j - 1) == NONE && S(i, j - 1) == NONE && S(i - 1, j) == NONE &&
S(i + 1, j + 1) == LOCAL &&
(S(i, j) == LOCAL || S(i, j) == LEFT || S(i, j) == BOTTOM))
S(i, j) = BOTTOMLEFT; // BOTTOMLEFT
if (S(i + 1, j - 1) == NONE && S(i, j - 1) == NONE && S(i + 1, j) == NONE &&
S(i - 1, j + 1) == LOCAL &&
(S(i, j) == LOCAL || S(i, j) == RIGHT || S(i, j) == BOTTOM))
S(i, j) = BOTTOMRIGHT; // BOTTOMRIGHT
if (S(i - 1, j + 1) == NONE && S(i - 1, j) == NONE && S(i, j + 1) == NONE &&
S(i + 1, j - 1) == LOCAL &&
(S(i, j) == LOCAL || S(i, j) == LEFT || S(i, j) == TOP))
S(i, j) = TOPLEFT; // TOPLEFT
if (S(i + 1, j + 1) == NONE && S(i + 1, j) == NONE && S(i, j + 1) == NONE &&
S(i - 1, j - 1) == LOCAL &&
(S(i, j) == LOCAL || S(i, j) == RIGHT || S(i, j) == TOP))
S(i, j) = TOPRIGHT; // TOPRIGHT
}
}
#ifdef VERBOSE
printConfig(solver);
#endif
@@ -613,7 +806,6 @@ int solveRBA(Solver* solver)
int pass, jsw, isw;
while ((res >= epssq) && (it < itermax)) {
res = 0.0;
jsw = 1;
@@ -634,11 +826,11 @@ int solveRBA(Solver* solver)
}
isw = 3 - isw;
}
jsw = 3 - jsw;
jsw = 3 - jsw;
omega = (it == 0 && pass == 0 ? 1.0 / (1.0 - 0.5 * rho * rho)
: 1.0 / (1.0 - 0.25 * rho * rho * omega));
}
if (solver->coords[JDIM] == 0) { // set bottom bc
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, 0) = P(i, 1);
@@ -685,6 +877,66 @@ int solveRBA(Solver* solver)
}
}
void setObjectBoundaryCondition(Solver* solver)
{
int imaxLocal = solver->imaxLocal;
int jmaxLocal = solver->jmaxLocal;
double* u = solver->u;
double* v = solver->v;
int* s = solver->s;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
switch (S(i, j)) {
case TOP:
U(i, j) = -U(i, j + 1);
U(i - 1, j) = -U(i - 1, j + 1);
V(i, j) = 0.0;
break;
case BOTTOM:
U(i, j) = -U(i, j - 1);
U(i - 1, j) = -U(i - 1, j - 1);
V(i, j) = 0.0;
break;
case LEFT:
U(i - 1, j) = 0.0;
V(i, j) = -V(i - 1, j);
V(i, j - 1) = -V(i - 1, j - 1);
break;
case RIGHT:
U(i, j) = 0.0;
V(i, j) = -V(i + 1, j);
V(i, j - 1) = -V(i + 1, j - 1);
break;
case TOPLEFT:
U(i, j) = -U(i, j + 1);
U(i - 1, j) = 0.0;
V(i, j) = 0.0;
V(i, j - 1) = -V(i - 1, j - 1);
break;
case TOPRIGHT:
U(i, j) = 0.0;
U(i - 1, j) = -U(i - 1, j + 1);
V(i, j) = 0.0;
V(i, j - 1) = -V(i + 1, j - 1);
break;
case BOTTOMLEFT:
U(i, j) = -U(i, j - 1);
U(i - 1, j) = 0.0;
V(i, j) = -V(i - 1, j);
V(i, j - 1) = 0.0;
break;
case BOTTOMRIGHT:
U(i, j) = 0.0;
U(i - 1, j) = -U(i - 1, j - 1);
V(i, j) = -V(i, j + 1);
V(i, j - 1) = 0.0;
break;
}
}
}
}
static double maxElement(Solver* solver, double* m)
{
int size = (solver->imaxLocal + 2) * (solver->jmaxLocal + 2);
@@ -833,6 +1085,7 @@ void setSpecialBoundaryCondition(Solver* solver)
int imaxLocal = solver->imaxLocal;
int jmaxLocal = solver->jmaxLocal;
double* u = solver->u;
int* s = solver->s;
if (strcmp(solver->problem, "dcavity") == 0) {
if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc
@@ -844,21 +1097,31 @@ void setSpecialBoundaryCondition(Solver* solver)
if (solver->coords[IDIM] == 0) { // set left bc
double ylength = solver->ylength;
double dy = solver->dy;
int rest = solver->jmax % solver->size;
int yc = solver->rank * (solver->jmax / solver->size) +
MIN(rest, solver->rank);
double ys = dy * (yc + 0.5);
int jOffset = solver->yOffset / dy;
int jOffsetEnd = solver->yOffsetEnd / dy;
double y;
/* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */
for (int j = 1; j < jmaxLocal + 1; j++) {
y = ys + dy * (j - 0.5);
for (int j = 1, jVal = jOffset + 1; j < jmaxLocal + 1; ++jVal, j++) {
y = dy * (jVal - 0.5);
U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength);
}
}
} else if (strcmp(solver->problem, "backstep") == 0) {
for (int j = 1; j < jmaxLocal + 1; j++) {
if (S(0, j) == NONE) U(0, j) = 1.0;
}
} else if (strcmp(solver->problem, "karman") == 0) {
for (int j = 1; j < jmaxLocal + 1; j++) {
U(0, j) = 1.0;
}
}
/* print(solver, solver->u); */
// print(solver, solver->u);
// MPI_Barrier(solver->comm);
// exit(0);
}
void computeFG(Solver* solver)
@@ -867,6 +1130,7 @@ void computeFG(Solver* solver)
double* v = solver->v;
double* f = solver->f;
double* g = solver->g;
int* s = solver->s;
int imaxLocal = solver->imaxLocal;
int jmaxLocal = solver->jmaxLocal;
double gx = solver->gx;
@@ -884,43 +1148,84 @@ void computeFG(Solver* solver)
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
du2dx = inverseDx * 0.25 *
((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) -
(U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) +
gamma * inverseDx * 0.25 *
(fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) +
fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j)));
if (S(i, j) == NONE) {
du2dx = inverseDx * 0.25 *
((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) -
(U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) +
gamma * inverseDx * 0.25 *
(fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) +
fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j)));
duvdy = inverseDy * 0.25 *
((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) -
(V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) +
gamma * inverseDy * 0.25 *
(fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) +
fabs(V(i, j - 1) + V(i + 1, j - 1)) *
(U(i, j) - U(i, j - 1)));
duvdy = inverseDy * 0.25 *
((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) -
(V(i, j - 1) + V(i + 1, j - 1)) *
(U(i, j) + U(i, j - 1))) +
gamma * inverseDy * 0.25 *
(fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) +
fabs(V(i, j - 1) + V(i + 1, j - 1)) *
(U(i, j) - U(i, j - 1)));
du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j));
du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1));
F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx);
du2dx2 = inverseDx * inverseDx *
(U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j));
du2dy2 = inverseDy * inverseDy *
(U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1));
F(i, j) = U(i, j) +
dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx);
duvdx = inverseDx * 0.25 *
((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) -
(U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) +
gamma * inverseDx * 0.25 *
(fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) +
fabs(U(i - 1, j) + U(i - 1, j + 1)) *
(V(i, j) - V(i - 1, j)));
duvdx = inverseDx * 0.25 *
((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) -
(U(i - 1, j) + U(i - 1, j + 1)) *
(V(i, j) + V(i - 1, j))) +
gamma * inverseDx * 0.25 *
(fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) +
fabs(U(i - 1, j) + U(i - 1, j + 1)) *
(V(i, j) - V(i - 1, j)));
dv2dy = inverseDy * 0.25 *
((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) -
(V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) +
gamma * inverseDy * 0.25 *
(fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) +
fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1)));
dv2dy = inverseDy * 0.25 *
((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) -
(V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) +
gamma * inverseDy * 0.25 *
(fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) +
fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1)));
dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j));
dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1));
G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy);
dv2dx2 = inverseDx * inverseDx *
(V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j));
dv2dy2 = inverseDy * inverseDy *
(V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1));
G(i, j) = V(i, j) +
dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy);
} else {
switch (S(i, j)) {
case TOP:
G(i, j) = V(i, j);
break;
case BOTTOM:
G(i, j - 1) = V(i, j - 1);
break;
case LEFT:
F(i - 1, j) = U(i - 1, j);
break;
case RIGHT:
F(i, j) = U(i, j);
break;
case TOPLEFT:
F(i - 1, j) = U(i - 1, j);
G(i, j) = V(i, j);
break;
case TOPRIGHT:
F(i, j) = U(i, j);
G(i, j) = V(i, j);
break;
case BOTTOMLEFT:
F(i - 1, j) = U(i - 1, j);
G(i, j - 1) = V(i, j - 1);
break;
case BOTTOMRIGHT:
F(i, j) = U(i, j);
G(i, j - 1) = V(i, j - 1);
break;
}
}
}
}

54
BasicSolver/2D-mpi-v3/src/solver.h
View File

@@ -7,11 +7,26 @@
#ifndef __SOLVER_H_
#define __SOLVER_H_
#include "parameter.h"
#include <mpi.h>
#define NDIMS 2
enum OBJECTBOUNDARY {
NONE = 0,
TOP,
BOTTOM,
LEFT,
RIGHT,
TOPLEFT,
BOTTOMLEFT,
TOPRIGHT,
BOTTOMRIGHT,
LOCAL
};
enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
/// @brief
enum SHAPE { NOSHAPE = 0, RECT, CIRCLE };
typedef struct {
/* geometry and grid information */
@@ -22,6 +37,8 @@ typedef struct {
double *p, *rhs;
double *f, *g;
double *u, *v;
int* s;
/* parameters */
double eps, omega, rho;
double re, tau, gamma;
@@ -38,23 +55,32 @@ typedef struct {
MPI_Comm comm;
MPI_Datatype bufferTypes[NDIMS * 2];
MPI_Aint sdispls[NDIMS * 2], rdispls[NDIMS * 2];
MPI_Datatype bufferIntTypes[NDIMS * 2];
MPI_Aint sdisplsInt[NDIMS * 2], rdisplsInt[NDIMS * 2];
int iNeighbours[NDIMS], jNeighbours[NDIMS];
int coords[NDIMS], dims[NDIMS];
int imaxLocal, jmaxLocal;
double xLocal, yLocal, xOffset, yOffset, xOffsetEnd, yOffsetEnd;
} Solver;
void initSolver(Solver*, Parameter*);
void computeRHS(Solver*);
int solve(Solver*);
int solveRB(Solver*);
int solveRBA(Solver*);
void computeTimestep(Solver*);
void setBoundaryConditions(Solver*);
void setSpecialBoundaryCondition(Solver*);
void computeFG(Solver*);
void adaptUV(Solver*);
void collectResult(Solver*);
void writeResult(Solver*, double*, double*, double*);
void debugExchange(Solver*);
void print(Solver*, double*);
extern void initSolver(Solver*, Parameter*);
extern void computeRHS(Solver*);
extern int solve(Solver*);
extern int solveRB(Solver*);
extern int solveRBA(Solver*);
extern void computeTimestep(Solver*);
extern void setBoundaryConditions(Solver*);
extern void setSpecialBoundaryCondition(Solver*);
extern void setObjectBoundaryCondition(Solver*);
extern void computeFG(Solver*);
extern void adaptUV(Solver*);
extern void collectResult(Solver*);
extern void writeResult(Solver*, double*, double*, double*);
extern void debugExchange(Solver*);
extern void print(Solver*, double*);
extern void printInt(Solver*, int*);
#endif

126
BasicSolver/2D-mpi-v3/src/vtkWriter.c
View File

@@ -1,126 +0,0 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "vtkWriter.h"
static float floatSwap(float f)
{
union {
float f;
char b[4];
} dat1, dat2;
dat1.f = f;
dat2.b[0] = dat1.b[3];
dat2.b[1] = dat1.b[2];
dat2.b[2] = dat1.b[1];
dat2.b[3] = dat1.b[0];
return dat2.f;
}
static void writeHeader(VtkOptions* o, int ts)
{
fprintf(o->fh, "# vtk DataFile Version 3.0\n");
fprintf(o->fh, "PAMPI cfd solver particle tracing file\n");
if (o->fmt == ASCII) {
fprintf(o->fh, "ASCII\n");
} else if (o->fmt == BINARY) {
fprintf(o->fh, "BINARY\n");
}
fprintf(o->fh, "DATASET UNSTRUCTURED_GRID\n");
fprintf(o->fh, "FIELD FieldData 2\n");
fprintf(o->fh, "TIME 1 1 double\n");
fprintf(o->fh, "%d\n", ts);
fprintf(o->fh, "CYCLE 1 1 int\n");
fprintf(o->fh, "1\n");
}
void vtkOpen(VtkOptions* o, char* problem, int ts)
{
o->fh = fopen(problem, "w");
if (o->fh == NULL) {
printf("vtkWriter not initialize! Call vtkOpen first!\n");
exit(EXIT_FAILURE);
}
writeHeader(o, ts);
printf("Writing VTK output for %s\n", problem);
}
void vtkParticle(VtkOptions* o, char* name)
{
Particle* particlePool = o->particletracer->particlePool;
if (o->fh == NULL) {
printf("vtkWriter not initialize! Call vtkOpen first!\n");
exit(EXIT_FAILURE);
}
fprintf(o->fh, "POINTS %d float\n", o->overallTotalParticles);
for (int i = 0; i < o->particletracer->size; ++i)
{
double x = particlePool[i].x;
double y = particlePool[i].y;
fprintf(o->fh, "%.2f %.2f 0.0\n", x, y);
}
fprintf(o->fh, "CELLS %d %d\n", o->particletracer->totalParticles, 2 * o->particletracer->totalParticles);
for (int i = 0; i < o->particletracer->totalParticles; ++i)
{
fprintf(o->fh, "1 %d\n", i);
}
fprintf(o->fh, "CELL_TYPES %d\n", o->particletracer->totalParticles);
for (int i = 0; i < o->particletracer->totalParticles; ++i)
{
fprintf(o->fh, "1\n");
}
/*
for (int k = 0; k < kmax; k++) {
for (int j = 0; j < jmax; j++) {
for (int i = 0; i < imax; i++) {
if (o->fmt == ASCII) {
fprintf(o->fh,
"%f %f %f\n",
G(vec.u, i, j, k),
G(vec.v, i, j, k),
G(vec.w, i, j, k));
} else if (o->fmt == BINARY) {
fwrite((float[3]) { floatSwap(G(vec.u, i, j, k)),
floatSwap(G(vec.v, i, j, k)),
floatSwap(G(vec.w, i, j, k)) },
sizeof(float),
3,
o->fh);
}
}
}
}
if (o->fmt == BINARY) fprintf(o->fh, "\n");
*/
}
void vtkClose(VtkOptions* o)
{
fclose(o->fh);
o->fh = NULL;
}

26
BasicSolver/2D-mpi-v3/src/vtkWriter.h
View File

@@ -1,26 +0,0 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#ifndef __VTKWRITER_H_
#define __VTKWRITER_H_
#include <stdio.h>
#include "particletracing.h"
#include "solver.h"
typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat;
typedef struct VtkOptions {
VtkFormat fmt;
FILE* fh;
int overallTotalParticles;
ParticleTracer* particletracer;
} VtkOptions;
extern void vtkOpen(VtkOptions* opts, char* filename, int ts);
extern void vtkParticle(VtkOptions* opts, char* name);
extern void vtkClose(VtkOptions* opts);
#endif // __VTKWRITER_H_