moebiusband/NuSiF-Solver
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revered dimension switch. Still not working correctly

Browse Source
This commit is contained in:
Jan Eitzinger 2024-02-12 10:58:37 +01:00
parent 2d759b106a
commit 59addafa5d
7 changed files with 64 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
BasicSolver/2D-mpi/config.mk
View File

@ -2,7 +2,7 @@
TAG ?= CLANG TAG ?= CLANG
ENABLE_MPI ?= true ENABLE_MPI ?= true
ENABLE_OPENMP ?= false ENABLE_OPENMP ?= false
COMM_TYPE ?= v2 COMM_TYPE ?= v3
#Feature options #Feature options
OPTIONS += -DARRAY_ALIGNMENT=64 OPTIONS += -DARRAY_ALIGNMENT=64

74
BasicSolver/2D-mpi/src/comm-v2.c
View File

@ -22,7 +22,7 @@ static int sum(int* sizes, int position)
return sum; return sum;
} }
static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax) static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax)
{ {
MPI_Request* requests; MPI_Request* requests;
int numRequests = 1; int numRequests = 1;
@ -35,10 +35,10 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
/* all ranks send their bulk array, but including external boundary layer */ /* all ranks send their bulk array, including the external boundary layer */
MPI_Datatype bulkType; MPI_Datatype bulkType;
int oldSizes[NDIMS] = { c->imaxLocal + 2, c->jmaxLocal + 2 }; int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
int newSizes[NDIMS] = { c->imaxLocal, c->jmaxLocal }; int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
int starts[NDIMS] = { 1, 1 }; int starts[NDIMS] = { 1, 1 };
if (commIsBoundary(c, LEFT)) { if (commIsBoundary(c, LEFT)) {
@ -75,12 +75,25 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax
if (c->rank == 0) { if (c->rank == 0) {
for (int i = 0; i < c->size; i++) { for (int i = 0; i < c->size; i++) {
MPI_Datatype domainType; MPI_Datatype domainType;
int oldSizes[NDIMS] = { imax + 2, jmax + 2 }; int oldSizes[NDIMS] = { jmax + 2, imax + 2 };
int newSizes[NDIMS] = { newSizesI[i], newSizesJ[i] }; int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
int coords[NDIMS]; int coords[NDIMS];
MPI_Cart_coords(c->comm, i, NDIMS, coords); MPI_Cart_coords(c->comm, i, NDIMS, coords);
int starts[NDIMS] = { sum(newSizesI, coords[IDIM]), int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
sum(newSizesJ, coords[JDIM]) }; sum(newSizesI, coords[IDIM]) };
printf(
"Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
"Starts(i,j): %d %d\n",
i,
coords[IDIM],
coords[JDIM],
oldSizes[IDIM],
oldSizes[JDIM],
newSizes[IDIM],
newSizes[JDIM],
starts[IDIM],
starts[JDIM]);
MPI_Type_create_subarray(NDIMS, MPI_Type_create_subarray(NDIMS,
oldSizes, oldSizes,
newSizes, newSizes,
@ -91,6 +104,7 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax
MPI_Type_commit(&domainType); MPI_Type_commit(&domainType);
MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]);
MPI_Type_free(&domainType);
} }
} }
@ -182,8 +196,8 @@ void commShift(Comm* c, double* f, double* g)
MPI_REQUEST_NULL }; MPI_REQUEST_NULL };
/* shift G */ /* shift G */
double* buf = g + 1;
/* receive ghost cells from bottom neighbor */ /* receive ghost cells from bottom neighbor */
double* buf = g + 1;
MPI_Irecv(buf, MPI_Irecv(buf,
1, 1,
c->bufferTypes[BOTTOM], c->bufferTypes[BOTTOM],
@ -192,13 +206,13 @@ void commShift(Comm* c, double* f, double* g)
c->comm, c->comm,
&requests[0]); &requests[0]);
buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
/* send ghost cells to top neighbor */ /* send ghost cells to top neighbor */
buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
MPI_Isend(buf, 1, c->bufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]); MPI_Isend(buf, 1, c->bufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]);
/* shift F */ /* shift F */
buf = f + (c->imaxLocal + 2);
/* receive ghost cells from left neighbor */ /* receive ghost cells from left neighbor */
buf = f + (c->imaxLocal + 2);
MPI_Irecv(buf, MPI_Irecv(buf,
1, 1,
c->bufferTypes[LEFT], c->bufferTypes[LEFT],
@ -207,8 +221,8 @@ void commShift(Comm* c, double* f, double* g)
c->comm, c->comm,
&requests[2]); &requests[2]);
buf = f + (c->imaxLocal + 2) + (c->imaxLocal + 1);
/* send ghost cells to right neighbor */ /* send ghost cells to right neighbor */
buf = f + (c->imaxLocal + 2) + (c->imaxLocal);
MPI_Isend(buf, MPI_Isend(buf,
1, 1,
c->bufferTypes[RIGHT], c->bufferTypes[RIGHT],
@ -228,44 +242,18 @@ void commCollectResult(Comm* c,
double* u, double* u,
double* v, double* v,
double* p, double* p,
int jmax, int imax,
int imax) int jmax)
{ {
#ifdef _MPI #ifdef _MPI
int offset[c->size * NDIMS];
int imaxLocal[c->size];
int jmaxLocal[c->size];
MPI_Gather(&c->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Gather(&c->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (c->rank == 0) {
for (int i = 0; i < c->size; i++) {
int coords[NDIMS];
MPI_Cart_coords(c->comm, i, NDIMS, coords);
offset[i * NDIMS + IDIM] = sum(imaxLocal, coords[IDIM]);
offset[i * NDIMS + JDIM] = sum(jmaxLocal, coords[JDIM]);
printf("Rank: %d, Coords(j,i): %d %d, Size(j,i): %d %d "
"Offset(j,i): %d %d\n",
i,
coords[JDIM],
coords[IDIM],
jmaxLocal[i],
imaxLocal[i],
offset[i * NDIMS + JDIM],
offset[i * NDIMS + IDIM]);
}
}
/* collect P */ /* collect P */
assembleResult(c, p, pg, jmax, imax); assembleResult(c, p, pg, imax, jmax);
/* collect U */ /* collect U */
assembleResult(c, u, ug, jmax, imax); assembleResult(c, u, ug, imax, jmax);
/* collect V */ /* collect V */
assembleResult(c, v, vg, jmax, imax); assembleResult(c, v, vg, imax, jmax);
#endif #endif
} }

54
BasicSolver/2D-mpi/src/comm-v3.c
View File

@ -23,7 +23,7 @@ static int sum(int* sizes, int position)
return sum; return sum;
} }
static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax) static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax)
{ {
MPI_Request* requests; MPI_Request* requests;
int numRequests = 1; int numRequests = 1;
@ -36,7 +36,7 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
/* all ranks send their bulk array, but including external boundary layer */ /* all ranks send their bulk array, including the external boundary layer */
MPI_Datatype bulkType; MPI_Datatype bulkType;
int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
@ -82,6 +82,19 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax
MPI_Cart_coords(c->comm, i, NDIMS, coords); MPI_Cart_coords(c->comm, i, NDIMS, coords);
int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]), int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
sum(newSizesI, coords[IDIM]) }; sum(newSizesI, coords[IDIM]) };
printf(
"Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
"Starts(i,j): %d %d\n",
i,
coords[IDIM],
coords[JDIM],
oldSizes[IDIM],
oldSizes[JDIM],
newSizes[IDIM],
newSizes[JDIM],
starts[IDIM],
starts[JDIM]);
MPI_Type_create_subarray(NDIMS, MPI_Type_create_subarray(NDIMS,
oldSizes, oldSizes,
newSizes, newSizes,
@ -92,6 +105,7 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax
MPI_Type_commit(&domainType); MPI_Type_commit(&domainType);
MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]);
MPI_Type_free(&domainType);
} }
} }
@ -194,44 +208,18 @@ void commCollectResult(Comm* c,
double* u, double* u,
double* v, double* v,
double* p, double* p,
int jmax, int imax,
int imax) int jmax)
{ {
#ifdef _MPI #ifdef _MPI
int offset[c->size * NDIMS];
int imaxLocal[c->size];
int jmaxLocal[c->size];
MPI_Gather(&c->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Gather(&c->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (c->rank == 0) {
for (int i = 0; i < c->size; i++) {
int coords[NDIMS];
MPI_Cart_coords(c->comm, i, NDIMS, coords);
offset[i * NDIMS + IDIM] = sum(imaxLocal, coords[IDIM]);
offset[i * NDIMS + JDIM] = sum(jmaxLocal, coords[JDIM]);
printf("Rank: %d, Coords(j,i): %d %d, Size(j,i): %d %d "
"Offset(j,i): %d %d\n",
i,
coords[JDIM],
coords[IDIM],
jmaxLocal[i],
imaxLocal[i],
offset[i * NDIMS + JDIM],
offset[i * NDIMS + IDIM]);
}
}
/* collect P */ /* collect P */
assembleResult(c, p, pg, jmax, imax); assembleResult(c, p, pg, imax, jmax);
/* collect U */ /* collect U */
assembleResult(c, u, ug, jmax, imax); assembleResult(c, u, ug, imax, jmax);
/* collect V */ /* collect V */
assembleResult(c, v, vg, jmax, imax); assembleResult(c, v, vg, imax, jmax);
#endif #endif
} }

10
BasicSolver/2D-mpi/src/comm.c
View File

@ -43,12 +43,18 @@ void commPrintConfig(Comm* c)
c->neighbours[TOP], c->neighbours[TOP],
c->neighbours[LEFT], c->neighbours[LEFT],
c->neighbours[RIGHT]); c->neighbours[RIGHT]);
printf("\tIs boundary:\n");
printf("\t\tLEFT: %d\n", commIsBoundary(c, LEFT));
printf("\t\tRIGHT: %d\n", commIsBoundary(c, RIGHT));
printf("\t\tBOTTOM: %d\n", commIsBoundary(c, BOTTOM));
printf("\t\tTOP: %d\n", commIsBoundary(c, TOP));
printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]); printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]);
printf("\tDims (i,j) %d %d\n", c->dims[IDIM], c->dims[JDIM]);
printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal); printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal);
fflush(stdout); fflush(stdout);
} }
MPI_Barrier(MPI_COMM_WORLD);
} }
MPI_Barrier(MPI_COMM_WORLD);
#endif #endif
} }
@ -90,7 +96,7 @@ static void testWriteFile(char* filename, double* grid, int imax, int jmax)
for (int j = 0; j < jmax + 2; j++) { for (int j = 0; j < jmax + 2; j++) {
for (int i = 0; i < imax + 2; i++) { for (int i = 0; i < imax + 2; i++) {
fprintf(fp, "%f ", grid[j * (imax + 2) + i]); fprintf(fp, "%.2f ", grid[j * (imax + 2) + i]);
} }
fprintf(fp, "\n"); fprintf(fp, "\n");
} }

2
BasicSolver/2D-mpi/src/comm.h
View File

@ -11,7 +11,7 @@
#endif #endif
enum direction { LEFT = 0, RIGHT, BOTTOM, TOP, NDIRS }; enum direction { LEFT = 0, RIGHT, BOTTOM, TOP, NDIRS };
enum dimension { IDIM = 0, JDIM, NDIMS }; enum dimension { JDIM = 0, IDIM, NDIMS };
enum layer { HALO = 0, BULK }; enum layer { HALO = 0, BULK };
enum op { MAX = 0, SUM }; enum op { MAX = 0, SUM };

2
BasicSolver/2D-mpi/src/main.c
View File

@ -24,7 +24,7 @@ static void writeResults(Solver* s)
double* vg = allocate(64, bytesize); double* vg = allocate(64, bytesize);
double* pg = allocate(64, bytesize); double* pg = allocate(64, bytesize);
commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->jmax, s->imax); commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->imax, s->jmax);
writeResult(s, ug, vg, pg); writeResult(s, ug, vg, pg);
free(ug); free(ug);

2
BasicSolver/2D-mpi/src/solver.c
View File

@ -515,7 +515,7 @@ void writeResult(Solver* s, double* u, double* v, double* p)
y = (double)(j - 0.5) * dy; y = (double)(j - 0.5) * dy;
for (int i = 1; i <= imax; i++) { for (int i = 1; i <= imax; i++) {
x = (double)(i - 0.5) * dx; x = (double)(i - 0.5) * dx;
fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]); fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax + 2) + i]);
} }
fprintf(fp, "\n"); fprintf(fp, "\n");
} }