diff --git a/BasicSolver/2D-mpi/config.mk b/BasicSolver/2D-mpi/config.mk index f9ec1d1..a7b7925 100644 --- a/BasicSolver/2D-mpi/config.mk +++ b/BasicSolver/2D-mpi/config.mk @@ -2,7 +2,7 @@ TAG ?= CLANG ENABLE_MPI ?= true ENABLE_OPENMP ?= false -COMM_TYPE ?= v2 +COMM_TYPE ?= v3 #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 diff --git a/BasicSolver/2D-mpi/src/comm-v2.c b/BasicSolver/2D-mpi/src/comm-v2.c index 79b6687..db4569b 100644 --- a/BasicSolver/2D-mpi/src/comm-v2.c +++ b/BasicSolver/2D-mpi/src/comm-v2.c @@ -22,7 +22,7 @@ static int sum(int* sizes, int position) return sum; } -static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax) +static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax) { MPI_Request* requests; int numRequests = 1; @@ -35,10 +35,10 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); - /* all ranks send their bulk array, but including external boundary layer */ + /* all ranks send their bulk array, including the external boundary layer */ MPI_Datatype bulkType; - int oldSizes[NDIMS] = { c->imaxLocal + 2, c->jmaxLocal + 2 }; - int newSizes[NDIMS] = { c->imaxLocal, c->jmaxLocal }; + int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; + int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; int starts[NDIMS] = { 1, 1 }; if (commIsBoundary(c, LEFT)) { @@ -75,12 +75,25 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax if (c->rank == 0) { for (int i = 0; i < c->size; i++) { MPI_Datatype domainType; - int oldSizes[NDIMS] = { imax + 2, jmax + 2 }; - int newSizes[NDIMS] = { newSizesI[i], newSizesJ[i] }; + int oldSizes[NDIMS] = { jmax + 2, imax + 2 }; + int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] }; int coords[NDIMS]; MPI_Cart_coords(c->comm, i, NDIMS, coords); - int starts[NDIMS] = { sum(newSizesI, coords[IDIM]), - sum(newSizesJ, coords[JDIM]) }; + int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]), + sum(newSizesI, coords[IDIM]) }; + printf( + "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d " + "Starts(i,j): %d %d\n", + i, + coords[IDIM], + coords[JDIM], + oldSizes[IDIM], + oldSizes[JDIM], + newSizes[IDIM], + newSizes[JDIM], + starts[IDIM], + starts[JDIM]); + MPI_Type_create_subarray(NDIMS, oldSizes, newSizes, @@ -91,6 +104,7 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax MPI_Type_commit(&domainType); MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); } } @@ -182,8 +196,8 @@ void commShift(Comm* c, double* f, double* g) MPI_REQUEST_NULL }; /* shift G */ - double* buf = g + 1; /* receive ghost cells from bottom neighbor */ + double* buf = g + 1; MPI_Irecv(buf, 1, c->bufferTypes[BOTTOM], @@ -192,13 +206,13 @@ void commShift(Comm* c, double* f, double* g) c->comm, &requests[0]); - buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; /* send ghost cells to top neighbor */ + buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; MPI_Isend(buf, 1, c->bufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]); /* shift F */ - buf = f + (c->imaxLocal + 2); /* receive ghost cells from left neighbor */ + buf = f + (c->imaxLocal + 2); MPI_Irecv(buf, 1, c->bufferTypes[LEFT], @@ -207,8 +221,8 @@ void commShift(Comm* c, double* f, double* g) c->comm, &requests[2]); - buf = f + (c->imaxLocal + 2) + (c->imaxLocal + 1); /* send ghost cells to right neighbor */ + buf = f + (c->imaxLocal + 2) + (c->imaxLocal); MPI_Isend(buf, 1, c->bufferTypes[RIGHT], @@ -228,44 +242,18 @@ void commCollectResult(Comm* c, double* u, double* v, double* p, - int jmax, - int imax) + int imax, + int jmax) { #ifdef _MPI - - int offset[c->size * NDIMS]; - int imaxLocal[c->size]; - int jmaxLocal[c->size]; - - MPI_Gather(&c->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Gather(&c->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); - - if (c->rank == 0) { - for (int i = 0; i < c->size; i++) { - int coords[NDIMS]; - MPI_Cart_coords(c->comm, i, NDIMS, coords); - offset[i * NDIMS + IDIM] = sum(imaxLocal, coords[IDIM]); - offset[i * NDIMS + JDIM] = sum(jmaxLocal, coords[JDIM]); - printf("Rank: %d, Coords(j,i): %d %d, Size(j,i): %d %d " - "Offset(j,i): %d %d\n", - i, - coords[JDIM], - coords[IDIM], - jmaxLocal[i], - imaxLocal[i], - offset[i * NDIMS + JDIM], - offset[i * NDIMS + IDIM]); - } - } - /* collect P */ - assembleResult(c, p, pg, jmax, imax); + assembleResult(c, p, pg, imax, jmax); /* collect U */ - assembleResult(c, u, ug, jmax, imax); + assembleResult(c, u, ug, imax, jmax); /* collect V */ - assembleResult(c, v, vg, jmax, imax); + assembleResult(c, v, vg, imax, jmax); #endif } diff --git a/BasicSolver/2D-mpi/src/comm-v3.c b/BasicSolver/2D-mpi/src/comm-v3.c index e2e9d8c..c8bd0b5 100644 --- a/BasicSolver/2D-mpi/src/comm-v3.c +++ b/BasicSolver/2D-mpi/src/comm-v3.c @@ -23,7 +23,7 @@ static int sum(int* sizes, int position) return sum; } -static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax) +static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax) { MPI_Request* requests; int numRequests = 1; @@ -36,7 +36,7 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); - /* all ranks send their bulk array, but including external boundary layer */ + /* all ranks send their bulk array, including the external boundary layer */ MPI_Datatype bulkType; int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; @@ -82,6 +82,19 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax MPI_Cart_coords(c->comm, i, NDIMS, coords); int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]), sum(newSizesI, coords[IDIM]) }; + printf( + "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d " + "Starts(i,j): %d %d\n", + i, + coords[IDIM], + coords[JDIM], + oldSizes[IDIM], + oldSizes[JDIM], + newSizes[IDIM], + newSizes[JDIM], + starts[IDIM], + starts[JDIM]); + MPI_Type_create_subarray(NDIMS, oldSizes, newSizes, @@ -92,6 +105,7 @@ static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax MPI_Type_commit(&domainType); MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); } } @@ -194,44 +208,18 @@ void commCollectResult(Comm* c, double* u, double* v, double* p, - int jmax, - int imax) + int imax, + int jmax) { #ifdef _MPI - - int offset[c->size * NDIMS]; - int imaxLocal[c->size]; - int jmaxLocal[c->size]; - - MPI_Gather(&c->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Gather(&c->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); - - if (c->rank == 0) { - for (int i = 0; i < c->size; i++) { - int coords[NDIMS]; - MPI_Cart_coords(c->comm, i, NDIMS, coords); - offset[i * NDIMS + IDIM] = sum(imaxLocal, coords[IDIM]); - offset[i * NDIMS + JDIM] = sum(jmaxLocal, coords[JDIM]); - printf("Rank: %d, Coords(j,i): %d %d, Size(j,i): %d %d " - "Offset(j,i): %d %d\n", - i, - coords[JDIM], - coords[IDIM], - jmaxLocal[i], - imaxLocal[i], - offset[i * NDIMS + JDIM], - offset[i * NDIMS + IDIM]); - } - } - /* collect P */ - assembleResult(c, p, pg, jmax, imax); + assembleResult(c, p, pg, imax, jmax); /* collect U */ - assembleResult(c, u, ug, jmax, imax); + assembleResult(c, u, ug, imax, jmax); /* collect V */ - assembleResult(c, v, vg, jmax, imax); + assembleResult(c, v, vg, imax, jmax); #endif } diff --git a/BasicSolver/2D-mpi/src/comm.c b/BasicSolver/2D-mpi/src/comm.c index 5568aa4..fab8f98 100644 --- a/BasicSolver/2D-mpi/src/comm.c +++ b/BasicSolver/2D-mpi/src/comm.c @@ -43,12 +43,18 @@ void commPrintConfig(Comm* c) c->neighbours[TOP], c->neighbours[LEFT], c->neighbours[RIGHT]); + printf("\tIs boundary:\n"); + printf("\t\tLEFT: %d\n", commIsBoundary(c, LEFT)); + printf("\t\tRIGHT: %d\n", commIsBoundary(c, RIGHT)); + printf("\t\tBOTTOM: %d\n", commIsBoundary(c, BOTTOM)); + printf("\t\tTOP: %d\n", commIsBoundary(c, TOP)); printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]); + printf("\tDims (i,j) %d %d\n", c->dims[IDIM], c->dims[JDIM]); printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal); fflush(stdout); } + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); #endif } @@ -90,7 +96,7 @@ static void testWriteFile(char* filename, double* grid, int imax, int jmax) for (int j = 0; j < jmax + 2; j++) { for (int i = 0; i < imax + 2; i++) { - fprintf(fp, "%f ", grid[j * (imax + 2) + i]); + fprintf(fp, "%.2f ", grid[j * (imax + 2) + i]); } fprintf(fp, "\n"); } diff --git a/BasicSolver/2D-mpi/src/comm.h b/BasicSolver/2D-mpi/src/comm.h index 53864c8..bde2405 100644 --- a/BasicSolver/2D-mpi/src/comm.h +++ b/BasicSolver/2D-mpi/src/comm.h @@ -11,7 +11,7 @@ #endif enum direction { LEFT = 0, RIGHT, BOTTOM, TOP, NDIRS }; -enum dimension { IDIM = 0, JDIM, NDIMS }; +enum dimension { JDIM = 0, IDIM, NDIMS }; enum layer { HALO = 0, BULK }; enum op { MAX = 0, SUM }; diff --git a/BasicSolver/2D-mpi/src/main.c b/BasicSolver/2D-mpi/src/main.c index 6e2ffda..616f0a1 100644 --- a/BasicSolver/2D-mpi/src/main.c +++ b/BasicSolver/2D-mpi/src/main.c @@ -24,7 +24,7 @@ static void writeResults(Solver* s) double* vg = allocate(64, bytesize); double* pg = allocate(64, bytesize); - commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->jmax, s->imax); + commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->imax, s->jmax); writeResult(s, ug, vg, pg); free(ug); diff --git a/BasicSolver/2D-mpi/src/solver.c b/BasicSolver/2D-mpi/src/solver.c index 83d4cd4..be2922d 100644 --- a/BasicSolver/2D-mpi/src/solver.c +++ b/BasicSolver/2D-mpi/src/solver.c @@ -515,7 +515,7 @@ void writeResult(Solver* s, double* u, double* v, double* p) y = (double)(j - 0.5) * dy; for (int i = 1; i <= imax; i++) { x = (double)(i - 0.5) * dx; - fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]); + fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax + 2) + i]); } fprintf(fp, "\n"); }