Debug 2D mpi version. Does not yet work v2 upwards.
This commit is contained in:
parent
6769c7acf0
commit
2d759b106a
@ -48,6 +48,8 @@ clean:
|
|||||||
distclean: clean
|
distclean: clean
|
||||||
$(info ===> DIST CLEAN)
|
$(info ===> DIST CLEAN)
|
||||||
@rm -f $(TARGET)
|
@rm -f $(TARGET)
|
||||||
|
@rm -f *.dat
|
||||||
|
@rm -f *.png
|
||||||
|
|
||||||
info:
|
info:
|
||||||
$(info $(CFLAGS))
|
$(info $(CFLAGS))
|
||||||
|
@ -2,11 +2,12 @@
|
|||||||
TAG ?= CLANG
|
TAG ?= CLANG
|
||||||
ENABLE_MPI ?= true
|
ENABLE_MPI ?= true
|
||||||
ENABLE_OPENMP ?= false
|
ENABLE_OPENMP ?= false
|
||||||
COMM_TYPE ?= v3
|
COMM_TYPE ?= v2
|
||||||
|
|
||||||
#Feature options
|
#Feature options
|
||||||
OPTIONS += -DARRAY_ALIGNMENT=64
|
OPTIONS += -DARRAY_ALIGNMENT=64
|
||||||
#OPTIONS += -DVERBOSE
|
OPTIONS += -DVERBOSE
|
||||||
|
# OPTIONS += -DTEST
|
||||||
#OPTIONS += -DVERBOSE_AFFINITY
|
#OPTIONS += -DVERBOSE_AFFINITY
|
||||||
#OPTIONS += -DVERBOSE_DATASIZE
|
#OPTIONS += -DVERBOSE_DATASIZE
|
||||||
#OPTIONS += -DVERBOSE_TIMER
|
#OPTIONS += -DVERBOSE_TIMER
|
||||||
|
@ -15,7 +15,7 @@ bcRight 1 #
|
|||||||
gx 0.0 # Body forces (e.g. gravity)
|
gx 0.0 # Body forces (e.g. gravity)
|
||||||
gy 0.0 #
|
gy 0.0 #
|
||||||
|
|
||||||
re 10.0 # Reynolds number
|
re 100.0 # Reynolds number
|
||||||
|
|
||||||
u_init 0.0 # initial value for velocity in x-direction
|
u_init 0.0 # initial value for velocity in x-direction
|
||||||
v_init 0.0 # initial value for velocity in y-direction
|
v_init 0.0 # initial value for velocity in y-direction
|
||||||
@ -26,13 +26,13 @@ p_init 0.0 # initial value for pressure
|
|||||||
|
|
||||||
xlength 1.0 # domain size in x-direction
|
xlength 1.0 # domain size in x-direction
|
||||||
ylength 1.0 # domain size in y-direction
|
ylength 1.0 # domain size in y-direction
|
||||||
imax 100 # number of interior cells in x-direction
|
imax 80 # number of interior cells in x-direction
|
||||||
jmax 100 # number of interior cells in y-direction
|
jmax 80 # number of interior cells in y-direction
|
||||||
|
|
||||||
# Time Data:
|
# Time Data:
|
||||||
# ---------
|
# ---------
|
||||||
|
|
||||||
te 5.0 # final time
|
te 10.0 # final time
|
||||||
dt 0.02 # time stepsize
|
dt 0.02 # time stepsize
|
||||||
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
||||||
|
|
||||||
@ -41,6 +41,6 @@ tau 0.5 # safety factor for time stepsize control (<0 constant delt)
|
|||||||
|
|
||||||
itermax 1000 # maximal number of pressure iteration in one time step
|
itermax 1000 # maximal number of pressure iteration in one time step
|
||||||
eps 0.001 # stopping tolerance for pressure iteration
|
eps 0.001 # stopping tolerance for pressure iteration
|
||||||
omg 1.7 # relaxation parameter for SOR iteration
|
omg 1.9 # relaxation parameter for SOR iteration
|
||||||
gamma 0.9 # upwind differencing factor gamma
|
gamma 0.9 # upwind differencing factor gamma
|
||||||
#===============================================================================
|
#===============================================================================
|
||||||
|
@ -1,21 +1,15 @@
|
|||||||
/*
|
/*
|
||||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
* All rights reserved. This file is part of nusif-solver.
|
* All rights reserved. This file is part of nusif-solver.
|
||||||
* Use of this source code is governed by a MIT style
|
* Use of this source code is governed by a MIT style
|
||||||
* license that can be found in the LICENSE file.
|
* license that can be found in the LICENSE file.
|
||||||
*/
|
*/
|
||||||
#include <stdio.h>
|
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
#include "comm.h"
|
#include "comm.h"
|
||||||
|
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
// subroutines local to this module
|
// subroutines local to this module
|
||||||
static int sizeOfRank(int rank, int size, int N)
|
|
||||||
{
|
|
||||||
return N / size + ((N % size > rank) ? 1 : 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int sum(int* sizes, int position)
|
static int sum(int* sizes, int position)
|
||||||
{
|
{
|
||||||
int sum = 0;
|
int sum = 0;
|
||||||
@ -49,20 +43,9 @@ static void gatherArray(
|
|||||||
#endif // defined _MPI
|
#endif // defined _MPI
|
||||||
|
|
||||||
// exported subroutines
|
// exported subroutines
|
||||||
void commReduction(double* v, int op)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
if (op == MAX) {
|
|
||||||
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
|
|
||||||
} else if (op == SUM) {
|
|
||||||
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
int commIsBoundary(Comm* c, int direction)
|
int commIsBoundary(Comm* c, int direction)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
switch (direction) {
|
switch (direction) {
|
||||||
case LEFT:
|
case LEFT:
|
||||||
return 1;
|
return 1;
|
||||||
@ -84,7 +67,7 @@ int commIsBoundary(Comm* c, int direction)
|
|||||||
|
|
||||||
void commExchange(Comm* c, double* grid)
|
void commExchange(Comm* c, double* grid)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
MPI_Request requests[4] = { MPI_REQUEST_NULL,
|
MPI_Request requests[4] = { MPI_REQUEST_NULL,
|
||||||
MPI_REQUEST_NULL,
|
MPI_REQUEST_NULL,
|
||||||
MPI_REQUEST_NULL,
|
MPI_REQUEST_NULL,
|
||||||
@ -116,7 +99,7 @@ void commExchange(Comm* c, double* grid)
|
|||||||
|
|
||||||
void commShift(Comm* c, double* f, double* g)
|
void commShift(Comm* c, double* f, double* g)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
MPI_Request requests[2] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL };
|
MPI_Request requests[2] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL };
|
||||||
|
|
||||||
/* shift G */
|
/* shift G */
|
||||||
@ -143,7 +126,6 @@ void commShift(Comm* c, double* f, double* g)
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO Merge with seq
|
|
||||||
void commCollectResult(Comm* c,
|
void commCollectResult(Comm* c,
|
||||||
double* ug,
|
double* ug,
|
||||||
double* vg,
|
double* vg,
|
||||||
@ -154,6 +136,7 @@ void commCollectResult(Comm* c,
|
|||||||
int jmax,
|
int jmax,
|
||||||
int imax)
|
int imax)
|
||||||
{
|
{
|
||||||
|
#ifdef _MPI
|
||||||
int *rcvCounts, *displs;
|
int *rcvCounts, *displs;
|
||||||
int cnt = c->jmaxLocal * (imax + 2);
|
int cnt = c->jmaxLocal * (imax + 2);
|
||||||
|
|
||||||
@ -183,49 +166,12 @@ void commCollectResult(Comm* c,
|
|||||||
gatherArray(c, cnt, rcvCounts, displs, p, pg);
|
gatherArray(c, cnt, rcvCounts, displs, p, pg);
|
||||||
gatherArray(c, cnt, rcvCounts, displs, u, ug);
|
gatherArray(c, cnt, rcvCounts, displs, u, ug);
|
||||||
gatherArray(c, cnt, rcvCounts, displs, v, vg);
|
gatherArray(c, cnt, rcvCounts, displs, v, vg);
|
||||||
}
|
|
||||||
|
|
||||||
void commPrintConfig(Comm* c)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
fflush(stdout);
|
|
||||||
MPI_Barrier(MPI_COMM_WORLD);
|
|
||||||
if (commIsMaster(c)) {
|
|
||||||
printf("Communication setup:\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < c->size; i++) {
|
|
||||||
if (i == c->rank) {
|
|
||||||
printf("\tRank %d of %d\n", c->rank, c->size);
|
|
||||||
printf("\tNeighbours (bottom, top, left, right): %d %d, %d, %d\n",
|
|
||||||
c->neighbours[BOTTOM],
|
|
||||||
c->neighbours[TOP],
|
|
||||||
c->neighbours[LEFT],
|
|
||||||
c->neighbours[RIGHT]);
|
|
||||||
printf("\tCoordinates (j,i) %d %d\n", c->coords[JDIM], c->coords[IDIM]);
|
|
||||||
printf("\tLocal domain size (j,i) %dx%d\n", c->jmaxLocal, c->imaxLocal);
|
|
||||||
fflush(stdout);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
MPI_Barrier(MPI_COMM_WORLD);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
void commInit(Comm* c, int argc, char** argv)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
MPI_Init(&argc, &argv);
|
|
||||||
MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
|
|
||||||
MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
|
|
||||||
#else
|
|
||||||
c->rank = 0;
|
|
||||||
c->size = 1;
|
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void commPartition(Comm* c, int jmax, int imax)
|
void commPartition(Comm* c, int jmax, int imax)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
c->imaxLocal = imax;
|
c->imaxLocal = imax;
|
||||||
c->jmaxLocal = sizeOfRank(c->rank, c->size, jmax);
|
c->jmaxLocal = sizeOfRank(c->rank, c->size, jmax);
|
||||||
#else
|
#else
|
||||||
@ -233,15 +179,3 @@ void commPartition(Comm* c, int jmax, int imax)
|
|||||||
c->jmaxLocal = jmax;
|
c->jmaxLocal = jmax;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void commFinalize(Comm* c)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
for (int i = 0; i < NDIRS; i++) {
|
|
||||||
MPI_Type_free(&c->sbufferTypes[i]);
|
|
||||||
MPI_Type_free(&c->rbufferTypes[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
MPI_Finalize();
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
@ -1,42 +1,28 @@
|
|||||||
/*
|
/*
|
||||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
* All rights reserved. This file is part of nusif-solver.
|
* All rights reserved. This file is part of nusif-solver.
|
||||||
* Use of this source code is governed by a MIT style
|
* Use of this source code is governed by a MIT style
|
||||||
* license that can be found in the LICENSE file.
|
* license that can be found in the LICENSE file.
|
||||||
*/
|
*/
|
||||||
#include <stddef.h>
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
#include "comm.h"
|
#include "comm.h"
|
||||||
|
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
// subroutines local to this module
|
// subroutines local to this module
|
||||||
static int sizeOfRank(int rank, int size, int N)
|
static int sum(int* sizes, int position)
|
||||||
{
|
{
|
||||||
return N / size + ((N % size > rank) ? 1 : 0);
|
int sum = 0;
|
||||||
|
|
||||||
|
for (int i = 0; i < position; i++) {
|
||||||
|
sum += sizes[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
return sum;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void setupCommunication(Comm* c, int direction, int layer)
|
static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax)
|
||||||
{
|
|
||||||
MPI_Datatype type;
|
|
||||||
size_t dblsize = sizeof(double);
|
|
||||||
int imaxLocal = c->imaxLocal;
|
|
||||||
int jmaxLocal = c->jmaxLocal;
|
|
||||||
int sizes[NDIMS];
|
|
||||||
int subSizes[NDIMS];
|
|
||||||
int starts[NDIMS];
|
|
||||||
int offset = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void assembleResult(Comm* c,
|
|
||||||
double* src,
|
|
||||||
double* dst,
|
|
||||||
int imaxLocal[],
|
|
||||||
int jmaxLocal[],
|
|
||||||
int offset[],
|
|
||||||
int jmax,
|
|
||||||
int imax)
|
|
||||||
{
|
{
|
||||||
MPI_Request* requests;
|
MPI_Request* requests;
|
||||||
int numRequests = 1;
|
int numRequests = 1;
|
||||||
@ -49,11 +35,27 @@ static void assembleResult(Comm* c,
|
|||||||
|
|
||||||
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
|
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
|
||||||
|
|
||||||
/* all ranks send their bulk array */
|
/* all ranks send their bulk array, but including external boundary layer */
|
||||||
MPI_Datatype bulkType;
|
MPI_Datatype bulkType;
|
||||||
int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
|
int oldSizes[NDIMS] = { c->imaxLocal + 2, c->jmaxLocal + 2 };
|
||||||
int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
|
int newSizes[NDIMS] = { c->imaxLocal, c->jmaxLocal };
|
||||||
int starts[NDIMS] = { 1, 1 };
|
int starts[NDIMS] = { 1, 1 };
|
||||||
|
|
||||||
|
if (commIsBoundary(c, LEFT)) {
|
||||||
|
newSizes[IDIM] += 1;
|
||||||
|
starts[IDIM] = 0;
|
||||||
|
}
|
||||||
|
if (commIsBoundary(c, RIGHT)) {
|
||||||
|
newSizes[IDIM] += 1;
|
||||||
|
}
|
||||||
|
if (commIsBoundary(c, BOTTOM)) {
|
||||||
|
newSizes[JDIM] += 1;
|
||||||
|
starts[JDIM] = 0;
|
||||||
|
}
|
||||||
|
if (commIsBoundary(c, TOP)) {
|
||||||
|
newSizes[JDIM] += 1;
|
||||||
|
}
|
||||||
|
|
||||||
MPI_Type_create_subarray(NDIMS,
|
MPI_Type_create_subarray(NDIMS,
|
||||||
oldSizes,
|
oldSizes,
|
||||||
newSizes,
|
newSizes,
|
||||||
@ -62,16 +64,23 @@ static void assembleResult(Comm* c,
|
|||||||
MPI_DOUBLE,
|
MPI_DOUBLE,
|
||||||
&bulkType);
|
&bulkType);
|
||||||
MPI_Type_commit(&bulkType);
|
MPI_Type_commit(&bulkType);
|
||||||
|
|
||||||
MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
|
MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
|
||||||
|
|
||||||
|
int newSizesI[c->size];
|
||||||
|
int newSizesJ[c->size];
|
||||||
|
MPI_Gather(&newSizes[IDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD);
|
||||||
|
MPI_Gather(&newSizes[JDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD);
|
||||||
|
|
||||||
/* rank 0 assembles the subdomains */
|
/* rank 0 assembles the subdomains */
|
||||||
if (c->rank == 0) {
|
if (c->rank == 0) {
|
||||||
for (int i = 0; i < c->size; i++) {
|
for (int i = 0; i < c->size; i++) {
|
||||||
MPI_Datatype domainType;
|
MPI_Datatype domainType;
|
||||||
int oldSizes[NDIMS] = { jmax, imax };
|
int oldSizes[NDIMS] = { imax + 2, jmax + 2 };
|
||||||
int newSizes[NDIMS] = { jmaxLocal[i], imaxLocal[i] };
|
int newSizes[NDIMS] = { newSizesI[i], newSizesJ[i] };
|
||||||
int starts[NDIMS] = { offset[i * NDIMS + JDIM], offset[i * NDIMS + IDIM] };
|
int coords[NDIMS];
|
||||||
|
MPI_Cart_coords(c->comm, i, NDIMS, coords);
|
||||||
|
int starts[NDIMS] = { sum(newSizesI, coords[IDIM]),
|
||||||
|
sum(newSizesJ, coords[JDIM]) };
|
||||||
MPI_Type_create_subarray(NDIMS,
|
MPI_Type_create_subarray(NDIMS,
|
||||||
oldSizes,
|
oldSizes,
|
||||||
newSizes,
|
newSizes,
|
||||||
@ -87,34 +96,12 @@ static void assembleResult(Comm* c,
|
|||||||
|
|
||||||
MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
|
MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int sum(int* sizes, int position)
|
|
||||||
{
|
|
||||||
int sum = 0;
|
|
||||||
|
|
||||||
for (int i = 0; i < position; i++) {
|
|
||||||
sum += sizes[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
return sum;
|
|
||||||
}
|
|
||||||
#endif // defined _MPI
|
#endif // defined _MPI
|
||||||
|
|
||||||
// exported subroutines
|
// exported subroutines
|
||||||
void commReduction(double* v, int op)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
if (op == MAX) {
|
|
||||||
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
|
|
||||||
} else if (op == SUM) {
|
|
||||||
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
int commIsBoundary(Comm* c, int direction)
|
int commIsBoundary(Comm* c, int direction)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
switch (direction) {
|
switch (direction) {
|
||||||
case LEFT:
|
case LEFT:
|
||||||
return c->coords[IDIM] == 0;
|
return c->coords[IDIM] == 0;
|
||||||
@ -136,63 +123,51 @@ int commIsBoundary(Comm* c, int direction)
|
|||||||
|
|
||||||
void commExchange(Comm* c, double* grid)
|
void commExchange(Comm* c, double* grid)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
double* buf[8];
|
double* sbuf[NDIRS];
|
||||||
|
double* rbuf[NDIRS];
|
||||||
MPI_Request requests[8];
|
MPI_Request requests[8];
|
||||||
for (int i = 0; i < 8; i++)
|
for (int i = 0; i < 8; i++)
|
||||||
requests[i] = MPI_REQUEST_NULL;
|
requests[i] = MPI_REQUEST_NULL;
|
||||||
|
|
||||||
buf[0] = grid + 1; // recv bottom
|
rbuf[LEFT] = grid + (c->imaxLocal + 2);
|
||||||
buf[1] = grid + (c->imaxLocal + 2) + 1; // send bottom
|
rbuf[RIGHT] = grid + (c->imaxLocal + 2) + (c->imaxLocal + 1);
|
||||||
buf[2] = grid + (c->jmaxLocal + 1) * (c->imaxLocal + 2) + 1; // recv top
|
rbuf[BOTTOM] = grid + 1;
|
||||||
buf[3] = grid + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; // send top
|
rbuf[TOP] = grid + (c->jmaxLocal + 1) * (c->imaxLocal + 2) + 1;
|
||||||
buf[4] = grid + (c->imaxLocal + 2); // recv left
|
sbuf[LEFT] = grid + (c->imaxLocal + 2) + 1;
|
||||||
buf[5] = grid + (c->imaxLocal + 2) + 1; // send left
|
sbuf[RIGHT] = grid + (c->imaxLocal + 2) + (c->imaxLocal);
|
||||||
buf[6] = grid + (c->imaxLocal + 2) + (c->imaxLocal + 1); // recv right
|
sbuf[BOTTOM] = grid + (c->imaxLocal + 2) + 1;
|
||||||
buf[7] = grid + (c->imaxLocal + 2) + (c->imaxLocal); // send right
|
sbuf[TOP] = grid + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
|
||||||
|
// buf[0] = grid + (c->imaxLocal + 2); // recv left
|
||||||
|
// buf[1] = grid + (c->imaxLocal + 2) + 1; // send left
|
||||||
|
// buf[2] = grid + (c->imaxLocal + 2) + (c->imaxLocal + 1); // recv right
|
||||||
|
// buf[3] = grid + (c->imaxLocal + 2) + (c->imaxLocal); // send right
|
||||||
|
// buf[4] = grid + 1; // recv bottom
|
||||||
|
// buf[5] = grid + (c->imaxLocal + 2) + 1; // send bottom
|
||||||
|
// buf[6] = grid + (c->jmaxLocal + 1) * (c->imaxLocal + 2) + 1; // recv top
|
||||||
|
// buf[7] = grid + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; // send top
|
||||||
|
|
||||||
for (int i = 2; i < 4; i++) {
|
for (int i = 0; i < NDIRS; i++) {
|
||||||
int tag = 0;
|
int tag = 0;
|
||||||
if (c->neighbours[i] != MPI_PROC_NULL) {
|
if (c->neighbours[i] != MPI_PROC_NULL) {
|
||||||
|
// printf("DEBUG: Rank %d - SendRecv with %d\n", c->rank, c->neighbours[i]);
|
||||||
tag = c->neighbours[i];
|
tag = c->neighbours[i];
|
||||||
}
|
}
|
||||||
/* exchange ghost cells with bottom/top neighbor */
|
MPI_Irecv(rbuf[i],
|
||||||
MPI_Irecv(buf[i * 2],
|
|
||||||
1,
|
1,
|
||||||
c->rbufferTypes[0],
|
c->bufferTypes[i],
|
||||||
c->neighbours[i],
|
c->neighbours[i],
|
||||||
tag,
|
tag,
|
||||||
c->comm,
|
c->comm,
|
||||||
&requests[i * 2]);
|
&requests[i * 2]);
|
||||||
MPI_Isend(buf[(i * 2) + 1],
|
MPI_Isend(sbuf[i],
|
||||||
1,
|
1,
|
||||||
c->rbufferTypes[0],
|
c->bufferTypes[i],
|
||||||
c->neighbours[i],
|
c->neighbours[i],
|
||||||
c->rank,
|
c->rank,
|
||||||
c->comm,
|
c->comm,
|
||||||
&requests[i * 2 + 1]);
|
&requests[i * 2 + 1]);
|
||||||
}
|
}
|
||||||
for (int i = 0; i < 2; i++) {
|
|
||||||
int tag = 0;
|
|
||||||
if (c->neighbours[i] != MPI_PROC_NULL) {
|
|
||||||
tag = c->neighbours[i];
|
|
||||||
}
|
|
||||||
/* exchange ghost cells with left/right neighbor */
|
|
||||||
MPI_Irecv(buf[i * 2 + 4],
|
|
||||||
1,
|
|
||||||
c->sbufferTypes[0],
|
|
||||||
c->neighbours[i],
|
|
||||||
tag,
|
|
||||||
c->comm,
|
|
||||||
&requests[i * 2 + 4]);
|
|
||||||
MPI_Isend(buf[i * 2 + 5],
|
|
||||||
1,
|
|
||||||
c->sbufferTypes[0],
|
|
||||||
c->neighbours[i],
|
|
||||||
c->rank,
|
|
||||||
c->comm,
|
|
||||||
&requests[(i * 2) + 5]);
|
|
||||||
}
|
|
||||||
|
|
||||||
MPI_Waitall(8, requests, MPI_STATUSES_IGNORE);
|
MPI_Waitall(8, requests, MPI_STATUSES_IGNORE);
|
||||||
#endif
|
#endif
|
||||||
@ -200,7 +175,7 @@ void commExchange(Comm* c, double* grid)
|
|||||||
|
|
||||||
void commShift(Comm* c, double* f, double* g)
|
void commShift(Comm* c, double* f, double* g)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
MPI_Request requests[4] = { MPI_REQUEST_NULL,
|
MPI_Request requests[4] = { MPI_REQUEST_NULL,
|
||||||
MPI_REQUEST_NULL,
|
MPI_REQUEST_NULL,
|
||||||
MPI_REQUEST_NULL,
|
MPI_REQUEST_NULL,
|
||||||
@ -211,7 +186,7 @@ void commShift(Comm* c, double* f, double* g)
|
|||||||
/* receive ghost cells from bottom neighbor */
|
/* receive ghost cells from bottom neighbor */
|
||||||
MPI_Irecv(buf,
|
MPI_Irecv(buf,
|
||||||
1,
|
1,
|
||||||
c->rbufferTypes[0],
|
c->bufferTypes[BOTTOM],
|
||||||
c->neighbours[BOTTOM],
|
c->neighbours[BOTTOM],
|
||||||
0,
|
0,
|
||||||
c->comm,
|
c->comm,
|
||||||
@ -219,16 +194,28 @@ void commShift(Comm* c, double* f, double* g)
|
|||||||
|
|
||||||
buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
|
buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
|
||||||
/* send ghost cells to top neighbor */
|
/* send ghost cells to top neighbor */
|
||||||
MPI_Isend(buf, 1, c->rbufferTypes[0], c->neighbours[TOP], 0, c->comm, &requests[1]);
|
MPI_Isend(buf, 1, c->bufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]);
|
||||||
|
|
||||||
/* shift F */
|
/* shift F */
|
||||||
buf = f + (c->imaxLocal + 2);
|
buf = f + (c->imaxLocal + 2);
|
||||||
/* receive ghost cells from left neighbor */
|
/* receive ghost cells from left neighbor */
|
||||||
MPI_Irecv(buf, 1, c->sbufferTypes[0], c->neighbours[LEFT], 1, c->comm, &requests[2]);
|
MPI_Irecv(buf,
|
||||||
|
1,
|
||||||
|
c->bufferTypes[LEFT],
|
||||||
|
c->neighbours[LEFT],
|
||||||
|
1,
|
||||||
|
c->comm,
|
||||||
|
&requests[2]);
|
||||||
|
|
||||||
buf = f + (c->imaxLocal + 2) + (c->imaxLocal);
|
buf = f + (c->imaxLocal + 2) + (c->imaxLocal + 1);
|
||||||
/* send ghost cells to right neighbor */
|
/* send ghost cells to right neighbor */
|
||||||
MPI_Isend(buf, 1, c->sbufferTypes[0], c->neighbours[RIGHT], 1, c->comm, &requests[3]);
|
MPI_Isend(buf,
|
||||||
|
1,
|
||||||
|
c->bufferTypes[RIGHT],
|
||||||
|
c->neighbours[RIGHT],
|
||||||
|
1,
|
||||||
|
c->comm,
|
||||||
|
&requests[3]);
|
||||||
|
|
||||||
MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
|
MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
|
||||||
#endif
|
#endif
|
||||||
@ -244,6 +231,8 @@ void commCollectResult(Comm* c,
|
|||||||
int jmax,
|
int jmax,
|
||||||
int imax)
|
int imax)
|
||||||
{
|
{
|
||||||
|
#ifdef _MPI
|
||||||
|
|
||||||
int offset[c->size * NDIMS];
|
int offset[c->size * NDIMS];
|
||||||
int imaxLocal[c->size];
|
int imaxLocal[c->size];
|
||||||
int jmaxLocal[c->size];
|
int jmaxLocal[c->size];
|
||||||
@ -270,56 +259,19 @@ void commCollectResult(Comm* c,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* collect P */
|
/* collect P */
|
||||||
assembleResult(c, p, pg, imaxLocal, jmaxLocal, offset, jmax, imax);
|
assembleResult(c, p, pg, jmax, imax);
|
||||||
|
|
||||||
/* collect U */
|
/* collect U */
|
||||||
assembleResult(c, u, ug, imaxLocal, jmaxLocal, offset, jmax, imax);
|
assembleResult(c, u, ug, jmax, imax);
|
||||||
|
|
||||||
/* collect V */
|
/* collect V */
|
||||||
assembleResult(c, v, vg, imaxLocal, jmaxLocal, offset, jmax, imax);
|
assembleResult(c, v, vg, jmax, imax);
|
||||||
}
|
|
||||||
|
|
||||||
void commPrintConfig(Comm* c)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
fflush(stdout);
|
|
||||||
MPI_Barrier(MPI_COMM_WORLD);
|
|
||||||
if (commIsMaster(c)) {
|
|
||||||
printf("Communication setup:\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < c->size; i++) {
|
|
||||||
if (i == c->rank) {
|
|
||||||
printf("\tRank %d of %d\n", c->rank, c->size);
|
|
||||||
printf("\tNeighbours (bottom, top, left, right): %d %d, %d, %d\n",
|
|
||||||
c->neighbours[BOTTOM],
|
|
||||||
c->neighbours[TOP],
|
|
||||||
c->neighbours[LEFT],
|
|
||||||
c->neighbours[RIGHT]);
|
|
||||||
printf("\tCoordinates (j,i) %d %d\n", c->coords[JDIM], c->coords[IDIM]);
|
|
||||||
printf("\tLocal domain size (j,i) %dx%d\n", c->jmaxLocal, c->imaxLocal);
|
|
||||||
fflush(stdout);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
MPI_Barrier(MPI_COMM_WORLD);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
void commInit(Comm* c, int argc, char** argv)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
MPI_Init(&argc, &argv);
|
|
||||||
MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
|
|
||||||
MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
|
|
||||||
#else
|
|
||||||
c->rank = 0;
|
|
||||||
c->size = 1;
|
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void commPartition(Comm* c, int jmax, int imax)
|
void commPartition(Comm* c, int jmax, int imax)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
int dims[NDIMS] = { 0, 0 };
|
int dims[NDIMS] = { 0, 0 };
|
||||||
int periods[NDIMS] = { 0, 0 };
|
int periods[NDIMS] = { 0, 0 };
|
||||||
MPI_Dims_create(c->size, NDIMS, dims);
|
MPI_Dims_create(c->size, NDIMS, dims);
|
||||||
@ -331,25 +283,20 @@ void commPartition(Comm* c, int jmax, int imax)
|
|||||||
c->imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
|
c->imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
|
||||||
c->jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
|
c->jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
|
||||||
|
|
||||||
MPI_Type_contiguous(c->imaxLocal, MPI_DOUBLE, &c->rbufferTypes[0]);
|
MPI_Datatype jBufferType;
|
||||||
MPI_Type_commit(&c->rbufferTypes[0]);
|
MPI_Type_contiguous(c->imaxLocal, MPI_DOUBLE, &jBufferType);
|
||||||
|
MPI_Type_commit(&jBufferType);
|
||||||
|
|
||||||
MPI_Type_vector(c->jmaxLocal, 1, c->imaxLocal + 2, MPI_DOUBLE, &c->sbufferTypes[0]);
|
MPI_Datatype iBufferType;
|
||||||
MPI_Type_commit(&c->sbufferTypes[0]);
|
MPI_Type_vector(c->jmaxLocal, 1, c->imaxLocal + 2, MPI_DOUBLE, &iBufferType);
|
||||||
|
MPI_Type_commit(&iBufferType);
|
||||||
|
|
||||||
|
c->bufferTypes[LEFT] = iBufferType;
|
||||||
|
c->bufferTypes[RIGHT] = iBufferType;
|
||||||
|
c->bufferTypes[BOTTOM] = jBufferType;
|
||||||
|
c->bufferTypes[TOP] = jBufferType;
|
||||||
#else
|
#else
|
||||||
c->imaxLocal = imax;
|
c->imaxLocal = imax;
|
||||||
c->jmaxLocal = jmax;
|
c->jmaxLocal = jmax;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void commFinalize(Comm* c)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
for (int i = 0; i < NDIRS; i++) {
|
|
||||||
MPI_Type_free(&c->sbufferTypes[i]);
|
|
||||||
MPI_Type_free(&c->rbufferTypes[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
MPI_Finalize();
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
/*
|
/*
|
||||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
* All rights reserved. This file is part of nusif-solver.
|
* All rights reserved. This file is part of nusif-solver.
|
||||||
* Use of this source code is governed by a MIT style
|
* Use of this source code is governed by a MIT style
|
||||||
* license that can be found in the LICENSE file.
|
* license that can be found in the LICENSE file.
|
||||||
@ -10,82 +10,20 @@
|
|||||||
|
|
||||||
#include "comm.h"
|
#include "comm.h"
|
||||||
|
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
// subroutines local to this module
|
// subroutines local to this module
|
||||||
static int sizeOfRank(int rank, int size, int N)
|
static int sum(int* sizes, int position)
|
||||||
{
|
{
|
||||||
return N / size + ((N % size > rank) ? 1 : 0);
|
int sum = 0;
|
||||||
|
|
||||||
|
for (int i = 0; i < position; i++) {
|
||||||
|
sum += sizes[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
return sum;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void setupCommunication(Comm* c, int direction, int layer)
|
static void assembleResult(Comm* c, double* src, double* dst, int jmax, int imax)
|
||||||
{
|
|
||||||
MPI_Datatype type;
|
|
||||||
size_t dblsize = sizeof(double);
|
|
||||||
int imaxLocal = c->imaxLocal;
|
|
||||||
int jmaxLocal = c->jmaxLocal;
|
|
||||||
int sizes[NDIMS];
|
|
||||||
int subSizes[NDIMS];
|
|
||||||
int starts[NDIMS];
|
|
||||||
int offset = 0;
|
|
||||||
|
|
||||||
sizes[IDIM] = imaxLocal + 2;
|
|
||||||
sizes[JDIM] = jmaxLocal + 2;
|
|
||||||
|
|
||||||
if (layer == HALO) {
|
|
||||||
offset = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
switch (direction) {
|
|
||||||
case LEFT:
|
|
||||||
subSizes[IDIM] = 1;
|
|
||||||
subSizes[JDIM] = jmaxLocal;
|
|
||||||
starts[IDIM] = 1 - offset;
|
|
||||||
starts[JDIM] = 1;
|
|
||||||
break;
|
|
||||||
case RIGHT:
|
|
||||||
subSizes[IDIM] = 1;
|
|
||||||
subSizes[JDIM] = jmaxLocal;
|
|
||||||
starts[IDIM] = imaxLocal + offset;
|
|
||||||
starts[JDIM] = 1;
|
|
||||||
break;
|
|
||||||
case BOTTOM:
|
|
||||||
subSizes[IDIM] = imaxLocal;
|
|
||||||
subSizes[JDIM] = 1;
|
|
||||||
starts[IDIM] = 1;
|
|
||||||
starts[JDIM] = 1 - offset;
|
|
||||||
break;
|
|
||||||
case TOP:
|
|
||||||
subSizes[IDIM] = imaxLocal;
|
|
||||||
subSizes[JDIM] = 1;
|
|
||||||
starts[IDIM] = 1;
|
|
||||||
starts[JDIM] = jmaxLocal + offset;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
MPI_Type_create_subarray(NDIMS,
|
|
||||||
sizes,
|
|
||||||
subSizes,
|
|
||||||
starts,
|
|
||||||
MPI_ORDER_C,
|
|
||||||
MPI_DOUBLE,
|
|
||||||
&type);
|
|
||||||
MPI_Type_commit(&type);
|
|
||||||
|
|
||||||
if (layer == HALO) {
|
|
||||||
c->rbufferTypes[direction] = type;
|
|
||||||
} else if (layer == BULK) {
|
|
||||||
c->sbufferTypes[direction] = type;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void assembleResult(Comm* c,
|
|
||||||
double* src,
|
|
||||||
double* dst,
|
|
||||||
int imaxLocal[],
|
|
||||||
int jmaxLocal[],
|
|
||||||
int offset[],
|
|
||||||
int jmax,
|
|
||||||
int imax)
|
|
||||||
{
|
{
|
||||||
MPI_Request* requests;
|
MPI_Request* requests;
|
||||||
int numRequests = 1;
|
int numRequests = 1;
|
||||||
@ -98,11 +36,27 @@ static void assembleResult(Comm* c,
|
|||||||
|
|
||||||
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
|
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
|
||||||
|
|
||||||
/* all ranks send their bulk array */
|
/* all ranks send their bulk array, but including external boundary layer */
|
||||||
MPI_Datatype bulkType;
|
MPI_Datatype bulkType;
|
||||||
int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
|
int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
|
||||||
int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
|
int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
|
||||||
int starts[NDIMS] = { 1, 1 };
|
int starts[NDIMS] = { 1, 1 };
|
||||||
|
|
||||||
|
if (commIsBoundary(c, LEFT)) {
|
||||||
|
newSizes[IDIM] += 1;
|
||||||
|
starts[IDIM] = 0;
|
||||||
|
}
|
||||||
|
if (commIsBoundary(c, RIGHT)) {
|
||||||
|
newSizes[IDIM] += 1;
|
||||||
|
}
|
||||||
|
if (commIsBoundary(c, BOTTOM)) {
|
||||||
|
newSizes[JDIM] += 1;
|
||||||
|
starts[JDIM] = 0;
|
||||||
|
}
|
||||||
|
if (commIsBoundary(c, TOP)) {
|
||||||
|
newSizes[JDIM] += 1;
|
||||||
|
}
|
||||||
|
|
||||||
MPI_Type_create_subarray(NDIMS,
|
MPI_Type_create_subarray(NDIMS,
|
||||||
oldSizes,
|
oldSizes,
|
||||||
newSizes,
|
newSizes,
|
||||||
@ -111,16 +65,23 @@ static void assembleResult(Comm* c,
|
|||||||
MPI_DOUBLE,
|
MPI_DOUBLE,
|
||||||
&bulkType);
|
&bulkType);
|
||||||
MPI_Type_commit(&bulkType);
|
MPI_Type_commit(&bulkType);
|
||||||
|
|
||||||
MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
|
MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
|
||||||
|
|
||||||
|
int newSizesI[c->size];
|
||||||
|
int newSizesJ[c->size];
|
||||||
|
MPI_Gather(&newSizes[IDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD);
|
||||||
|
MPI_Gather(&newSizes[JDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD);
|
||||||
|
|
||||||
/* rank 0 assembles the subdomains */
|
/* rank 0 assembles the subdomains */
|
||||||
if (c->rank == 0) {
|
if (c->rank == 0) {
|
||||||
for (int i = 0; i < c->size; i++) {
|
for (int i = 0; i < c->size; i++) {
|
||||||
MPI_Datatype domainType;
|
MPI_Datatype domainType;
|
||||||
int oldSizes[NDIMS] = { jmax, imax };
|
int oldSizes[NDIMS] = { jmax + 2, imax + 2 };
|
||||||
int newSizes[NDIMS] = { jmaxLocal[i], imaxLocal[i] };
|
int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
|
||||||
int starts[NDIMS] = { offset[i * NDIMS + JDIM], offset[i * NDIMS + IDIM] };
|
int coords[NDIMS];
|
||||||
|
MPI_Cart_coords(c->comm, i, NDIMS, coords);
|
||||||
|
int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
|
||||||
|
sum(newSizesI, coords[IDIM]) };
|
||||||
MPI_Type_create_subarray(NDIMS,
|
MPI_Type_create_subarray(NDIMS,
|
||||||
oldSizes,
|
oldSizes,
|
||||||
newSizes,
|
newSizes,
|
||||||
@ -136,34 +97,12 @@ static void assembleResult(Comm* c,
|
|||||||
|
|
||||||
MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
|
MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int sum(int* sizes, int position)
|
|
||||||
{
|
|
||||||
int sum = 0;
|
|
||||||
|
|
||||||
for (int i = 0; i < position; i++) {
|
|
||||||
sum += sizes[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
return sum;
|
|
||||||
}
|
|
||||||
#endif // defined _MPI
|
#endif // defined _MPI
|
||||||
|
|
||||||
// exported subroutines
|
// exported subroutines
|
||||||
void commReduction(double* v, int op)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
if (op == MAX) {
|
|
||||||
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
|
|
||||||
} else if (op == SUM) {
|
|
||||||
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
int commIsBoundary(Comm* c, int direction)
|
int commIsBoundary(Comm* c, int direction)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
switch (direction) {
|
switch (direction) {
|
||||||
case LEFT:
|
case LEFT:
|
||||||
return c->coords[IDIM] == 0;
|
return c->coords[IDIM] == 0;
|
||||||
@ -185,25 +124,24 @@ int commIsBoundary(Comm* c, int direction)
|
|||||||
|
|
||||||
void commExchange(Comm* c, double* grid)
|
void commExchange(Comm* c, double* grid)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
int counts[NDIRS] = { 1, 1, 1, 1 };
|
int counts[NDIRS] = { 1, 1, 1, 1 };
|
||||||
MPI_Aint displs[NDIRS] = { 0, 0, 0, 0 };
|
|
||||||
|
|
||||||
MPI_Neighbor_alltoallw(grid,
|
MPI_Neighbor_alltoallw(grid,
|
||||||
counts,
|
counts,
|
||||||
displs,
|
c->sdispls,
|
||||||
c->sbufferTypes,
|
c->bufferTypes,
|
||||||
grid,
|
grid,
|
||||||
counts,
|
counts,
|
||||||
displs,
|
c->rdispls,
|
||||||
c->rbufferTypes,
|
c->bufferTypes,
|
||||||
c->comm);
|
c->comm);
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void commShift(Comm* c, double* f, double* g)
|
void commShift(Comm* c, double* f, double* g)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
MPI_Request requests[4] = { MPI_REQUEST_NULL,
|
MPI_Request requests[4] = { MPI_REQUEST_NULL,
|
||||||
MPI_REQUEST_NULL,
|
MPI_REQUEST_NULL,
|
||||||
MPI_REQUEST_NULL,
|
MPI_REQUEST_NULL,
|
||||||
@ -211,25 +149,35 @@ void commShift(Comm* c, double* f, double* g)
|
|||||||
|
|
||||||
/* shift G */
|
/* shift G */
|
||||||
/* receive ghost cells from bottom neighbor */
|
/* receive ghost cells from bottom neighbor */
|
||||||
MPI_Irecv(g,
|
double* buf = g + 1;
|
||||||
|
MPI_Irecv(buf,
|
||||||
1,
|
1,
|
||||||
c->rbufferTypes[BOTTOM],
|
c->bufferTypes[BOTTOM],
|
||||||
c->neighbours[BOTTOM],
|
c->neighbours[BOTTOM],
|
||||||
0,
|
0,
|
||||||
c->comm,
|
c->comm,
|
||||||
&requests[0]);
|
&requests[0]);
|
||||||
|
|
||||||
/* send ghost cells to top neighbor */
|
/* send ghost cells to top neighbor */
|
||||||
MPI_Isend(g, 1, c->sbufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]);
|
buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
|
||||||
|
MPI_Isend(buf, 1, c->bufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]);
|
||||||
|
|
||||||
/* shift F */
|
/* shift F */
|
||||||
/* receive ghost cells from left neighbor */
|
/* receive ghost cells from left neighbor */
|
||||||
MPI_Irecv(f, 1, c->rbufferTypes[LEFT], c->neighbours[LEFT], 1, c->comm, &requests[2]);
|
buf = f + (c->imaxLocal + 2);
|
||||||
|
MPI_Irecv(buf,
|
||||||
|
1,
|
||||||
|
c->bufferTypes[LEFT],
|
||||||
|
c->neighbours[LEFT],
|
||||||
|
1,
|
||||||
|
c->comm,
|
||||||
|
&requests[2]);
|
||||||
|
|
||||||
/* send ghost cells to right neighbor */
|
/* send ghost cells to right neighbor */
|
||||||
MPI_Isend(f,
|
buf = f + (c->imaxLocal + 2) + (c->imaxLocal);
|
||||||
|
MPI_Isend(buf,
|
||||||
1,
|
1,
|
||||||
c->sbufferTypes[RIGHT],
|
c->bufferTypes[RIGHT],
|
||||||
c->neighbours[RIGHT],
|
c->neighbours[RIGHT],
|
||||||
1,
|
1,
|
||||||
c->comm,
|
c->comm,
|
||||||
@ -239,7 +187,6 @@ void commShift(Comm* c, double* f, double* g)
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO Merge with seq
|
|
||||||
void commCollectResult(Comm* c,
|
void commCollectResult(Comm* c,
|
||||||
double* ug,
|
double* ug,
|
||||||
double* vg,
|
double* vg,
|
||||||
@ -250,6 +197,8 @@ void commCollectResult(Comm* c,
|
|||||||
int jmax,
|
int jmax,
|
||||||
int imax)
|
int imax)
|
||||||
{
|
{
|
||||||
|
#ifdef _MPI
|
||||||
|
|
||||||
int offset[c->size * NDIMS];
|
int offset[c->size * NDIMS];
|
||||||
int imaxLocal[c->size];
|
int imaxLocal[c->size];
|
||||||
int jmaxLocal[c->size];
|
int jmaxLocal[c->size];
|
||||||
@ -276,56 +225,19 @@ void commCollectResult(Comm* c,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* collect P */
|
/* collect P */
|
||||||
assembleResult(c, p, pg, imaxLocal, jmaxLocal, offset, jmax, imax);
|
assembleResult(c, p, pg, jmax, imax);
|
||||||
|
|
||||||
/* collect U */
|
/* collect U */
|
||||||
assembleResult(c, u, ug, imaxLocal, jmaxLocal, offset, jmax, imax);
|
assembleResult(c, u, ug, jmax, imax);
|
||||||
|
|
||||||
/* collect V */
|
/* collect V */
|
||||||
assembleResult(c, v, vg, imaxLocal, jmaxLocal, offset, jmax, imax);
|
assembleResult(c, v, vg, jmax, imax);
|
||||||
}
|
|
||||||
|
|
||||||
void commPrintConfig(Comm* c)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
fflush(stdout);
|
|
||||||
MPI_Barrier(MPI_COMM_WORLD);
|
|
||||||
if (commIsMaster(c)) {
|
|
||||||
printf("Communication setup:\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < c->size; i++) {
|
|
||||||
if (i == c->rank) {
|
|
||||||
printf("\tRank %d of %d\n", c->rank, c->size);
|
|
||||||
printf("\tNeighbours (bottom, top, left, right): %d %d, %d, %d\n",
|
|
||||||
c->neighbours[BOTTOM],
|
|
||||||
c->neighbours[TOP],
|
|
||||||
c->neighbours[LEFT],
|
|
||||||
c->neighbours[RIGHT]);
|
|
||||||
printf("\tCoordinates (j,i) %d %d\n", c->coords[JDIM], c->coords[IDIM]);
|
|
||||||
printf("\tLocal domain size (j,i) %dx%d\n", c->jmaxLocal, c->imaxLocal);
|
|
||||||
fflush(stdout);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
MPI_Barrier(MPI_COMM_WORLD);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
void commInit(Comm* c, int argc, char** argv)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
MPI_Init(&argc, &argv);
|
|
||||||
MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
|
|
||||||
MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
|
|
||||||
#else
|
|
||||||
c->rank = 0;
|
|
||||||
c->size = 1;
|
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void commPartition(Comm* c, int jmax, int imax)
|
void commPartition(Comm* c, int jmax, int imax)
|
||||||
{
|
{
|
||||||
#if defined(_MPI)
|
#ifdef _MPI
|
||||||
int dims[NDIMS] = { 0, 0 };
|
int dims[NDIMS] = { 0, 0 };
|
||||||
int periods[NDIMS] = { 0, 0 };
|
int periods[NDIMS] = { 0, 0 };
|
||||||
MPI_Dims_create(c->size, NDIMS, dims);
|
MPI_Dims_create(c->size, NDIMS, dims);
|
||||||
@ -337,29 +249,35 @@ void commPartition(Comm* c, int jmax, int imax)
|
|||||||
c->imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
|
c->imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
|
||||||
c->jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
|
c->jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
|
||||||
|
|
||||||
// setup buffer types for communication
|
MPI_Datatype jBufferType;
|
||||||
setupCommunication(c, LEFT, BULK);
|
MPI_Type_contiguous(c->imaxLocal, MPI_DOUBLE, &jBufferType);
|
||||||
setupCommunication(c, LEFT, HALO);
|
MPI_Type_commit(&jBufferType);
|
||||||
setupCommunication(c, RIGHT, BULK);
|
|
||||||
setupCommunication(c, RIGHT, HALO);
|
MPI_Datatype iBufferType;
|
||||||
setupCommunication(c, BOTTOM, BULK);
|
MPI_Type_vector(c->jmaxLocal, 1, c->imaxLocal + 2, MPI_DOUBLE, &iBufferType);
|
||||||
setupCommunication(c, BOTTOM, HALO);
|
MPI_Type_commit(&iBufferType);
|
||||||
setupCommunication(c, TOP, BULK);
|
|
||||||
setupCommunication(c, TOP, HALO);
|
// in the order of the dimensions i->0, j->1
|
||||||
|
// first negative direction, then positive direction
|
||||||
|
size_t dblsize = sizeof(double);
|
||||||
|
int imaxLocal = c->imaxLocal;
|
||||||
|
int jmaxLocal = c->jmaxLocal;
|
||||||
|
c->bufferTypes[LEFT] = iBufferType;
|
||||||
|
c->bufferTypes[RIGHT] = iBufferType;
|
||||||
|
c->bufferTypes[BOTTOM] = jBufferType;
|
||||||
|
c->bufferTypes[TOP] = jBufferType;
|
||||||
|
|
||||||
|
c->sdispls[LEFT] = ((imaxLocal + 2) + 1) * dblsize; // send left
|
||||||
|
c->sdispls[RIGHT] = ((imaxLocal + 2) + imaxLocal) * dblsize; // send right
|
||||||
|
c->sdispls[BOTTOM] = ((imaxLocal + 2) + 1) * dblsize; // send bottom
|
||||||
|
c->sdispls[TOP] = (jmaxLocal * (imaxLocal + 2) + 1) * dblsize; // send top
|
||||||
|
|
||||||
|
c->rdispls[LEFT] = (imaxLocal + 2) * dblsize; // recv left
|
||||||
|
c->rdispls[RIGHT] = ((imaxLocal + 2) + (imaxLocal + 1)) * dblsize; // recv right
|
||||||
|
c->rdispls[BOTTOM] = 1 * dblsize; // recv bottom
|
||||||
|
c->rdispls[TOP] = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * dblsize; // recv top
|
||||||
#else
|
#else
|
||||||
c->imaxLocal = imax;
|
c->imaxLocal = imax;
|
||||||
c->jmaxLocal = jmax;
|
c->jmaxLocal = jmax;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void commFinalize(Comm* c)
|
|
||||||
{
|
|
||||||
#if defined(_MPI)
|
|
||||||
for (int i = 0; i < NDIRS; i++) {
|
|
||||||
MPI_Type_free(&c->sbufferTypes[i]);
|
|
||||||
MPI_Type_free(&c->rbufferTypes[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
MPI_Finalize();
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
123
BasicSolver/2D-mpi/src/comm.c
Normal file
123
BasicSolver/2D-mpi/src/comm.c
Normal file
@ -0,0 +1,123 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
|
* All rights reserved. This file is part of nusif-solver.
|
||||||
|
* Use of this source code is governed by a MIT style
|
||||||
|
* license that can be found in the LICENSE file.
|
||||||
|
*/
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
|
||||||
|
#include "comm.h"
|
||||||
|
|
||||||
|
// subroutines local to this module
|
||||||
|
int sizeOfRank(int rank, int size, int N)
|
||||||
|
{
|
||||||
|
return N / size + ((N % size > rank) ? 1 : 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
void commReduction(double* v, int op)
|
||||||
|
{
|
||||||
|
#ifdef _MPI
|
||||||
|
if (op == MAX) {
|
||||||
|
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
|
||||||
|
} else if (op == SUM) {
|
||||||
|
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
void commPrintConfig(Comm* c)
|
||||||
|
{
|
||||||
|
#ifdef _MPI
|
||||||
|
fflush(stdout);
|
||||||
|
MPI_Barrier(MPI_COMM_WORLD);
|
||||||
|
if (commIsMaster(c)) {
|
||||||
|
printf("Communication setup:\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < c->size; i++) {
|
||||||
|
if (i == c->rank) {
|
||||||
|
printf("\tRank %d of %d\n", c->rank, c->size);
|
||||||
|
printf("\tNeighbours (bottom, top, left, right): %d %d, %d, %d\n",
|
||||||
|
c->neighbours[BOTTOM],
|
||||||
|
c->neighbours[TOP],
|
||||||
|
c->neighbours[LEFT],
|
||||||
|
c->neighbours[RIGHT]);
|
||||||
|
printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]);
|
||||||
|
printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal);
|
||||||
|
fflush(stdout);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
MPI_Barrier(MPI_COMM_WORLD);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
void commInit(Comm* c, int argc, char** argv)
|
||||||
|
{
|
||||||
|
#ifdef _MPI
|
||||||
|
MPI_Init(&argc, &argv);
|
||||||
|
MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
|
||||||
|
MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
|
||||||
|
#else
|
||||||
|
c->rank = 0;
|
||||||
|
c->size = 1;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
void commTestInit(Comm* c, double* p, double* f, double* g)
|
||||||
|
{
|
||||||
|
int imax = c->imaxLocal;
|
||||||
|
int jmax = c->jmaxLocal;
|
||||||
|
int rank = c->rank;
|
||||||
|
|
||||||
|
for (int j = 0; j < jmax + 2; j++) {
|
||||||
|
for (int i = 0; i < imax + 2; i++) {
|
||||||
|
p[j * (imax + 2) + i] = rank;
|
||||||
|
f[j * (imax + 2) + i] = rank;
|
||||||
|
g[j * (imax + 2) + i] = rank;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void testWriteFile(char* filename, double* grid, int imax, int jmax)
|
||||||
|
{
|
||||||
|
FILE* fp = fopen(filename, "w");
|
||||||
|
|
||||||
|
if (fp == NULL) {
|
||||||
|
printf("Error!\n");
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int j = 0; j < jmax + 2; j++) {
|
||||||
|
for (int i = 0; i < imax + 2; i++) {
|
||||||
|
fprintf(fp, "%f ", grid[j * (imax + 2) + i]);
|
||||||
|
}
|
||||||
|
fprintf(fp, "\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
fclose(fp);
|
||||||
|
}
|
||||||
|
|
||||||
|
void commTestWrite(Comm* c, double* p, double* f, double* g)
|
||||||
|
{
|
||||||
|
int imax = c->imaxLocal;
|
||||||
|
int jmax = c->jmaxLocal;
|
||||||
|
int rank = c->rank;
|
||||||
|
|
||||||
|
char filename[30];
|
||||||
|
snprintf(filename, 30, "ptest-%d.dat", rank);
|
||||||
|
testWriteFile(filename, p, imax, jmax);
|
||||||
|
|
||||||
|
snprintf(filename, 30, "ftest-%d.dat", rank);
|
||||||
|
testWriteFile(filename, f, imax, jmax);
|
||||||
|
|
||||||
|
snprintf(filename, 30, "gtest-%d.dat", rank);
|
||||||
|
testWriteFile(filename, g, imax, jmax);
|
||||||
|
}
|
||||||
|
|
||||||
|
void commFinalize(Comm* c)
|
||||||
|
{
|
||||||
|
#ifdef _MPI
|
||||||
|
MPI_Finalize();
|
||||||
|
#endif
|
||||||
|
}
|
@ -11,7 +11,7 @@
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
enum direction { LEFT = 0, RIGHT, BOTTOM, TOP, NDIRS };
|
enum direction { LEFT = 0, RIGHT, BOTTOM, TOP, NDIRS };
|
||||||
enum dimension { JDIM = 0, IDIM, NDIMS };
|
enum dimension { IDIM = 0, JDIM, NDIMS };
|
||||||
enum layer { HALO = 0, BULK };
|
enum layer { HALO = 0, BULK };
|
||||||
enum op { MAX = 0, SUM };
|
enum op { MAX = 0, SUM };
|
||||||
|
|
||||||
@ -20,15 +20,19 @@ typedef struct {
|
|||||||
int size;
|
int size;
|
||||||
#if defined(_MPI)
|
#if defined(_MPI)
|
||||||
MPI_Comm comm;
|
MPI_Comm comm;
|
||||||
MPI_Datatype sbufferTypes[NDIRS];
|
MPI_Datatype bufferTypes[NDIRS];
|
||||||
MPI_Datatype rbufferTypes[NDIRS];
|
MPI_Aint sdispls[NDIRS];
|
||||||
|
MPI_Aint rdispls[NDIRS];
|
||||||
#endif
|
#endif
|
||||||
int neighbours[NDIRS];
|
int neighbours[NDIRS];
|
||||||
int coords[NDIMS], dims[NDIMS];
|
int coords[NDIMS], dims[NDIMS];
|
||||||
int imaxLocal, jmaxLocal;
|
int imaxLocal, jmaxLocal;
|
||||||
} Comm;
|
} Comm;
|
||||||
|
|
||||||
|
extern int sizeOfRank(int rank, int size, int N);
|
||||||
extern void commInit(Comm* c, int argc, char** argv);
|
extern void commInit(Comm* c, int argc, char** argv);
|
||||||
|
extern void commTestInit(Comm* c, double* p, double* f, double* g);
|
||||||
|
extern void commTestWrite(Comm* c, double* p, double* f, double* g);
|
||||||
extern void commFinalize(Comm* c);
|
extern void commFinalize(Comm* c);
|
||||||
extern void commPartition(Comm* c, int jmax, int imax);
|
extern void commPartition(Comm* c, int jmax, int imax);
|
||||||
extern void commPrintConfig(Comm*);
|
extern void commPrintConfig(Comm*);
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
/*
|
/*
|
||||||
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
||||||
* All rights reserved.
|
* All rights reserved.
|
||||||
* Use of this source code is governed by a MIT-style
|
* Use of this source code is governed by a MIT-style
|
||||||
* license that can be found in the LICENSE file.
|
* license that can be found in the LICENSE file.
|
||||||
@ -15,6 +15,26 @@
|
|||||||
#include "solver.h"
|
#include "solver.h"
|
||||||
#include "timing.h"
|
#include "timing.h"
|
||||||
|
|
||||||
|
static void writeResults(Solver* s)
|
||||||
|
{
|
||||||
|
#ifdef _MPI
|
||||||
|
size_t bytesize = (s->imax + 2) * (s->jmax + 2) * sizeof(double);
|
||||||
|
|
||||||
|
double* ug = allocate(64, bytesize);
|
||||||
|
double* vg = allocate(64, bytesize);
|
||||||
|
double* pg = allocate(64, bytesize);
|
||||||
|
|
||||||
|
commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->jmax, s->imax);
|
||||||
|
writeResult(s, ug, vg, pg);
|
||||||
|
|
||||||
|
free(ug);
|
||||||
|
free(vg);
|
||||||
|
free(pg);
|
||||||
|
#else
|
||||||
|
writeResult(s, s->u, s->v, s->p);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char** argv)
|
int main(int argc, char** argv)
|
||||||
{
|
{
|
||||||
int rank;
|
int rank;
|
||||||
@ -35,7 +55,18 @@ int main(int argc, char** argv)
|
|||||||
if (commIsMaster(&s.comm)) {
|
if (commIsMaster(&s.comm)) {
|
||||||
printParameter(&p);
|
printParameter(&p);
|
||||||
}
|
}
|
||||||
|
|
||||||
initSolver(&s, &p);
|
initSolver(&s, &p);
|
||||||
|
#ifdef TEST
|
||||||
|
commPrintConfig(&s.comm);
|
||||||
|
commTestInit(&s.comm, s.p, s.f, s.g);
|
||||||
|
commExchange(&s.comm, s.p);
|
||||||
|
commShift(&s.comm, s.f, s.g);
|
||||||
|
commTestWrite(&s.comm, s.p, s.f, s.g);
|
||||||
|
writeResults(&s);
|
||||||
|
commFinalize(&s.comm);
|
||||||
|
exit(EXIT_SUCCESS);
|
||||||
|
#endif
|
||||||
#ifndef VERBOSE
|
#ifndef VERBOSE
|
||||||
initProgress(s.te);
|
initProgress(s.te);
|
||||||
#endif
|
#endif
|
||||||
@ -70,15 +101,8 @@ int main(int argc, char** argv)
|
|||||||
if (commIsMaster(&s.comm)) {
|
if (commIsMaster(&s.comm)) {
|
||||||
printf("Solution took %.2fs\n", timeStop - timeStart);
|
printf("Solution took %.2fs\n", timeStop - timeStart);
|
||||||
}
|
}
|
||||||
size_t bytesize = s.imax * s.jmax * sizeof(double);
|
|
||||||
|
|
||||||
double* ug = allocate(64, bytesize);
|
|
||||||
double* vg = allocate(64, bytesize);
|
|
||||||
double* pg = allocate(64, bytesize);
|
|
||||||
|
|
||||||
commCollectResult(&s.comm, ug, vg, pg, s.u, s.v, s.p, s.jmax, s.imax);
|
|
||||||
writeResult(&s, ug, vg, pg);
|
|
||||||
|
|
||||||
|
writeResults(&s);
|
||||||
commFinalize(&s.comm);
|
commFinalize(&s.comm);
|
||||||
return EXIT_SUCCESS;
|
return EXIT_SUCCESS;
|
||||||
}
|
}
|
||||||
|
@ -7,54 +7,45 @@
|
|||||||
#include <math.h>
|
#include <math.h>
|
||||||
#include <mpi.h>
|
#include <mpi.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
#include "progress.h"
|
#include "progress.h"
|
||||||
|
|
||||||
static double _end;
|
static double _end;
|
||||||
static int _current;
|
static int _current;
|
||||||
static int _rank = -1;
|
|
||||||
|
|
||||||
void initProgress(double end)
|
void initProgress(double end)
|
||||||
{
|
{
|
||||||
MPI_Comm_rank(MPI_COMM_WORLD, &_rank);
|
|
||||||
_end = end;
|
_end = end;
|
||||||
_current = 0;
|
_current = 0;
|
||||||
|
|
||||||
if (_rank == 0) {
|
printf("[ ]");
|
||||||
printf("[ ]");
|
fflush(stdout);
|
||||||
fflush(stdout);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void printProgress(double current)
|
void printProgress(double current)
|
||||||
{
|
{
|
||||||
if (_rank == 0) {
|
int new = (int)rint((current / _end) * 10.0);
|
||||||
int new = (int)rint((current / _end) * 10.0);
|
|
||||||
|
|
||||||
if (new > _current) {
|
if (new > _current) {
|
||||||
char progress[11];
|
char progress[11];
|
||||||
_current = new;
|
_current = new;
|
||||||
progress[0] = 0;
|
progress[0] = 0;
|
||||||
|
|
||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
if (i < _current) {
|
if (i < _current) {
|
||||||
sprintf(progress + strlen(progress), "#");
|
sprintf(progress + strlen(progress), "#");
|
||||||
} else {
|
} else {
|
||||||
sprintf(progress + strlen(progress), " ");
|
sprintf(progress + strlen(progress), " ");
|
||||||
}
|
|
||||||
}
|
}
|
||||||
printf("\r[%s]", progress);
|
|
||||||
}
|
}
|
||||||
fflush(stdout);
|
printf("\r[%s]", progress);
|
||||||
}
|
}
|
||||||
|
fflush(stdout);
|
||||||
}
|
}
|
||||||
|
|
||||||
void stopProgress()
|
void stopProgress()
|
||||||
{
|
{
|
||||||
if (_rank == 0) {
|
printf("\n");
|
||||||
printf("\n");
|
fflush(stdout);
|
||||||
fflush(stdout);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
@ -511,9 +511,9 @@ void writeResult(Solver* s, double* u, double* v, double* p)
|
|||||||
exit(EXIT_FAILURE);
|
exit(EXIT_FAILURE);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int j = 1; j < jmax; j++) {
|
for (int j = 1; j <= jmax; j++) {
|
||||||
y = (double)(j - 0.5) * dy;
|
y = (double)(j - 0.5) * dy;
|
||||||
for (int i = 1; i < imax; i++) {
|
for (int i = 1; i <= imax; i++) {
|
||||||
x = (double)(i - 0.5) * dx;
|
x = (double)(i - 0.5) * dx;
|
||||||
fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]);
|
fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]);
|
||||||
}
|
}
|
||||||
@ -529,14 +529,14 @@ void writeResult(Solver* s, double* u, double* v, double* p)
|
|||||||
exit(EXIT_FAILURE);
|
exit(EXIT_FAILURE);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int j = 1; j < jmax; j++) {
|
for (int j = 1; j <= jmax; j++) {
|
||||||
y = dy * (j - 0.5);
|
y = dy * (j - 0.5);
|
||||||
for (int i = 1; i < imax; i++) {
|
for (int i = 1; i <= imax; i++) {
|
||||||
x = dx * (i - 0.5);
|
x = dx * (i - 0.5);
|
||||||
double vel_u = (u[j * (imax) + i] + u[j * (imax) + (i - 1)]) / 2.0;
|
double velU = (u[j * (imax + 2) + i] + u[j * (imax + 2) + (i - 1)]) / 2.0;
|
||||||
double vel_v = (v[j * (imax) + i] + v[(j - 1) * (imax) + i]) / 2.0;
|
double velV = (v[j * (imax + 2) + i] + v[(j - 1) * (imax + 2) + i]) / 2.0;
|
||||||
double len = sqrt((vel_u * vel_u) + (vel_v * vel_v));
|
double len = sqrt((velU * velU) + (velV * velV));
|
||||||
fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len);
|
fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, velU, velV, len);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Binary file not shown.
Before Width: | Height: | Size: 9.1 KiB |
@ -47,6 +47,8 @@ clean:
|
|||||||
distclean: clean
|
distclean: clean
|
||||||
$(info ===> DIST CLEAN)
|
$(info ===> DIST CLEAN)
|
||||||
@rm -f $(TARGET)
|
@rm -f $(TARGET)
|
||||||
|
@rm -f *.dat
|
||||||
|
@rm -f *.png
|
||||||
|
|
||||||
info:
|
info:
|
||||||
$(info $(CFLAGS))
|
$(info $(CFLAGS))
|
||||||
|
@ -15,7 +15,7 @@ bcRight 1 #
|
|||||||
gx 0.0 # Body forces (e.g. gravity)
|
gx 0.0 # Body forces (e.g. gravity)
|
||||||
gy 0.0 #
|
gy 0.0 #
|
||||||
|
|
||||||
re 10.0 # Reynolds number
|
re 100.0 # Reynolds number
|
||||||
|
|
||||||
u_init 0.0 # initial value for velocity in x-direction
|
u_init 0.0 # initial value for velocity in x-direction
|
||||||
v_init 0.0 # initial value for velocity in y-direction
|
v_init 0.0 # initial value for velocity in y-direction
|
||||||
|
@ -6,7 +6,6 @@
|
|||||||
*/
|
*/
|
||||||
#include <math.h>
|
#include <math.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
#include "progress.h"
|
#include "progress.h"
|
||||||
|
Loading…
Reference in New Issue
Block a user