moebiusband/NuSiF-Solver
2
0
Fork 2
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merging the new branch

Browse Source
This commit is contained in:
Aditya Ujeniya
2024-07-08 09:52:11 +02:00
parent 28fec03be9
commit d5053b96ea
433 changed files with 5948 additions and 4848038 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
BasicSolver/2D-mpi/src/affinity.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.

2
BasicSolver/2D-mpi/src/affinity.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.

7
BasicSolver/2D-mpi/src/allocate.c
View File

@@ -1,14 +1,17 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
*/
#include <errno.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
void* allocate(int alignment, size_t bytesize)
#include "allocate.h"
void* allocate(size_t alignment, size_t bytesize)
{
int errorCode;
void* ptr;

4
BasicSolver/2D-mpi/src/allocate.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
@@ -8,6 +8,6 @@
#define __ALLOCATE_H_
#include <stdlib.h>
extern void* allocate(int alignment, size_t bytesize);
extern void* allocate(size_t alignment, size_t bytesize);
#endif

181
BasicSolver/2D-mpi/src/comm-v1.c Normal file
View File

@@ -0,0 +1,181 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <stdlib.h>
#include "comm.h"
#ifdef _MPI
// subroutines local to this module
static int sum(int* sizes, int position)
{
int sum = 0;
for (int i = 0; i < position; i++) {
sum += sizes[i];
}
return sum;
}
static void gatherArray(
Comm* c, int cnt, int* rcvCounts, int* displs, double* src, double* dst)
{
double* sendbuffer = src + (c->imaxLocal + 2);
if (c->rank == 0) {
sendbuffer = src;
}
MPI_Gatherv(sendbuffer,
cnt,
MPI_DOUBLE,
dst,
rcvCounts,
displs,
MPI_DOUBLE,
0,
MPI_COMM_WORLD);
}
#endif // defined _MPI
// exported subroutines
int commIsBoundary(Comm* c, int direction)
{
#ifdef _MPI
switch (direction) {
case LEFT:
return 1;
break;
case RIGHT:
return 1;
break;
case BOTTOM:
return c->rank == 0;
break;
case TOP:
return c->rank == (c->size - 1);
break;
}
#endif
return 1;
}
void commExchange(Comm* c, double* grid)
{
#ifdef _MPI
MPI_Request requests[4] = { MPI_REQUEST_NULL,
MPI_REQUEST_NULL,
MPI_REQUEST_NULL,
MPI_REQUEST_NULL };
/* exchange ghost cells with top neighbor */
if (c->rank + 1 < c->size) {
int top = c->rank + 1;
double* src = grid + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
double* dst = grid + (c->jmaxLocal + 1) * (c->imaxLocal + 2) + 1;
MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, top, 1, MPI_COMM_WORLD, &requests[0]);
MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, top, 2, MPI_COMM_WORLD, &requests[1]);
}
/* exchange ghost cells with bottom neighbor */
if (c->rank > 0) {
int bottom = c->rank - 1;
double* src = grid + (c->imaxLocal + 2) + 1;
double* dst = grid + 1;
MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]);
MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]);
}
MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
#endif
}
void commShift(Comm* c, double* f, double* g)
{
#ifdef _MPI
MPI_Request requests[2] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL };
/* shift G */
/* receive ghost cells from bottom neighbor */
if (c->rank > 0) {
int bottom = c->rank - 1;
MPI_Irecv(g + 1,
c->imaxLocal,
MPI_DOUBLE,
bottom,
0,
MPI_COMM_WORLD,
&requests[0]);
}
if (c->rank + 1 < c->size) {
int top = c->rank + 1;
double* buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
/* send ghost cells to top neighbor */
MPI_Isend(buf, c->imaxLocal, MPI_DOUBLE, top, 0, MPI_COMM_WORLD, &requests[1]);
}
MPI_Waitall(2, requests, MPI_STATUSES_IGNORE);
#endif
}
void commCollectResult(Comm* c,
double* ug,
double* vg,
double* pg,
double* u,
double* v,
double* p,
int jmax,
int imax)
{
#ifdef _MPI
int *rcvCounts, *displs;
int cnt = c->jmaxLocal * (imax + 2);
if (c->rank == 0) {
rcvCounts = (int*)malloc(c->size * sizeof(int));
displs = (int*)malloc(c->size * sizeof(int));
}
if (c->rank == 0 && c->size == 1) {
cnt = (c->jmaxLocal + 2) * (imax + 2);
} else if (c->rank == 0 || c->rank == (c->size - 1)) {
cnt = (c->jmaxLocal + 1) * (imax + 2);
}
MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD);
if (c->rank == 0) {
displs[0] = 0;
int cursor = rcvCounts[0];
for (int i = 1; i < c->size; i++) {
displs[i] = cursor;
cursor += rcvCounts[i];
}
}
gatherArray(c, cnt, rcvCounts, displs, p, pg);
gatherArray(c, cnt, rcvCounts, displs, u, ug);
gatherArray(c, cnt, rcvCounts, displs, v, vg);
#endif
}
void commPartition(Comm* c, int jmax, int imax)
{
#ifdef _MPI
c->imaxLocal = imax;
c->jmaxLocal = sizeOfRank(c->rank, c->size, jmax);
#else
c->imaxLocal = imax;
c->jmaxLocal = jmax;
#endif
}

287
BasicSolver/2D-mpi/src/comm-v2.c Normal file
View File

@@ -0,0 +1,287 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <stdio.h>
#include <stdlib.h>
#include "comm.h"
#ifdef _MPI
// subroutines local to this module
static int sum(int* sizes, int position)
{
int sum = 0;
for (int i = 0; i < position; i++) {
sum += sizes[i];
}
return sum;
}
static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax)
{
MPI_Request* requests;
int numRequests = 1;
if (c->rank == 0) {
numRequests = c->size + 1;
} else {
numRequests = 1;
}
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
/* all ranks send their bulk array, including the external boundary layer */
MPI_Datatype bulkType;
int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
int starts[NDIMS] = { 1, 1 };
if (commIsBoundary(c, LEFT)) {
newSizes[CIDIM] += 1;
starts[CIDIM] = 0;
}
if (commIsBoundary(c, RIGHT)) {
newSizes[CIDIM] += 1;
}
if (commIsBoundary(c, BOTTOM)) {
newSizes[CJDIM] += 1;
starts[CJDIM] = 0;
}
if (commIsBoundary(c, TOP)) {
newSizes[CJDIM] += 1;
}
MPI_Type_create_subarray(NDIMS,
oldSizes,
newSizes,
starts,
MPI_ORDER_C,
MPI_DOUBLE,
&bulkType);
MPI_Type_commit(&bulkType);
MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
int newSizesI[c->size];
int newSizesJ[c->size];
MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD);
/* rank 0 assembles the subdomains */
if (c->rank == 0) {
for (int i = 0; i < c->size; i++) {
MPI_Datatype domainType;
int oldSizes[NDIMS] = { jmax + 2, imax + 2 };
int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
int coords[NDIMS];
MPI_Cart_coords(c->comm, i, NDIMS, coords);
int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
sum(newSizesI, coords[IDIM]) };
printf(
"Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
"Starts(i,j): %d %d\n",
i,
coords[IDIM],
coords[JDIM],
oldSizes[CIDIM],
oldSizes[CJDIM],
newSizes[CIDIM],
newSizes[CJDIM],
starts[CIDIM],
starts[CJDIM]);
MPI_Type_create_subarray(NDIMS,
oldSizes,
newSizes,
starts,
MPI_ORDER_C,
MPI_DOUBLE,
&domainType);
MPI_Type_commit(&domainType);
MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]);
MPI_Type_free(&domainType);
}
}
MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
}
#endif // defined _MPI
// exported subroutines
int commIsBoundary(Comm* c, int direction)
{
#ifdef _MPI
switch (direction) {
case LEFT:
return c->coords[IDIM] == 0;
break;
case RIGHT:
return c->coords[IDIM] == (c->dims[IDIM] - 1);
break;
case BOTTOM:
return c->coords[JDIM] == 0;
break;
case TOP:
return c->coords[JDIM] == (c->dims[JDIM] - 1);
break;
}
#endif
return 1;
}
void commExchange(Comm* c, double* grid)
{
#ifdef _MPI
MPI_Request requests[8];
for (int i = 0; i < 8; i++)
requests[i] = MPI_REQUEST_NULL;
for (int i = 0; i < NDIRS; i++) {
double* sbuf = grid + c->sdispls[i];
double* rbuf = grid + c->rdispls[i];
int tag = 0;
if (c->neighbours[i] != MPI_PROC_NULL) {
// printf("DEBUG: Rank %d - SendRecv with %d\n", c->rank, c->neighbours[i]);
tag = c->neighbours[i];
}
MPI_Irecv(rbuf,
1,
c->bufferTypes[i],
c->neighbours[i],
tag,
c->comm,
&requests[i * 2]);
MPI_Isend(sbuf,
1,
c->bufferTypes[i],
c->neighbours[i],
c->rank,
c->comm,
&requests[i * 2 + 1]);
}
MPI_Waitall(8, requests, MPI_STATUSES_IGNORE);
#endif
}
void commShift(Comm* c, double* f, double* g)
{
#ifdef _MPI
MPI_Request requests[4] = { MPI_REQUEST_NULL,
MPI_REQUEST_NULL,
MPI_REQUEST_NULL,
MPI_REQUEST_NULL };
/* shift G */
/* receive ghost cells from bottom neighbor */
double* buf = g + 1;
MPI_Irecv(buf,
1,
c->bufferTypes[BOTTOM],
c->neighbours[BOTTOM],
0,
c->comm,
&requests[0]);
/* send ghost cells to top neighbor */
buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
MPI_Isend(buf, 1, c->bufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]);
/* shift F */
/* receive ghost cells from left neighbor */
buf = f + (c->imaxLocal + 2);
MPI_Irecv(buf,
1,
c->bufferTypes[LEFT],
c->neighbours[LEFT],
1,
c->comm,
&requests[2]);
/* send ghost cells to right neighbor */
buf = f + (c->imaxLocal + 2) + (c->imaxLocal);
MPI_Isend(buf,
1,
c->bufferTypes[RIGHT],
c->neighbours[RIGHT],
1,
c->comm,
&requests[3]);
MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
#endif
}
void commCollectResult(Comm* c,
double* ug,
double* vg,
double* pg,
double* u,
double* v,
double* p,
int imax,
int jmax)
{
#ifdef _MPI
/* collect P */
assembleResult(c, p, pg, imax, jmax);
/* collect U */
assembleResult(c, u, ug, imax, jmax);
/* collect V */
assembleResult(c, v, vg, imax, jmax);
#endif
}
void commPartition(Comm* c, int jmax, int imax)
{
#ifdef _MPI
int dims[NDIMS] = { 0, 0 };
int periods[NDIMS] = { 0, 0 };
MPI_Dims_create(c->size, NDIMS, dims);
MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm);
MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]);
MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]);
MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
int imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
int jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
c->imaxLocal = imaxLocal;
c->jmaxLocal = jmaxLocal;
MPI_Datatype jBufferType;
MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
MPI_Type_commit(&jBufferType);
MPI_Datatype iBufferType;
MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
MPI_Type_commit(&iBufferType);
c->bufferTypes[LEFT] = iBufferType;
c->bufferTypes[RIGHT] = iBufferType;
c->bufferTypes[BOTTOM] = jBufferType;
c->bufferTypes[TOP] = jBufferType;
c->sdispls[LEFT] = (imaxLocal + 2) + 1;
c->sdispls[RIGHT] = (imaxLocal + 2) + imaxLocal;
c->sdispls[BOTTOM] = (imaxLocal + 2) + 1;
c->sdispls[TOP] = jmaxLocal * (imaxLocal + 2) + 1;
c->rdispls[LEFT] = (imaxLocal + 2);
c->rdispls[RIGHT] = (imaxLocal + 2) + (imaxLocal + 1);
c->rdispls[BOTTOM] = 1;
c->rdispls[TOP] = (jmaxLocal + 1) * (imaxLocal + 2) + 1;
#else
c->imaxLocal = imax;
c->jmaxLocal = jmax;
#endif
}

269
BasicSolver/2D-mpi/src/comm-v3.c Normal file
View File

@@ -0,0 +1,269 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <stdio.h>
#include <stdlib.h>
#include "comm.h"
#ifdef _MPI
// subroutines local to this module
static int sum(int* sizes, int position)
{
int sum = 0;
for (int i = 0; i < position; i++) {
sum += sizes[i];
}
return sum;
}
static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax)
{
MPI_Request* requests;
int numRequests = 1;
if (c->rank == 0) {
numRequests = c->size + 1;
} else {
numRequests = 1;
}
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
/* all ranks send their bulk array, including the external boundary layer */
MPI_Datatype bulkType;
int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
int starts[NDIMS] = { 1, 1 };
if (commIsBoundary(c, LEFT)) {
newSizes[CIDIM] += 1;
starts[CIDIM] = 0;
}
if (commIsBoundary(c, RIGHT)) {
newSizes[CIDIM] += 1;
}
if (commIsBoundary(c, BOTTOM)) {
newSizes[CJDIM] += 1;
starts[CJDIM] = 0;
}
if (commIsBoundary(c, TOP)) {
newSizes[CJDIM] += 1;
}
MPI_Type_create_subarray(NDIMS,
oldSizes,
newSizes,
starts,
MPI_ORDER_C,
MPI_DOUBLE,
&bulkType);
MPI_Type_commit(&bulkType);
MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
int newSizesI[c->size];
int newSizesJ[c->size];
MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD);
/* rank 0 assembles the subdomains */
if (c->rank == 0) {
for (int i = 0; i < c->size; i++) {
MPI_Datatype domainType;
int oldSizes[NDIMS] = { jmax + 2, imax + 2 };
int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] };
int coords[NDIMS];
MPI_Cart_coords(c->comm, i, NDIMS, coords);
int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]),
sum(newSizesI, coords[IDIM]) };
printf(
"Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d "
"Starts(i,j): %d %d\n",
i,
coords[IDIM],
coords[JDIM],
oldSizes[CIDIM],
oldSizes[CJDIM],
newSizes[CIDIM],
newSizes[CJDIM],
starts[CIDIM],
starts[CJDIM]);
MPI_Type_create_subarray(NDIMS,
oldSizes,
newSizes,
starts,
MPI_ORDER_C,
MPI_DOUBLE,
&domainType);
MPI_Type_commit(&domainType);
MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]);
MPI_Type_free(&domainType);
}
}
MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
}
#endif // defined _MPI
// exported subroutines
int commIsBoundary(Comm* c, int direction)
{
#ifdef _MPI
switch (direction) {
case LEFT:
return c->coords[IDIM] == 0;
break;
case RIGHT:
return c->coords[IDIM] == (c->dims[IDIM] - 1);
break;
case BOTTOM:
return c->coords[JDIM] == 0;
break;
case TOP:
return c->coords[JDIM] == (c->dims[JDIM] - 1);
break;
}
#endif
return 1;
}
void commExchange(Comm* c, double* grid)
{
#ifdef _MPI
int counts[NDIRS] = { 1, 1, 1, 1 };
MPI_Neighbor_alltoallw(grid,
counts,
c->sdispls,
c->bufferTypes,
grid,
counts,
c->rdispls,
c->bufferTypes,
c->comm);
#endif
}
void commShift(Comm* c, double* f, double* g)
{
#ifdef _MPI
MPI_Request requests[4] = { MPI_REQUEST_NULL,
MPI_REQUEST_NULL,
MPI_REQUEST_NULL,
MPI_REQUEST_NULL };
/* shift G */
/* receive ghost cells from bottom neighbor */
double* buf = g + 1;
MPI_Irecv(buf,
1,
c->bufferTypes[BOTTOM],
c->neighbours[BOTTOM],
0,
c->comm,
&requests[0]);
/* send ghost cells to top neighbor */
buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1;
MPI_Isend(buf, 1, c->bufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]);
/* shift F */
/* receive ghost cells from left neighbor */
buf = f + (c->imaxLocal + 2);
MPI_Irecv(buf,
1,
c->bufferTypes[LEFT],
c->neighbours[LEFT],
1,
c->comm,
&requests[2]);
/* send ghost cells to right neighbor */
buf = f + (c->imaxLocal + 2) + (c->imaxLocal);
MPI_Isend(buf,
1,
c->bufferTypes[RIGHT],
c->neighbours[RIGHT],
1,
c->comm,
&requests[3]);
MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
#endif
}
void commCollectResult(Comm* c,
double* ug,
double* vg,
double* pg,
double* u,
double* v,
double* p,
int imax,
int jmax)
{
#ifdef _MPI
/* collect P */
assembleResult(c, p, pg, imax, jmax);
/* collect U */
assembleResult(c, u, ug, imax, jmax);
/* collect V */
assembleResult(c, v, vg, imax, jmax);
#endif
}
void commPartition(Comm* c, int jmax, int imax)
{
#ifdef _MPI
int dims[NDIMS] = { 0, 0 };
int periods[NDIMS] = { 0, 0 };
MPI_Dims_create(c->size, NDIMS, dims);
MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm);
MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]);
MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]);
MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
int imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
int jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
c->imaxLocal = imaxLocal;
c->jmaxLocal = jmaxLocal;
MPI_Datatype jBufferType;
MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType);
MPI_Type_commit(&jBufferType);
MPI_Datatype iBufferType;
MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType);
MPI_Type_commit(&iBufferType);
c->bufferTypes[LEFT] = iBufferType;
c->bufferTypes[RIGHT] = iBufferType;
c->bufferTypes[BOTTOM] = jBufferType;
c->bufferTypes[TOP] = jBufferType;
size_t dblsize = sizeof(double);
c->sdispls[LEFT] = ((imaxLocal + 2) + 1) * dblsize;
c->sdispls[RIGHT] = ((imaxLocal + 2) + imaxLocal) * dblsize;
c->sdispls[BOTTOM] = ((imaxLocal + 2) + 1) * dblsize;
c->sdispls[TOP] = (jmaxLocal * (imaxLocal + 2) + 1) * dblsize;
c->rdispls[LEFT] = (imaxLocal + 2) * dblsize;
c->rdispls[RIGHT] = ((imaxLocal + 2) + (imaxLocal + 1)) * dblsize;
c->rdispls[BOTTOM] = 1 * dblsize;
c->rdispls[TOP] = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * dblsize;
#else
c->imaxLocal = imax;
c->jmaxLocal = jmax;
#endif
}

357
BasicSolver/2D-mpi/src/comm.c
View File

@@ -1,281 +1,34 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include "comm.h"
// subroutines local to this module
static int sizeOfRank(int rank, int size, int N)
int sizeOfRank(int rank, int size, int N)
{
return N / size + ((N % size > rank) ? 1 : 0);
}
static void setupCommunication(Comm* c, int direction, int layer)
{
MPI_Datatype type;
size_t dblsize = sizeof(double);
int imaxLocal = c->imaxLocal;
int jmaxLocal = c->jmaxLocal;
int sizes[NDIMS];
int subSizes[NDIMS];
int starts[NDIMS];
int offset = 0;
sizes[IDIM] = imaxLocal + 2;
sizes[JDIM] = jmaxLocal + 2;
if (layer == HALO) {
offset = 1;
}
switch (direction) {
case LEFT:
subSizes[IDIM] = 1;
subSizes[JDIM] = jmaxLocal;
starts[IDIM] = 1 - offset;
starts[JDIM] = 1;
break;
case RIGHT:
subSizes[IDIM] = 1;
subSizes[JDIM] = jmaxLocal;
starts[IDIM] = imaxLocal + offset;
starts[JDIM] = 1;
break;
case BOTTOM:
subSizes[IDIM] = imaxLocal;
subSizes[JDIM] = 1;
starts[IDIM] = 1;
starts[JDIM] = 1 - offset;
break;
case TOP:
subSizes[IDIM] = imaxLocal;
subSizes[JDIM] = 1;
starts[IDIM] = 1;
starts[JDIM] = jmaxLocal + offset;
break;
}
MPI_Type_create_subarray(NDIMS,
sizes,
subSizes,
starts,
MPI_ORDER_C,
MPI_DOUBLE,
&type);
MPI_Type_commit(&type);
if (layer == HALO) {
c->rbufferTypes[direction] = type;
} else if (layer == BULK) {
c->sbufferTypes[direction] = type;
}
}
static void assembleResult(Comm* c,
double* src,
double* dst,
int imaxLocal[],
int jmaxLocal[],
int offset[],
int jmax,
int imax)
{
MPI_Request* requests;
int numRequests = 1;
if (c->rank == 0) {
numRequests = c->size + 1;
} else {
numRequests = 1;
}
requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request));
/* all ranks send their bulk array */
MPI_Datatype bulkType;
int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 };
int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal };
int starts[NDIMS] = { 1, 1 };
MPI_Type_create_subarray(NDIMS,
oldSizes,
newSizes,
starts,
MPI_ORDER_C,
MPI_DOUBLE,
&bulkType);
MPI_Type_commit(&bulkType);
MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]);
/* rank 0 assembles the subdomains */
if (c->rank == 0) {
for (int i = 0; i < c->size; i++) {
MPI_Datatype domainType;
int oldSizes[NDIMS] = { jmax, imax };
int newSizes[NDIMS] = { jmaxLocal[i], imaxLocal[i] };
int starts[NDIMS] = { offset[i * NDIMS + JDIM], offset[i * NDIMS + IDIM] };
MPI_Type_create_subarray(NDIMS,
oldSizes,
newSizes,
starts,
MPI_ORDER_C,
MPI_DOUBLE,
&domainType);
MPI_Type_commit(&domainType);
MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]);
}
}
MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE);
}
static int sum(int* sizes, int position)
{
int sum = 0;
for (int i = 0; i < position; i++) {
sum += sizes[i];
}
return sum;
}
// exported subroutines
void commReduction(double* v, int op)
{
#ifdef _MPI
if (op == MAX) {
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
} else if (op == SUM) {
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
}
}
int commIsBoundary(Comm* c, int direction)
{
switch (direction) {
case LEFT:
return c->coords[IDIM] == 0;
break;
case RIGHT:
return c->coords[IDIM] == (c->dims[IDIM] - 1);
break;
case BOTTOM:
return c->coords[JDIM] == 0;
break;
case TOP:
return c->coords[JDIM] == (c->dims[JDIM] - 1);
break;
}
return 0;
}
void commExchange(Comm* c, double* grid)
{
int counts[NDIRS] = { 1, 1, 1, 1 };
MPI_Aint displs[NDIRS] = { 0, 0, 0, 0 };
MPI_Neighbor_alltoallw(grid,
counts,
displs,
c->sbufferTypes,
grid,
counts,
displs,
c->rbufferTypes,
c->comm);
}
void commShift(Comm* c, double* f, double* g)
{
MPI_Request requests[4] = { MPI_REQUEST_NULL,
MPI_REQUEST_NULL,
MPI_REQUEST_NULL,
MPI_REQUEST_NULL };
/* shift G */
/* receive ghost cells from bottom neighbor */
MPI_Irecv(g,
1,
c->rbufferTypes[BOTTOM],
c->neighbours[BOTTOM],
0,
c->comm,
&requests[0]);
/* send ghost cells to top neighbor */
MPI_Isend(g, 1, c->sbufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]);
/* shift F */
/* receive ghost cells from left neighbor */
MPI_Irecv(f, 1, c->rbufferTypes[LEFT], c->neighbours[LEFT], 1, c->comm, &requests[2]);
/* send ghost cells to right neighbor */
MPI_Isend(f,
1,
c->sbufferTypes[RIGHT],
c->neighbours[RIGHT],
1,
c->comm,
&requests[3]);
MPI_Waitall(4, requests, MPI_STATUSES_IGNORE);
}
void commCollectResult(Comm* c,
double* ug,
double* vg,
double* pg,
double* u,
double* v,
double* p,
int jmax,
int imax)
{
int offset[c->size * NDIMS];
int imaxLocal[c->size];
int jmaxLocal[c->size];
MPI_Gather(&c->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Gather(&c->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (c->rank == 0) {
for (int i = 0; i < c->size; i++) {
int coords[NDIMS];
MPI_Cart_coords(c->comm, i, NDIMS, coords);
offset[i * NDIMS + IDIM] = sum(imaxLocal, coords[IDIM]);
offset[i * NDIMS + JDIM] = sum(jmaxLocal, coords[JDIM]);
printf("Rank: %d, Coords(j,i): %d %d, Size(j,i): %d %d "
"Offset(j,i): %d %d\n",
i,
coords[JDIM],
coords[IDIM],
jmaxLocal[i],
imaxLocal[i],
offset[i * NDIMS + JDIM],
offset[i * NDIMS + IDIM]);
}
}
/* collect P */
assembleResult(c, p, pg, imaxLocal, jmaxLocal, offset, jmax, imax);
/* collect U */
assembleResult(c, u, ug, imaxLocal, jmaxLocal, offset, jmax, imax);
/* collect V */
assembleResult(c, v, vg, imaxLocal, jmaxLocal, offset, jmax, imax);
#endif
}
void commPrintConfig(Comm* c)
{
#ifdef _MPI
fflush(stdout);
MPI_Barrier(MPI_COMM_WORLD);
if (commIsMaster(c)) {
@@ -290,37 +43,87 @@ void commPrintConfig(Comm* c)
c->neighbours[TOP],
c->neighbours[LEFT],
c->neighbours[RIGHT]);
printf("\tCoordinates (j,i) %d %d\n", c->coords[JDIM], c->coords[IDIM]);
printf("\tLocal domain size (j,i) %dx%d\n", c->jmaxLocal, c->imaxLocal);
printf("\tIs boundary:\n");
printf("\t\tLEFT: %d\n", commIsBoundary(c, LEFT));
printf("\t\tRIGHT: %d\n", commIsBoundary(c, RIGHT));
printf("\t\tBOTTOM: %d\n", commIsBoundary(c, BOTTOM));
printf("\t\tTOP: %d\n", commIsBoundary(c, TOP));
printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]);
printf("\tDims (i,j) %d %d\n", c->dims[IDIM], c->dims[JDIM]);
printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal);
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
}
MPI_Barrier(MPI_COMM_WORLD);
#endif
}
void commInit(Comm* c, int jmax, int imax)
void commInit(Comm* c, int argc, char** argv)
{
/* setup communication */
#ifdef _MPI
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
int dims[NDIMS] = { 0, 0 };
int periods[NDIMS] = { 0, 0 };
MPI_Dims_create(c->size, NDIMS, dims);
MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm);
MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]);
MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]);
MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords);
c->imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax);
c->jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax);
// setup buffer types for communication
setupCommunication(c, LEFT, BULK);
setupCommunication(c, LEFT, HALO);
setupCommunication(c, RIGHT, BULK);
setupCommunication(c, RIGHT, HALO);
setupCommunication(c, BOTTOM, BULK);
setupCommunication(c, BOTTOM, HALO);
setupCommunication(c, TOP, BULK);
setupCommunication(c, TOP, HALO);
#else
c->rank = 0;
c->size = 1;
#endif
}
void commTestInit(Comm* c, double* p, double* f, double* g)
{
int imax = c->imaxLocal;
int jmax = c->jmaxLocal;
int rank = c->rank;
for (int j = 0; j < jmax + 2; j++) {
for (int i = 0; i < imax + 2; i++) {
p[j * (imax + 2) + i] = rank;
f[j * (imax + 2) + i] = rank;
g[j * (imax + 2) + i] = rank;
}
}
}
static void testWriteFile(char* filename, double* grid, int imax, int jmax)
{
FILE* fp = fopen(filename, "w");
if (fp == NULL) {
printf("Error!\n");
exit(EXIT_FAILURE);
}
for (int j = 0; j < jmax + 2; j++) {
for (int i = 0; i < imax + 2; i++) {
fprintf(fp, "%.2f ", grid[j * (imax + 2) + i]);
}
fprintf(fp, "\n");
}
fclose(fp);
}
void commTestWrite(Comm* c, double* p, double* f, double* g)
{
int imax = c->imaxLocal;
int jmax = c->jmaxLocal;
int rank = c->rank;
char filename[30];
snprintf(filename, 30, "ptest-%d.dat", rank);
testWriteFile(filename, p, imax, jmax);
snprintf(filename, 30, "ftest-%d.dat", rank);
testWriteFile(filename, f, imax, jmax);
snprintf(filename, 30, "gtest-%d.dat", rank);
testWriteFile(filename, g, imax, jmax);
}
void commFinalize(Comm* c)
{
#ifdef _MPI
MPI_Finalize();
#endif
}

21
BasicSolver/2D-mpi/src/comm.h
View File

@@ -1,30 +1,41 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#ifndef __COMM_H_
#define __COMM_H_
#if defined(_MPI)
#include <mpi.h>
#endif
enum direction { LEFT = 0, RIGHT, BOTTOM, TOP, NDIRS };
enum dimension { JDIM = 0, IDIM, NDIMS };
enum dimension { IDIM = 0, JDIM, NDIMS };
enum cdimension { CJDIM = 0, CIDIM };
enum layer { HALO = 0, BULK };
enum op { MAX = 0, SUM };
typedef struct {
int rank;
int size;
#if defined(_MPI)
MPI_Comm comm;
MPI_Datatype sbufferTypes[NDIRS];
MPI_Datatype rbufferTypes[NDIRS];
MPI_Datatype bufferTypes[NDIRS];
MPI_Aint sdispls[NDIRS];
MPI_Aint rdispls[NDIRS];
#endif
int neighbours[NDIRS];
int coords[NDIMS], dims[NDIMS];
int imaxLocal, jmaxLocal;
} Comm;
extern void commInit(Comm* c, int jmax, int imax);
extern int sizeOfRank(int rank, int size, int N);
extern void commInit(Comm* c, int argc, char** argv);
extern void commTestInit(Comm* c, double* p, double* f, double* g);
extern void commTestWrite(Comm* c, double* p, double* f, double* g);
extern void commFinalize(Comm* c);
extern void commPartition(Comm* c, int jmax, int imax);
extern void commPrintConfig(Comm*);
extern void commExchange(Comm*, double*);
extern void commShift(Comm* c, double* f, double* g);

453
BasicSolver/2D-mpi/src/discretization.c Normal file
View File

@@ -0,0 +1,453 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <float.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "allocate.h"
#include "comm.h"
#include "discretization.h"
#include "parameter.h"
#include "util.h"
static void printConfig(Discretization* s)
{
if (commIsMaster(&s->comm)) {
printf("Parameters for #%s#\n", s->problem);
printf("BC Left:%d Right:%d Bottom:%d Top:%d\n",
s->bcLeft,
s->bcRight,
s->bcBottom,
s->bcTop);
printf("\tReynolds number: %.2f\n", s->re);
printf("\tGx Gy: %.2f %.2f\n", s->gx, s->gy);
printf("Geometry data:\n");
printf("\tDomain box size (x, y): %.2f, %.2f\n",
s->grid.xlength,
s->grid.ylength);
printf("\tCells (x, y): %d, %d\n", s->grid.imax, s->grid.jmax);
printf("\tCell size (dx, dy): %f, %f\n", s->grid.dx, s->grid.dy);
printf("Timestep parameters:\n");
printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te);
printf("\tdt bound: %.6f\n", s->dtBound);
printf("\tTau factor: %.2f\n", s->tau);
printf("Iterative s parameters:\n");
printf("\tgamma factor: %f\n", s->gamma);
}
commPrintConfig(&s->comm);
}
void initDiscretiztion(Discretization* s, Parameter* params)
{
s->problem = params->name;
s->bcLeft = params->bcLeft;
s->bcRight = params->bcRight;
s->bcBottom = params->bcBottom;
s->bcTop = params->bcTop;
s->grid.imax = params->imax;
s->grid.jmax = params->jmax;
s->grid.xlength = params->xlength;
s->grid.ylength = params->ylength;
s->grid.dx = params->xlength / params->imax;
s->grid.dy = params->ylength / params->jmax;
s->re = params->re;
s->gx = params->gx;
s->gy = params->gy;
s->dt = params->dt;
s->te = params->te;
s->tau = params->tau;
s->gamma = params->gamma;
/* allocate arrays */
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
size_t size = (imaxLocal + 2) * (jmaxLocal + 2);
s->u = allocate(64, size * sizeof(double));
s->v = allocate(64, size * sizeof(double));
s->p = allocate(64, size * sizeof(double));
s->rhs = allocate(64, size * sizeof(double));
s->f = allocate(64, size * sizeof(double));
s->g = allocate(64, size * sizeof(double));
for (int i = 0; i < size; i++) {
s->u[i] = params->u_init;
s->v[i] = params->v_init;
s->p[i] = params->p_init;
s->rhs[i] = 0.0;
s->f[i] = 0.0;
s->g[i] = 0.0;
}
double dx = s->grid.dx;
double dy = s->grid.dy;
double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy);
s->dtBound = 0.5 * s->re * 1.0 / invSqrSum;
#ifdef VERBOSE
printConfig(s);
#endif
}
void computeRHS(Discretization* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double idx = 1.0 / s->grid.dx;
double idy = 1.0 / s->grid.dy;
double idt = 1.0 / s->dt;
double* rhs = s->rhs;
double* f = s->f;
double* g = s->g;
commShift(&s->comm, f, g);
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) *
idt;
}
}
}
static double maxElement(Discretization* s, double* m)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
int size = (imaxLocal + 2) * (jmaxLocal + 2);
double maxval = DBL_MIN;
for (int i = 0; i < size; i++) {
maxval = MAX(maxval, fabs(m[i]));
}
commReduction(&maxval, MAX);
return maxval;
}
void computeTimestep(Discretization* s)
{
double dt = s->dtBound;
double dx = s->grid.dx;
double dy = s->grid.dy;
double umax = maxElement(s, s->u);
double vmax = maxElement(s, s->v);
if (umax > 0) {
dt = (dt > dx / umax) ? dx / umax : dt;
}
if (vmax > 0) {
dt = (dt > dy / vmax) ? dy / vmax : dt;
}
s->dt = dt * s->tau;
}
void setBoundaryConditions(Discretization* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double* u = s->u;
double* v = s->v;
if (commIsBoundary(&s->comm, TOP)) {
switch (s->bcTop) {
case NOSLIP:
for (int i = 1; i < imaxLocal + 1; i++) {
V(i, jmaxLocal) = 0.0;
U(i, jmaxLocal + 1) = -U(i, jmaxLocal);
}
break;
case SLIP:
for (int i = 1; i < imaxLocal + 1; i++) {
V(i, jmaxLocal) = 0.0;
U(i, jmaxLocal + 1) = U(i, jmaxLocal);
}
break;
case OUTFLOW:
for (int i = 1; i < imaxLocal + 1; i++) {
U(i, jmaxLocal + 1) = U(i, jmaxLocal);
V(i, jmaxLocal) = V(i, jmaxLocal - 1);
}
break;
case PERIODIC:
break;
}
}
if (commIsBoundary(&s->comm, BOTTOM)) {
switch (s->bcBottom) {
case NOSLIP:
for (int i = 1; i < imaxLocal + 1; i++) {
V(i, 0) = 0.0;
U(i, 0) = -U(i, 1);
}
break;
case SLIP:
for (int i = 1; i < imaxLocal + 1; i++) {
V(i, 0) = 0.0;
U(i, 0) = U(i, 1);
}
break;
case OUTFLOW:
for (int i = 1; i < imaxLocal + 1; i++) {
U(i, 0) = U(i, 1);
V(i, 0) = V(i, 1);
}
break;
case PERIODIC:
break;
}
}
if (commIsBoundary(&s->comm, RIGHT)) {
switch (s->bcRight) {
case NOSLIP:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(imaxLocal, j) = 0.0;
V(imaxLocal + 1, j) = -V(imaxLocal, j);
}
break;
case SLIP:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(imaxLocal, j) = 0.0;
V(imaxLocal + 1, j) = V(imaxLocal, j);
}
break;
case OUTFLOW:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(imaxLocal, j) = U(imaxLocal - 1, j);
V(imaxLocal + 1, j) = V(imaxLocal, j);
}
break;
case PERIODIC:
break;
}
}
if (commIsBoundary(&s->comm, LEFT)) {
switch (s->bcLeft) {
case NOSLIP:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(0, j) = 0.0;
V(0, j) = -V(1, j);
}
break;
case SLIP:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(0, j) = 0.0;
V(0, j) = V(1, j);
}
break;
case OUTFLOW:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(0, j) = U(1, j);
V(0, j) = V(1, j);
}
break;
case PERIODIC:
break;
}
}
}
void setSpecialBoundaryCondition(Discretization* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double* u = s->u;
if (strcmp(s->problem, "dcavity") == 0) {
if (commIsBoundary(&s->comm, TOP)) {
for (int i = 1; i < imaxLocal + 1; i++) {
U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal);
}
}
} else if (strcmp(s->problem, "canal") == 0) {
if (commIsBoundary(&s->comm, LEFT)) {
double ylength = s->grid.ylength;
double dy = s->grid.dy;
int rest = s->grid.jmax % s->comm.size;
int yc = s->comm.rank * (s->grid.jmax / s->comm.size) +
MIN(rest, s->comm.rank);
double ys = dy * (yc + 0.5);
double y;
/* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */
for (int j = 1; j < jmaxLocal + 1; j++) {
y = ys + dy * (j - 0.5);
U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength);
}
}
}
/* print(solver, solver->u); */
}
void computeFG(Discretization* s)
{
double* u = s->u;
double* v = s->v;
double* f = s->f;
double* g = s->g;
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double gx = s->gx;
double gy = s->gy;
double gamma = s->gamma;
double dt = s->dt;
double inverseRe = 1.0 / s->re;
double inverseDx = 1.0 / s->grid.dx;
double inverseDy = 1.0 / s->grid.dy;
double du2dx, dv2dy, duvdx, duvdy;
double du2dx2, du2dy2, dv2dx2, dv2dy2;
commExchange(&s->comm, u);
commExchange(&s->comm, v);
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
du2dx = inverseDx * 0.25 *
((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) -
(U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) +
gamma * inverseDx * 0.25 *
(fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) +
fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j)));
duvdy = inverseDy * 0.25 *
((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) -
(V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) +
gamma * inverseDy * 0.25 *
(fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) +
fabs(V(i, j - 1) + V(i + 1, j - 1)) *
(U(i, j) - U(i, j - 1)));
du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j));
du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1));
F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx);
duvdx = inverseDx * 0.25 *
((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) -
(U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) +
gamma * inverseDx * 0.25 *
(fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) +
fabs(U(i - 1, j) + U(i - 1, j + 1)) *
(V(i, j) - V(i - 1, j)));
dv2dy = inverseDy * 0.25 *
((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) -
(V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) +
gamma * inverseDy * 0.25 *
(fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) +
fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1)));
dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j));
dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1));
G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy);
}
}
/* ----------------------------- boundary of F --------------------------- */
if (commIsBoundary(&s->comm, LEFT)) {
for (int j = 1; j < jmaxLocal + 1; j++) {
F(0, j) = U(0, j);
}
}
if (commIsBoundary(&s->comm, RIGHT)) {
for (int j = 1; j < jmaxLocal + 1; j++) {
F(imaxLocal, j) = U(imaxLocal, j);
}
}
/* ----------------------------- boundary of G --------------------------- */
if (commIsBoundary(&s->comm, BOTTOM)) {
for (int i = 1; i < imaxLocal + 1; i++) {
G(i, 0) = V(i, 0);
}
}
if (commIsBoundary(&s->comm, TOP)) {
for (int i = 1; i < imaxLocal + 1; i++) {
G(i, jmaxLocal) = V(i, jmaxLocal);
}
}
}
void adaptUV(Discretization* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double* p = s->p;
double* u = s->u;
double* v = s->v;
double* f = s->f;
double* g = s->g;
double factorX = s->dt / s->grid.dx;
double factorY = s->dt / s->grid.dy;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX;
V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY;
}
}
}
void writeResult(Discretization* s, double* u, double* v, double* p)
{
int imax = s->grid.imax;
int jmax = s->grid.jmax;
double dx = s->grid.dx;
double dy = s->grid.dy;
double x = 0.0, y = 0.0;
FILE* fp;
fp = fopen("pressure.dat", "w");
if (fp == NULL) {
printf("Error!\n");
exit(EXIT_FAILURE);
}
for (int j = 1; j <= jmax; j++) {
y = (double)(j - 0.5) * dy;
for (int i = 1; i <= imax; i++) {
x = (double)(i - 0.5) * dx;
fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax + 2) + i]);
}
fprintf(fp, "\n");
}
fclose(fp);
fp = fopen("velocity.dat", "w");
if (fp == NULL) {
printf("Error!\n");
exit(EXIT_FAILURE);
}
for (int j = 1; j <= jmax; j++) {
y = dy * (j - 0.5);
for (int i = 1; i <= imax; i++) {
x = dx * (i - 0.5);
double velU = (u[j * (imax + 2) + i] + u[j * (imax + 2) + (i - 1)]) / 2.0;
double velV = (v[j * (imax + 2) + i] + v[(j - 1) * (imax + 2) + i]) / 2.0;
double len = sqrt((velU * velU) + (velV * velV));
fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, velU, velV, len);
}
}
fclose(fp);
}

43
BasicSolver/2D-mpi/src/discretization.h Normal file
View File

@@ -0,0 +1,43 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#ifndef __DISCRETIZATION_H_
#define __DISCRETIZATION_H_
#include "comm.h"
#include "grid.h"
#include "parameter.h"
enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
typedef struct {
/* geometry and grid information */
Grid grid;
/* arrays */
double *p, *rhs;
double *f, *g;
double *u, *v;
/* parameters */
double re, tau, gamma;
double gx, gy;
/* time stepping */
double dt, te;
double dtBound;
char* problem;
int bcLeft, bcRight, bcBottom, bcTop;
/* communication */
Comm comm;
} Discretization;
void initDiscretiztion(Discretization*, Parameter*);
void computeRHS(Discretization*);
void normalizePressure(Discretization*);
void computeTimestep(Discretization*);
void setBoundaryConditions(Discretization*);
void setSpecialBoundaryCondition(Discretization*);
void computeFG(Discretization*);
void adaptUV(Discretization*);
void writeResult(Discretization* s, double* u, double* v, double* p);
#endif

16
BasicSolver/2D-mpi/src/grid.h Normal file
View File

@@ -0,0 +1,16 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#ifndef __GRID_H_
#define __GRID_H_
typedef struct {
double dx, dy;
int imax, jmax;
double xlength, ylength;
} Grid;
#endif // __GRID_H_

211
BasicSolver/2D-mpi/src/main.c
View File

@@ -1,148 +1,111 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
*/
#include <float.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include "allocate.h"
#include "comm.h"
#include "discretization.h"
#include "parameter.h"
#include "progress.h"
#include "solver.h"
#include "timing.h"
#include <mpi.h>
enum VARIANT { SOR = 1, RB, RBA };
int main (int argc, char** argv)
static void writeResults(Discretization* s)
{
int rank;
int variant = SOR;
double S, E;
Parameter params;
Solver solver;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
initParameter(&params);
if (argc < 2) {
printf("Usage: %s <configFile>\n", argv[0]);
exit(EXIT_SUCCESS);
}
readParameter(&params, argv[1]);
if (argc == 3)
{
variant = atoi(argv[2]);
}
if (rank == 0) {
printParameter(&params);
}
initSolver(&solver, &params);
/* debugExchange(&solver); */
/* exit(EXIT_SUCCESS); */
initProgress(solver.te);
double tau = solver.tau;
double te = solver.te;
double t = 0.0;
S = getTimeStamp();
switch (variant) {
case SOR:
printf("Plain SOR\n");
while (t <= te) {
if (tau > 0.0) {
computeTimestep(&solver);
}
setBoundaryConditions(&solver);
setSpecialBoundaryCondition(&solver);
computeFG(&solver);
computeRHS(&solver);
solve(&solver);
adaptUV(&solver);
t += solver.dt;
}
break;
case RB:
printf("Red-black SOR\n");
while (t <= te) {
if (tau > 0.0) {
computeTimestep(&solver);
}
setBoundaryConditions(&solver);
setSpecialBoundaryCondition(&solver);
computeFG(&solver);
computeRHS(&solver);
solveRB(&solver);
adaptUV(&solver);
t += solver.dt;
}
break;
case RBA:
printf("Red-black SOR with acceleration\n");
while (t <= te) {
if (tau > 0.0) {
computeTimestep(&solver);
}
setBoundaryConditions(&solver);
setSpecialBoundaryCondition(&solver);
computeFG(&solver);
computeRHS(&solver);
solveRBA(&solver);
adaptUV(&solver);
t += solver.dt;
}
break;
}
#ifdef VERBOSE
if (rank == 0) {
printf("TIME %f , TIMESTEP %f\n", t, solver.dt);
}
#else
printProgress(t);
#endif
E = getTimeStamp();
stopProgress();
if (rank == 0) {
printf("Solution took %.2fs\n", E - S);
}
size_t bytesize = solver.imax * solver.jmax * sizeof(double);
#ifdef _MPI
size_t bytesize = (s->grid.imax + 2) * (s->grid.jmax + 2) * sizeof(double);
double* ug = allocate(64, bytesize);
double* vg = allocate(64, bytesize);
double* pg = allocate(64, bytesize);
commCollectResult(&solver.comm,
ug,
vg,
pg,
solver.u,
solver.v,
solver.p,
solver.jmax,
solver.imax);
writeResult(&solver, ug, vg, pg);
commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->grid.imax, s->grid.jmax);
writeResult(s, ug, vg, pg);
MPI_Finalize();
free(ug);
free(vg);
free(pg);
#else
writeResult(s, s->u, s->v, s->p);
#endif
}
int main(int argc, char** argv)
{
int rank;
double timeStart, timeStop;
Parameter p;
Discretization d;
Solver s;
commInit(&d.comm, argc, argv);
initParameter(&p);
if (argc != 2) {
printf("Usage: %s <configFile>\n", argv[0]);
exit(EXIT_SUCCESS);
}
readParameter(&p, argv[1]);
commPartition(&d.comm, p.jmax, p.imax);
if (commIsMaster(&d.comm)) {
printParameter(&p);
}
initDiscretiztion(&d, &p);
initSolver(&s, &d, &p);
#ifdef TEST
commPrintConfig(&d.comm);
commTestInit(&d.comm, d.p, d.f, d.g);
commExchange(&d.comm, d.p);
commShift(&d.comm, d.f, d.g);
commTestWrite(&d.comm, d.p, d.f, d.g);
writeResults(&d);
commFinalize(&d.comm);
exit(EXIT_SUCCESS);
#endif
#ifndef VERBOSE
initProgress(d.te);
#endif
double tau = d.tau;
double te = d.te;
double t = 0.0;
timeStart = getTimeStamp();
while (t <= te) {
if (tau > 0.0) computeTimestep(&d);
setBoundaryConditions(&d);
setSpecialBoundaryCondition(&d);
computeFG(&d);
computeRHS(&d);
solve(&s, d.p, d.rhs);
adaptUV(&d);
t += d.dt;
#ifdef VERBOSE
if (commIsMaster(s.comm)) {
printf("TIME %f , TIMESTEP %f\n", t, d.dt);
}
#else
printProgress(t);
#endif
}
timeStop = getTimeStamp();
#ifndef VERBOSE
stopProgress();
#endif
if (commIsMaster(s.comm)) {
printf("Solution took %.2fs\n", timeStop - timeStart);
}
writeResults(&d);
commFinalize(s.comm);
return EXIT_SUCCESS;
}

5
BasicSolver/2D-mpi/src/parameter.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -21,7 +21,6 @@ void initParameter(Parameter* param)
param->itermax = 1000;
param->eps = 0.0001;
param->omg = 1.8;
param->rho = 0.99;
}
void readParameter(Parameter* param, const char* filename)
@@ -76,8 +75,6 @@ void readParameter(Parameter* param, const char* filename)
PARSE_REAL(u_init);
PARSE_REAL(v_init);
PARSE_REAL(p_init);
PARSE_REAL(rho);
}
}

4
BasicSolver/2D-mpi/src/parameter.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -11,7 +11,7 @@ typedef struct {
double xlength, ylength;
int imax, jmax;
int itermax;
double eps, omg, rho;
double eps, omg;
double re, tau, gamma;
double te, dt;
double gx, gy;

43
BasicSolver/2D-mpi/src/progress.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -7,54 +7,45 @@
#include <math.h>
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "progress.h"
static double _end;
static int _current;
static int _rank = -1;
void initProgress(double end)
{
MPI_Comm_rank(MPI_COMM_WORLD, &_rank);
_end = end;
_current = 0;
if (_rank == 0) {
printf("[ ]");
fflush(stdout);
}
printf("[ ]");
fflush(stdout);
}
void printProgress(double current)
{
if (_rank == 0) {
int new = (int)rint((current / _end) * 10.0);
int new = (int)rint((current / _end) * 10.0);
if (new > _current) {
char progress[11];
_current = new;
progress[0] = 0;
if (new > _current) {
char progress[11];
_current = new;
progress[0] = 0;
for (int i = 0; i < 10; i++) {
if (i < _current) {
sprintf(progress + strlen(progress), "#");
} else {
sprintf(progress + strlen(progress), " ");
}
for (int i = 0; i < 10; i++) {
if (i < _current) {
sprintf(progress + strlen(progress), "#");
} else {
sprintf(progress + strlen(progress), " ");
}
printf("\r[%s]", progress);
}
fflush(stdout);
printf("\r[%s]", progress);
}
fflush(stdout);
}
void stopProgress()
{
if (_rank == 0) {
printf("\n");
fflush(stdout);
}
printf("\n");
fflush(stdout);
}

2
BasicSolver/2D-mpi/src/progress.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.

669
BasicSolver/2D-mpi/src/solver.c
View File

@@ -1,188 +1,86 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <float.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "allocate.h"
#include "comm.h"
#include "discretization.h"
#include "parameter.h"
#include "solver.h"
#include "util.h"
#define P(i, j) p[(j) * (imaxLocal + 2) + (i)]
#define F(i, j) f[(j) * (imaxLocal + 2) + (i)]
#define G(i, j) g[(j) * (imaxLocal + 2) + (i)]
#define U(i, j) u[(j) * (imaxLocal + 2) + (i)]
#define V(i, j) v[(j) * (imaxLocal + 2) + (i)]
#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)]
static void printConfig(Solver* s)
void initSolver(Solver* s, Discretization* d, Parameter* p)
{
if (commIsMaster(&s->comm)) {
printf("Parameters for #%s#\n", s->problem);
printf("BC Left:%d Right:%d Bottom:%d Top:%d\n",
s->bcLeft,
s->bcRight,
s->bcBottom,
s->bcTop);
printf("\tReynolds number: %.2f\n", s->re);
printf("\tGx Gy: %.2f %.2f\n", s->gx, s->gy);
printf("Geometry data:\n");
printf("\tDomain box size (x, y): %.2f, %.2f\n", s->xlength, s->ylength);
printf("\tCells (x, y): %d, %d\n", s->imax, s->jmax);
printf("\tCell size (dx, dy): %f, %f\n", s->dx, s->dy);
printf("Timestep parameters:\n");
printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te);
printf("\tdt bound: %.6f\n", s->dtBound);
printf("\tTau factor: %.2f\n", s->tau);
printf("Iterative s parameters:\n");
printf("\tMax iterations: %d\n", s->itermax);
printf("\tepsilon (stopping tolerance) : %f\n", s->eps);
printf("\tgamma factor: %f\n", s->gamma);
printf("\tomega (SOR relaxation): %f\n", s->omega);
}
commPrintConfig(&s->comm);
s->grid = &d->grid;
s->eps = p->eps;
s->omega = p->omg;
s->itermax = p->itermax;
s->comm = &d->comm;
}
void initSolver(Solver* s, Parameter* params)
void solve(Solver* s, double* p, double* rhs)
{
s->problem = params->name;
s->bcLeft = params->bcLeft;
s->bcRight = params->bcRight;
s->bcBottom = params->bcBottom;
s->bcTop = params->bcTop;
s->imax = params->imax;
s->jmax = params->jmax;
s->xlength = params->xlength;
s->ylength = params->ylength;
s->dx = params->xlength / params->imax;
s->dy = params->ylength / params->jmax;
s->eps = params->eps;
s->omega = params->omg;
s->itermax = params->itermax;
s->re = params->re;
s->gx = params->gx;
s->gy = params->gy;
s->dt = params->dt;
s->te = params->te;
s->tau = params->tau;
s->gamma = params->gamma;
commInit(&s->comm, s->jmax, s->imax);
/* allocate arrays */
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
size_t size = (imaxLocal + 2) * (jmaxLocal + 2);
s->u = allocate(64, size * sizeof(double));
s->v = allocate(64, size * sizeof(double));
s->p = allocate(64, size * sizeof(double));
s->rhs = allocate(64, size * sizeof(double));
s->f = allocate(64, size * sizeof(double));
s->g = allocate(64, size * sizeof(double));
for (int i = 0; i < size; i++) {
s->u[i] = params->u_init;
s->v[i] = params->v_init;
s->p[i] = params->p_init;
s->rhs[i] = 0.0;
s->f[i] = 0.0;
s->g[i] = 0.0;
}
double dx = s->dx;
double dy = s->dy;
double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy);
s->dtBound = 0.5 * s->re * 1.0 / invSqrSum;
#ifdef VERBOSE
printConfig(s);
#endif
}
void computeRHS(Solver* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double idx = 1.0 / s->dx;
double idy = 1.0 / s->dy;
double idt = 1.0 / s->dt;
double* rhs = s->rhs;
double* f = s->f;
double* g = s->g;
commShift(&s->comm, f, g);
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) *
idt;
}
}
}
int solve(Solver* s)
{
int imax = s->imax;
int jmax = s->jmax;
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
int imax = s->grid->imax;
int jmax = s->grid->jmax;
int imaxLocal = s->comm->imaxLocal;
int jmaxLocal = s->comm->jmaxLocal;
double eps = s->eps;
int itermax = s->itermax;
double dx2 = s->dx * s->dx;
double dy2 = s->dy * s->dy;
double dx2 = s->grid->dx * s->grid->dx;
double dy2 = s->grid->dy * s->grid->dy;
double idx2 = 1.0 / dx2;
double idy2 = 1.0 / dy2;
double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2);
double* p = s->p;
double* rhs = s->rhs;
double epssq = eps * eps;
int it = 0;
double res = 1.0;
commExchange(&s->comm, p);
int pass, jsw, isw;
int it = 0;
double res = 1.0;
while ((res >= epssq) && (it < itermax)) {
res = 0.0;
jsw = 1;
for (pass = 0; pass < 2; pass++) {
isw = jsw;
commExchange(s->comm, p);
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = isw; i < imaxLocal + 1; i += 2) {
double r = RHS(i, j) -
((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
(P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2);
double r = RHS(i, j) -
((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
(P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2);
P(i, j) -= (factor * r);
res += (r * r);
P(i, j) -= (factor * r);
res += (r * r);
}
isw = 3 - isw;
}
jsw = 3 - jsw;
}
if (commIsBoundary(&s->comm, BOTTOM)) { // set bottom bc
if (commIsBoundary(s->comm, BOTTOM)) { // set bottom bc
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, 0) = P(i, 1);
}
}
if (commIsBoundary(&s->comm, TOP)) { // set top bc
if (commIsBoundary(s->comm, TOP)) { // set top bc
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, jmaxLocal + 1) = P(i, jmaxLocal);
}
}
if (commIsBoundary(&s->comm, LEFT)) { // set left bc
if (commIsBoundary(s->comm, LEFT)) { // set left bc
for (int j = 1; j < jmaxLocal + 1; j++) {
P(0, j) = P(1, j);
}
}
if (commIsBoundary(&s->comm, RIGHT)) { // set right bc
if (commIsBoundary(s->comm, RIGHT)) { // set right bc
for (int j = 1; j < jmaxLocal + 1; j++) {
P(imaxLocal + 1, j) = P(imaxLocal, j);
}
@@ -191,7 +89,7 @@ int solve(Solver* s)
commReduction(&res, SUM);
res = res / (double)(imax * jmax);
#ifdef DEBUG
if (commIsMaster(&s->comm)) {
if (commIsMaster(s->comm)) {
printf("%d Residuum: %e\n", it, res);
}
#endif
@@ -199,499 +97,8 @@ int solve(Solver* s)
}
#ifdef VERBOSE
if (commIsMaster(&s->comm)) {
if (commIsMaster(s->comm)) {
printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
}
#endif
if (res < eps) {
return 0;
} else {
return 1;
}
}
static double maxElement(Solver* s, double* m)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
int size = (imaxLocal + 2) * (jmaxLocal + 2);
double maxval = DBL_MIN;
for (int i = 0; i < size; i++) {
maxval = MAX(maxval, fabs(m[i]));
}
commReduction(&maxval, MAX);
return maxval;
}
void computeTimestep(Solver* s)
{
double dt = s->dtBound;
double dx = s->dx;
double dy = s->dy;
double umax = maxElement(s, s->u);
double vmax = maxElement(s, s->v);
if (umax > 0) {
dt = (dt > dx / umax) ? dx / umax : dt;
}
if (vmax > 0) {
dt = (dt > dy / vmax) ? dy / vmax : dt;
}
s->dt = dt * s->tau;
}
int solveRB(Solver* s)
{
int imax = s->imax;
int jmax = s->jmax;
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double eps = s->eps;
int itermax = s->itermax;
double dx2 = s->dx * s->dx;
double dy2 = s->dy * s->dy;
double idx2 = 1.0 / dx2;
double idy2 = 1.0 / dy2;
double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2);
double* p = s->p;
double* rhs = s->rhs;
double epssq = eps * eps;
int it = 0;
double res = 1.0;
commExchange(&s->comm, p);
while ((res >= epssq) && (it < itermax)) {
res = 0.0;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
double r = RHS(i, j) -
((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
(P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2);
P(i, j) -= (factor * r);
res += (r * r);
}
}
if (commIsBoundary(&s->comm, BOTTOM)) { // set bottom bc
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, 0) = P(i, 1);
}
}
if (commIsBoundary(&s->comm, TOP)) { // set top bc
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, jmaxLocal + 1) = P(i, jmaxLocal);
}
}
if (commIsBoundary(&s->comm, LEFT)) { // set left bc
for (int j = 1; j < jmaxLocal + 1; j++) {
P(0, j) = P(1, j);
}
}
if (commIsBoundary(&s->comm, RIGHT)) { // set right bc
for (int j = 1; j < jmaxLocal + 1; j++) {
P(imaxLocal + 1, j) = P(imaxLocal, j);
}
}
commReduction(&res, SUM);
res = res / (double)(imax * jmax);
#ifdef DEBUG
if (commIsMaster(&s->comm)) {
printf("%d Residuum: %e\n", it, res);
}
#endif
it++;
}
}
int solveRBA(Solver* s)
{
int imax = s->imax;
int jmax = s->jmax;
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double eps = s->eps;
int itermax = s->itermax;
double dx2 = s->dx * s->dx;
double dy2 = s->dy * s->dy;
double idx2 = 1.0 / dx2;
double idy2 = 1.0 / dy2;
double factor = 0.5 * (dx2 * dy2) / (dx2 + dy2);
double* p = s->p;
double* rhs = s->rhs;
double epssq = eps * eps;
int it = 0;
double res = 1.0;
commExchange(&s->comm, p);
while ((res >= epssq) && (it < itermax)) {
res = 0.0;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
double r = RHS(i, j) -
((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 +
(P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2);
P(i, j) -= (factor * r);
res += (r * r);
}
}
if (commIsBoundary(&s->comm, BOTTOM)) { // set bottom bc
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, 0) = P(i, 1);
}
}
if (commIsBoundary(&s->comm, TOP)) { // set top bc
for (int i = 1; i < imaxLocal + 1; i++) {
P(i, jmaxLocal + 1) = P(i, jmaxLocal);
}
}
if (commIsBoundary(&s->comm, LEFT)) { // set left bc
for (int j = 1; j < jmaxLocal + 1; j++) {
P(0, j) = P(1, j);
}
}
if (commIsBoundary(&s->comm, RIGHT)) { // set right bc
for (int j = 1; j < jmaxLocal + 1; j++) {
P(imaxLocal + 1, j) = P(imaxLocal, j);
}
}
commReduction(&res, SUM);
res = res / (double)(imax * jmax);
#ifdef DEBUG
if (commIsMaster(&s->comm)) {
printf("%d Residuum: %e\n", it, res);
}
#endif
it++;
}
#ifdef VERBOSE
if (commIsMaster(&s->comm)) {
printf("Solver took %d iterations to reach %f\n", it, sqrt(res));
}
#endif
if (res < eps) {
return 0;
} else {
return 1;
}
}
void setBoundaryConditions(Solver* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double* u = s->u;
double* v = s->v;
if (commIsBoundary(&s->comm, TOP)) {
switch (s->bcTop) {
case NOSLIP:
for (int i = 1; i < imaxLocal + 1; i++) {
V(i, jmaxLocal) = 0.0;
U(i, jmaxLocal + 1) = -U(i, jmaxLocal);
}
break;
case SLIP:
for (int i = 1; i < imaxLocal + 1; i++) {
V(i, jmaxLocal) = 0.0;
U(i, jmaxLocal + 1) = U(i, jmaxLocal);
}
break;
case OUTFLOW:
for (int i = 1; i < imaxLocal + 1; i++) {
U(i, jmaxLocal + 1) = U(i, jmaxLocal);
V(i, jmaxLocal) = V(i, jmaxLocal - 1);
}
break;
case PERIODIC:
break;
}
}
if (commIsBoundary(&s->comm, BOTTOM)) {
switch (s->bcBottom) {
case NOSLIP:
for (int i = 1; i < imaxLocal + 1; i++) {
V(i, 0) = 0.0;
U(i, 0) = -U(i, 1);
}
break;
case SLIP:
for (int i = 1; i < imaxLocal + 1; i++) {
V(i, 0) = 0.0;
U(i, 0) = U(i, 1);
}
break;
case OUTFLOW:
for (int i = 1; i < imaxLocal + 1; i++) {
U(i, 0) = U(i, 1);
V(i, 0) = V(i, 1);
}
break;
case PERIODIC:
break;
}
}
if (commIsBoundary(&s->comm, RIGHT)) {
switch (s->bcRight) {
case NOSLIP:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(imaxLocal, j) = 0.0;
V(imaxLocal + 1, j) = -V(imaxLocal, j);
}
break;
case SLIP:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(imaxLocal, j) = 0.0;
V(imaxLocal + 1, j) = V(imaxLocal, j);
}
break;
case OUTFLOW:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(imaxLocal, j) = U(imaxLocal - 1, j);
V(imaxLocal + 1, j) = V(imaxLocal, j);
}
break;
case PERIODIC:
break;
}
}
if (commIsBoundary(&s->comm, LEFT)) {
switch (s->bcLeft) {
case NOSLIP:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(0, j) = 0.0;
V(0, j) = -V(1, j);
}
break;
case SLIP:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(0, j) = 0.0;
V(0, j) = V(1, j);
}
break;
case OUTFLOW:
for (int j = 1; j < jmaxLocal + 1; j++) {
U(0, j) = U(1, j);
V(0, j) = V(1, j);
}
break;
case PERIODIC:
break;
}
}
}
void setSpecialBoundaryCondition(Solver* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double* u = s->u;
if (strcmp(s->problem, "dcavity") == 0) {
if (commIsBoundary(&s->comm, TOP)) {
for (int i = 1; i < imaxLocal + 1; i++) {
U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal);
}
}
} else if (strcmp(s->problem, "canal") == 0) {
if (commIsBoundary(&s->comm, LEFT)) {
double ylength = s->ylength;
double dy = s->dy;
int rest = s->jmax % s->comm.size;
int yc = s->comm.rank * (s->jmax / s->comm.size) + MIN(rest, s->comm.rank);
double ys = dy * (yc + 0.5);
double y;
/* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */
for (int j = 1; j < jmaxLocal + 1; j++) {
y = ys + dy * (j - 0.5);
U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength);
}
}
}
/* print(solver, solver->u); */
}
void computeFG(Solver* s)
{
double* u = s->u;
double* v = s->v;
double* f = s->f;
double* g = s->g;
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double gx = s->gx;
double gy = s->gy;
double gamma = s->gamma;
double dt = s->dt;
double inverseRe = 1.0 / s->re;
double inverseDx = 1.0 / s->dx;
double inverseDy = 1.0 / s->dy;
double du2dx, dv2dy, duvdx, duvdy;
double du2dx2, du2dy2, dv2dx2, dv2dy2;
commExchange(&s->comm, u);
commExchange(&s->comm, v);
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
du2dx = inverseDx * 0.25 *
((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) -
(U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) +
gamma * inverseDx * 0.25 *
(fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) +
fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j)));
duvdy = inverseDy * 0.25 *
((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) -
(V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) +
gamma * inverseDy * 0.25 *
(fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) +
fabs(V(i, j - 1) + V(i + 1, j - 1)) *
(U(i, j) - U(i, j - 1)));
du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j));
du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1));
F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx);
duvdx = inverseDx * 0.25 *
((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) -
(U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) +
gamma * inverseDx * 0.25 *
(fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) +
fabs(U(i - 1, j) + U(i - 1, j + 1)) *
(V(i, j) - V(i - 1, j)));
dv2dy = inverseDy * 0.25 *
((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) -
(V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) +
gamma * inverseDy * 0.25 *
(fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) +
fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1)));
dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j));
dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1));
G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy);
}
}
/* ----------------------------- boundary of F --------------------------- */
if (commIsBoundary(&s->comm, LEFT)) {
for (int j = 1; j < jmaxLocal + 1; j++) {
F(0, j) = U(0, j);
}
}
if (commIsBoundary(&s->comm, RIGHT)) {
for (int j = 1; j < jmaxLocal + 1; j++) {
F(imaxLocal, j) = U(imaxLocal, j);
}
}
/* ----------------------------- boundary of G --------------------------- */
if (commIsBoundary(&s->comm, BOTTOM)) {
for (int i = 1; i < imaxLocal + 1; i++) {
G(i, 0) = V(i, 0);
}
}
if (commIsBoundary(&s->comm, TOP)) {
for (int i = 1; i < imaxLocal + 1; i++) {
G(i, jmaxLocal) = V(i, jmaxLocal);
}
}
}
void adaptUV(Solver* s)
{
int imaxLocal = s->comm.imaxLocal;
int jmaxLocal = s->comm.jmaxLocal;
double* p = s->p;
double* u = s->u;
double* v = s->v;
double* f = s->f;
double* g = s->g;
double factorX = s->dt / s->dx;
double factorY = s->dt / s->dy;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX;
V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY;
}
}
}
void writeResult(Solver* s, double* u, double* v, double* p)
{
int imax = s->imax;
int jmax = s->jmax;
double dx = s->dx;
double dy = s->dy;
double x = 0.0, y = 0.0;
FILE* fp;
fp = fopen("pressure.dat", "w");
if (fp == NULL) {
printf("Error!\n");
exit(EXIT_FAILURE);
}
for (int j = 1; j < jmax; j++) {
y = (double)(j - 0.5) * dy;
for (int i = 1; i < imax; i++) {
x = (double)(i - 0.5) * dx;
fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]);
}
fprintf(fp, "\n");
}
fclose(fp);
fp = fopen("velocity.dat", "w");
if (fp == NULL) {
printf("Error!\n");
exit(EXIT_FAILURE);
}
for (int j = 1; j < jmax; j++) {
y = dy * (j - 0.5);
for (int i = 1; i < imax; i++) {
x = dx * (i - 0.5);
double vel_u = (u[j * (imax) + i] + u[j * (imax) + (i - 1)]) / 2.0;
double vel_v = (v[j * (imax) + i] + v[(j - 1) * (imax) + i]) / 2.0;
double len = sqrt((vel_u * vel_u) + (vel_v * vel_v));
fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len);
}
}
fclose(fp);
}

39
BasicSolver/2D-mpi/src/solver.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -7,43 +7,20 @@
#ifndef __SOLVER_H_
#define __SOLVER_H_
#include "comm.h"
#include "discretization.h"
#include "grid.h"
#include "parameter.h"
enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
typedef struct {
/* geometry and grid information */
double dx, dy;
int imax, jmax;
double xlength, ylength;
/* arrays */
double *p, *rhs;
double *f, *g;
double *u, *v;
Grid* grid;
/* parameters */
double eps, omega, rho;
double re, tau, gamma;
double gx, gy;
/* time stepping */
double eps, omega;
int itermax;
double dt, te;
double dtBound;
char* problem;
int bcLeft, bcRight, bcBottom, bcTop;
/* communication */
Comm comm;
Comm* comm;
} Solver;
void initSolver(Solver*, Parameter*);
void computeRHS(Solver*);
int solve(Solver*);
int solveRB(Solver*);
int solveRBA(Solver*);
void normalizePressure(Solver*);
void computeTimestep(Solver*);
void setBoundaryConditions(Solver*);
void setSpecialBoundaryCondition(Solver*);
void computeFG(Solver*);
void adaptUV(Solver*);
void writeResult(Solver* s, double* u, double* v, double* p);
void initSolver(Solver*, Discretization*, Parameter*);
void solve(Solver*, double*, double*);
#endif

8
BasicSolver/2D-mpi/src/timing.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
@@ -7,18 +7,16 @@
#include <stdlib.h>
#include <time.h>
double getTimeStamp()
double getTimeStamp(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9;
}
double getTimeResolution()
double getTimeResolution(void)
{
struct timespec ts;
clock_getres(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9;
}
double getTimeStamp_() { return getTimeStamp(); }

7
BasicSolver/2D-mpi/src/timing.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
@@ -7,8 +7,7 @@
#ifndef __TIMING_H_
#define __TIMING_H_
extern double getTimeStamp();
extern double getTimeResolution();
extern double getTimeStamp_();
extern double getTimeStamp(void);
extern double getTimeResolution(void);
#endif // __TIMING_H_

9
BasicSolver/2D-mpi/src/util.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
@@ -19,4 +19,11 @@
#define ABS(a) ((a) >= 0 ? (a) : -(a))
#endif
#define P(i, j) p[(j) * (imaxLocal + 2) + (i)]
#define F(i, j) f[(j) * (imaxLocal + 2) + (i)]
#define G(i, j) g[(j) * (imaxLocal + 2) + (i)]
#define U(i, j) u[(j) * (imaxLocal + 2) + (i)]
#define V(i, j) v[(j) * (imaxLocal + 2) + (i)]
#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)]
#endif // __UTIL_H_