moebiusband/NuSiF-Solver
2
0
Fork 2
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merge 3D mpi versions

Browse Source
This commit is contained in:
Jan Eitzinger
2024-01-31 17:40:36 +01:00
parent d53da89b95
commit 91cbb29f6e
52 changed files with 149 additions and 2344 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
BasicSolver/3D-mpi/src/allocate.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.

2
BasicSolver/3D-mpi/src/allocate.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.

38
BasicSolver/3D-mpi/src/comm.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -296,6 +296,30 @@ void commShift(Comm* c, double* f, double* g, double* h)
#endif
}
void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax)
{
#if defined(_MPI)
int sum = 0;
for (int i = 0; i < c->coords[ICORD]; i++) {
sum += sizeOfRank(i, c->dims[ICORD], imax);
}
offsets[IDIM] = sum;
sum = 0;
for (int i = 0; i < c->coords[JCORD]; i++) {
sum += sizeOfRank(i, c->dims[JCORD], jmax);
}
offsets[JDIM] = sum;
sum = 0;
for (int i = 0; i < c->coords[KCORD]; i++) {
sum += sizeOfRank(i, c->dims[KCORD], kmax);
}
offsets[KDIM] = sum;
#endif
}
#define G(v, i, j, k) \
v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
@@ -445,7 +469,7 @@ void commCollectResult(Comm* c,
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
pg[idx++] = G(s->p, i, j, k);
pg[idx++] = G(p, i, j, k);
}
}
}
@@ -455,7 +479,7 @@ void commCollectResult(Comm* c,
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
ug[idx++] = (G(s->u, i, j, k) + G(s->u, i - 1, j, k)) / 2.0;
ug[idx++] = (G(u, i, j, k) + G(u, i - 1, j, k)) / 2.0;
}
}
}
@@ -465,7 +489,7 @@ void commCollectResult(Comm* c,
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
vg[idx++] = (G(s->v, i, j, k) + G(s->v, i, j - 1, k)) / 2.0;
vg[idx++] = (G(v, i, j, k) + G(v, i, j - 1, k)) / 2.0;
}
}
}
@@ -475,7 +499,7 @@ void commCollectResult(Comm* c,
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
wg[idx++] = (G(s->w, i, j, k) + G(s->w, i, j, k - 1)) / 2.0;
wg[idx++] = (G(w, i, j, k) + G(w, i, j, k - 1)) / 2.0;
}
}
}
@@ -524,8 +548,8 @@ void commInit(Comm* c, int argc, char** argv)
MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
#else
c->rank = 0;
c->size = 1;
c->rank = 0;
c->size = 1;
#endif
}

3
BasicSolver/3D-mpi/src/comm.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2024 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -44,6 +44,7 @@ extern void commExchange(Comm*, double*);
extern void commShift(Comm* c, double* f, double* g, double* h);
extern void commReduction(double* v, int op);
extern int commIsBoundary(Comm* c, Direction direction);
extern void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax);
extern void commCollectResult(Comm* c,
double* ug,
double* vg,

2
BasicSolver/3D-mpi/src/grid.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.

12
BasicSolver/3D-mpi/src/main.c
View File

@@ -1,11 +1,9 @@
/*
* Copyright (C) 2024 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
*/
#include <float.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
@@ -73,6 +71,13 @@ int main(int argc, char** argv)
printf("Solution took %.2fs\n", timeStop - timeStart);
}
#ifdef _VTK_WRITER_MPI
VtkOptions opts = { .grid = s.grid, .comm = s.comm };
vtkOpen(&opts, s.problem);
vtkScalar(&opts, "pressure", s.p);
vtkVector(&opts, "velocity", (VtkVector) { s.u, s.v, s.w });
vtkClose(&opts);
#else
double *pg, *ug, *vg, *wg;
if (commIsMaster(&s.comm)) {
@@ -104,6 +109,7 @@ int main(int argc, char** argv)
vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg });
vtkClose(&opts);
}
#endif
commFinalize(&s.comm);
return EXIT_SUCCESS;

2
BasicSolver/3D-mpi/src/parameter.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.

2
BasicSolver/3D-mpi/src/parameter.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.

2
BasicSolver/3D-mpi/src/progress.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.

2
BasicSolver/3D-mpi/src/progress.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.

22
BasicSolver/3D-mpi/src/solver.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -207,24 +207,24 @@ void solve(Solver* s)
jsw = ksw;
commExchange(&s->comm, p);
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int k = 1; k < kmaxLocal + 1; k++) {
isw = jsw;
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = isw; i < imaxLocal + 1; i += 2) {
double r =
RHS(i, j, k) -
((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 +
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
idy2 +
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
idz2);
(P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) *
idy2 +
(P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) *
idz2);
P(i, j, k) -= (factor * r);
res += (r * r);
}
isw = 3 - isw;
P(i, j, k) -= (factor * r);
res += (r * r);
}
isw = 3 - isw;
}
jsw = 3 - jsw;
}
ksw = 3 - ksw;

2
BasicSolver/3D-mpi/src/solver.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.

21
BasicSolver/3D-mpi/src/test.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
@@ -26,10 +26,21 @@ void testInit(Solver* s)
for (int k = 0; k < kmaxLocal + 2; k++) {
for (int j = 0; j < jmaxLocal + 2; j++) {
for (int i = 0; i < imaxLocal + 2; i++) {
G(p, i, j, k) = myrank;
G(f, i, j, k) = myrank;
G(g, i, j, k) = myrank;
G(h, i, j, k) = myrank;
G(p, i, j, k) = 10.0;
G(f, i, j, k) = myrank + 1.0;
G(g, i, j, k) = myrank + 1.0;
G(h, i, j, k) = myrank + 1.0;
}
}
}
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
G(p, i, j, k) = myrank + 1.0;
G(f, i, j, k) = myrank + 1.0;
G(g, i, j, k) = myrank + 1.0;
G(h, i, j, k) = myrank + 1.0;
}
}
}

2
BasicSolver/3D-mpi/src/test.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.

2
BasicSolver/3D-mpi/src/timing.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.

2
BasicSolver/3D-mpi/src/timing.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.

2
BasicSolver/3D-mpi/src/util.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.

229
BasicSolver/3D-mpi/src/vtkWriter-mpi.c Normal file
View File

@@ -0,0 +1,229 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <mpi.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "allocate.h"
#include "comm.h"
#include "vtkWriter.h"
// reset fileview for output of string headers
static void resetFileview(VtkOptions* o)
{
MPI_Offset disp;
MPI_File_sync(o->fh);
MPI_Barrier(o->comm.comm);
MPI_File_get_size(o->fh, &disp);
MPI_File_set_view(o->fh, disp, MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL);
}
static void writeVersion(VtkOptions* o)
{
char header[50] = "# vtk DataFile Version 3.0\n";
// always overwrite exiting files
MPI_File_set_view(o->fh, 0, MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
}
}
static void writeHeader(VtkOptions* o)
{
char header[400];
char* cursor = header;
cursor += sprintf(cursor, "PAMPI cfd solver output\n");
cursor += sprintf(cursor, "BINARY\n");
cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n");
cursor += sprintf(cursor,
"DIMENSIONS %d %d %d\n",
o->grid.imax,
o->grid.jmax,
o->grid.kmax);
cursor += sprintf(cursor,
"ORIGIN %f %f %f\n",
o->grid.dx * 0.5,
o->grid.dy * 0.5,
o->grid.dz * 0.5);
cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz);
cursor += sprintf(cursor,
"POINT_DATA %d\n",
o->grid.imax * o->grid.jmax * o->grid.kmax);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
}
}
void vtkOpen(VtkOptions* o, char* problem)
{
char filename[50];
snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size);
MPI_File_open(o->comm.comm,
filename,
MPI_MODE_WRONLY | MPI_MODE_CREATE,
MPI_INFO_NULL,
&o->fh);
if (commIsMaster(&o->comm)) {
printf("Writing VTK output for %s\n", problem);
}
writeVersion(o);
writeHeader(o);
}
void vtkScalar(VtkOptions* o, char* name, double* s)
{
resetFileview(o);
if (commIsMaster(&o->comm)) printf("Register scalar %s\n", name);
char header[100];
char* cursor = header;
cursor += sprintf(cursor, "SCALARS %s double 1\n", name);
cursor += sprintf(cursor, "LOOKUP_TABLE default\n");
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
}
int offsets[NDIMS];
commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax);
// set global view in file
MPI_Offset disp;
MPI_Datatype fileViewType;
MPI_File_sync(o->fh);
MPI_Barrier(o->comm.comm);
MPI_File_get_size(o->fh, &disp);
MPI_Type_create_subarray(NDIMS,
(int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax },
(int[NDIMS]) { o->comm.kmaxLocal, o->comm.jmaxLocal, o->comm.imaxLocal },
offsets,
MPI_ORDER_C,
MPI_DOUBLE,
&fileViewType);
MPI_Type_commit(&fileViewType);
MPI_File_set_view(o->fh, disp, MPI_DOUBLE, fileViewType, "external32", MPI_INFO_NULL);
#ifdef VERBOSE
printf("Rank: %d, Disp: %lld, Size(k,j,i): %d %d %d, Offset(k,j,i): %d %d %d\n",
o->comm.rank,
disp,
o->comm.kmaxLocal,
o->comm.jmaxLocal,
o->comm.imaxLocal,
offsets[KDIM],
offsets[JDIM],
offsets[IDIM]);
#endif
// create local bulk type
MPI_Datatype bulkType;
MPI_Type_create_subarray(NDIMS,
(int[NDIMS]) { o->comm.kmaxLocal + 2,
o->comm.jmaxLocal + 2,
o->comm.imaxLocal + 2 }, // oldsizes
(int[NDIMS]) { o->comm.kmaxLocal,
o->comm.jmaxLocal,
o->comm.imaxLocal }, // newsizes
(int[NDIMS]) { 1, 1, 1 }, // offsets
MPI_ORDER_C,
MPI_DOUBLE,
&bulkType);
MPI_Type_commit(&bulkType);
MPI_File_write(o->fh, s, 1, bulkType, MPI_STATUS_IGNORE);
MPI_Type_free(&bulkType);
MPI_Type_free(&fileViewType);
// Binary segment must be terminated with newline character
resetFileview(o);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, "\n", 1, MPI_CHAR, MPI_STATUS_IGNORE);
}
}
#define G(v, i, j, k) \
v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
void vtkVector(VtkOptions* o, char* name, VtkVector vec)
{
int imaxLocal = o->comm.imaxLocal;
int jmaxLocal = o->comm.jmaxLocal;
int kmaxLocal = o->comm.kmaxLocal;
if (commIsMaster(&o->comm)) printf("Register vector %s\n", name);
const size_t MAX_HEADER = 100;
char* header = (char*)malloc(MAX_HEADER);
sprintf(header, "VECTORS %s double\n", name);
resetFileview(o);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
}
int offsets[NDIMS];
commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax);
// set global view in file
MPI_Offset disp;
MPI_Datatype fileViewType, vectorType;
MPI_File_sync(o->fh);
MPI_Barrier(o->comm.comm);
MPI_File_get_size(o->fh, &disp);
MPI_Type_contiguous(NDIMS, MPI_DOUBLE, &vectorType);
MPI_Type_commit(&vectorType);
MPI_Type_create_subarray(NDIMS,
(int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax },
(int[NDIMS]) { kmaxLocal, jmaxLocal, imaxLocal },
offsets,
MPI_ORDER_C,
vectorType,
&fileViewType);
MPI_Type_commit(&fileViewType);
MPI_File_set_view(o->fh, disp, MPI_DOUBLE, fileViewType, "external32", MPI_INFO_NULL);
size_t cnt = imaxLocal * jmaxLocal * kmaxLocal;
double* tmp = allocate(64, cnt * NDIMS * sizeof(double));
int idx = 0;
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
tmp[idx++] = (G(vec.u, i, j, k) + G(vec.u, i - 1, j, k)) / 2.0;
tmp[idx++] = (G(vec.v, i, j, k) + G(vec.v, i, j - 1, k)) / 2.0;
tmp[idx++] = (G(vec.w, i, j, k) + G(vec.w, i, j, k - 1)) / 2.0;
}
}
}
if (commIsMaster(&o->comm)) printf("Write %d vectors\n", (int)cnt);
MPI_File_write(o->fh, tmp, cnt, vectorType, MPI_STATUS_IGNORE);
MPI_Type_free(&fileViewType);
MPI_Type_free(&vectorType);
// Binary segment must be terminated with newline character
resetFileview(o);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, "\n", 1, MPI_CHAR, MPI_STATUS_IGNORE);
}
}
void vtkClose(VtkOptions* o) { MPI_File_close(&o->fh); }

2
BasicSolver/3D-mpi/src/vtkWriter.c → BasicSolver/3D-mpi/src/vtkWriter-seq.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.

10
BasicSolver/3D-mpi/src/vtkWriter.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -8,14 +8,20 @@
#define __VTKWRITER_H_
#include <stdio.h>
#include "comm.h"
#include "grid.h"
typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat;
typedef struct VtkOptions {
VtkFormat fmt;
Grid grid;
#ifdef _VTK_WRITER_MPI
MPI_File fh;
#else
FILE* fh;
VtkFormat fmt;
#endif // _VTK_WRITER_MPI
Comm comm;
} VtkOptions;
typedef struct VtkVector {