moebiusband/NuSiF-Solver
2
0
Fork 2
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Merging the new branch

Browse Source
This commit is contained in:
Aditya Ujeniya
2024-07-08 09:52:11 +02:00
parent 28fec03be9
commit d5053b96ea
433 changed files with 5948 additions and 4848038 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
BasicSolver/3D-mpi/Makefile
View File

@@ -1,5 +1,5 @@
#=======================================================================================
# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
# Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
# All rights reserved.
# Use of this source code is governed by a MIT-style
# license that can be found in the LICENSE file.
@@ -18,13 +18,18 @@ include $(MAKE_DIR)/include_$(TAG).mk
INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR)
VPATH = $(SRC_DIR)
SRC = $(wildcard $(SRC_DIR)/*.c)
SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c))
ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC))
OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC))
OBJ += $(BUILD_DIR)/vtkWriter-$(VTK_OUTPUT_FMT).o
SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h)
ifeq ($(VTK_OUTPUT_FMT),mpi)
DEFINES += -D_VTK_WRITER_MPI
endif
CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES)
${TARGET}: $(BUILD_DIR) $(OBJ)
${TARGET}: sanity-checks $(BUILD_DIR) $(OBJ)
$(info ===> LINKING $(TARGET))
$(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS)
@@ -39,15 +44,11 @@ $(BUILD_DIR)/%.s: %.c
.PHONY: clean distclean tags info asm format
clean: vis
clean:
$(info ===> CLEAN)
@rm -rf $(BUILD_DIR)
@rm -f tags
vis:
$(info ===> REMOVING VIZUALISATION FILES)
@rm -f vtk_files/particle*.vtk
distclean: clean
$(info ===> DIST CLEAN)
@rm -f $(TARGET)
@@ -69,6 +70,14 @@ format:
done
@echo "Done"
sanity-checks:
ifeq ($(VTK_OUTPUT_FMT),mpi)
ifeq ($(ENABLE_MPI),false)
$(error VTK_OUTPUT_FMT mpi only supported for ENABLE_MPI true!)
endif
endif
$(BUILD_DIR):
@mkdir $(BUILD_DIR)

21011
BasicSolver/3D-mpi/backstep-p4.vtk
View File

File diff suppressed because it is too large Load Diff

82
BasicSolver/3D-mpi/backstep.par
View File

@@ -1,82 +0,0 @@
#==============================================================================
# Laminar Canal Flow
#==============================================================================
# Problem specific Data:
# ---------------------
name backstep # name of flow setup
bcLeft 3 # flags for boundary conditions
bcRight 3 # 1 = no-slip 3 = outflow
bcBottom 1 # 2 = free-slip 4 = periodic
bcTop 1 #
bcFront 1 #
bcBack 1 #
gx 0.0 # Body forces (e.g. gravity)
gy 0.0 #
gz 0.0 #
re 5000.0 # Reynolds number
u_init 1.0 # initial value for velocity in x-direction
v_init 0.0 # initial value for velocity in y-direction
w_init 0.0 # initial value for velocity in z-direction
p_init 1.0 # initial value for pressure
# Geometry Data:
# -------------
xlength 7.0 # domain size in x-direction
ylength 1.5 # domain size in y-direction
zlength 1.0 # domain size in z-direction
imax 70 # number of interior cells in x-direction
jmax 15 # number of interior cells in y-direction
kmax 10 # number of interior cells in z-direction
# Time Data:
# ---------
te 100.0 # final time
dt 0.02 # time stepsize
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
# Pressure Iteration Data:
# -----------------------
itermax 500 # maximal number of pressure iteration in one time step
eps 0.0001 # stopping tolerance for pressure iteration
rho 0.52
omg 1.7 # relaxation parameter for SOR iteration
gamma 0.9 # upwind differencing factor gamma
# Particle Tracing Data:
# -----------------------
numberOfParticles 500
startTime 30
injectTimePeriod 1.0
writeTimePeriod 0.2
x1 0.0
y1 0.5
z1 0.0
x2 0.0
y2 1.45
z2 1.0
# Obstacle Geometry Data:
# -----------------------
# Shape 0 disable, 1 Rectangle/Square, 2 Circle
shape 1
xCenter 0.0
yCenter 0.0
zCenter 0.0
xRectLength 2.0
yRectLength 1.0
zRectLength 2.0
circleRadius 1.0
#===============================================================================

43
BasicSolver/3D-mpi/canal.par
View File

@@ -18,9 +18,9 @@ gx 0.0 # Body forces (e.g. gravity)
gy 0.0 #
gz 0.0 #
re 5000.0 # Reynolds number
re 100.0 # Reynolds number
u_init 0.0 # initial value for velocity in x-direction
u_init 1.0 # initial value for velocity in x-direction
v_init 0.0 # initial value for velocity in y-direction
w_init 0.0 # initial value for velocity in z-direction
p_init 0.0 # initial value for pressure
@@ -31,14 +31,14 @@ p_init 0.0 # initial value for pressure
xlength 30.0 # domain size in x-direction
ylength 4.0 # domain size in y-direction
zlength 4.0 # domain size in z-direction
imax 100 # number of interior cells in x-direction
jmax 40 # number of interior cells in y-direction
kmax 40 # number of interior cells in z-direction
imax 200 # number of interior cells in x-direction
jmax 50 # number of interior cells in y-direction
kmax 50 # number of interior cells in z-direction
# Time Data:
# ---------
te 200.0 # final time
te 100.0 # final time
dt 0.02 # time stepsize
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
@@ -47,35 +47,6 @@ tau 0.5 # safety factor for time stepsize control (<0 constant delt)
itermax 500 # maximal number of pressure iteration in one time step
eps 0.0001 # stopping tolerance for pressure iteration
omg 0.52
omg 1.7 # relaxation parameter for SOR iteration
omg 1.3 # relaxation parameter for SOR iteration
gamma 0.9 # upwind differencing factor gamma
# Particle Tracing Data:
# -----------------------
numberOfParticles 200
startTime 0
injectTimePeriod 2.0
writeTimePeriod 1.0
x1 1.0
y1 0.0
z1 1.0
x2 1.0
y2 4.0
z2 1.0
# Obstacle Geometry Data:
# -----------------------
# Shape 0 disable, 1 Rectangle/Square, 2 Circle
shape 1
xCenter 10.0
yCenter 2.0
zCenter 2.0
xRectLength 8.0
yRectLength 2.0
zRectLength 2.0
circleRadius 1.0
#===============================================================================

5
BasicSolver/3D-mpi/config.mk
View File

@@ -1,6 +1,9 @@
# Supported: GCC, CLANG, ICC
TAG ?= ICC
TAG ?= CLANG
ENABLE_MPI ?= true
ENABLE_OPENMP ?= false
# Supported: seq, mpi
VTK_OUTPUT_FMT ?= seq
#Feature options
OPTIONS += -DARRAY_ALIGNMENT=64

39
BasicSolver/3D-mpi/dcavity.par
View File

@@ -31,14 +31,14 @@ p_init 0.0 # initial value for pressure
xlength 1.0 # domain size in x-direction
ylength 1.0 # domain size in y-direction
zlength 1.0 # domain size in z-direction
imax 40 # number of interior cells in x-direction
jmax 40 # number of interior cells in y-direction
kmax 40 # number of interior cells in z-direction
imax 128 # number of interior cells in x-direction
jmax 128 # number of interior cells in y-direction
kmax 128 # number of interior cells in z-direction
# Time Data:
# ---------
te 50.0 # final time
te 10.0 # final time
dt 0.02 # time stepsize
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
@@ -47,35 +47,6 @@ tau 0.5 # safety factor for time stepsize control (<0 constant delt)
itermax 1000 # maximal number of pressure iteration in one time step
eps 0.001 # stopping tolerance for pressure iteration
rho 0.5
omg 1.7 # relaxation parameter for SOR iteration
omg 1.8 # relaxation parameter for SOR iteration
gamma 0.9 # upwind differencing factor gamma
# Particle Tracing Data:
# -----------------------
numberOfParticles 30
startTime 10
injectTimePeriod 3.0
writeTimePeriod 1.0
x1 0.1
y1 0.0
z1 1.0
x2 1.0
y2 4.0
z2 1.0
# Obstacle Geometry Data:
# -----------------------
# Shape 0 disable, 1 Rectangle/Square, 2 Circle
shape 1
xCenter 10.0
yCenter 2.0
zCenter 2.0
xRectLength 8.0
yRectLength 2.0
zRectLength 2.0
circleRadius 1.0
#===============================================================================

12
BasicSolver/3D-mpi/include_CLANG.mk
View File

@@ -1,4 +1,10 @@
ifeq ($(ENABLE_MPI),true)
CC = mpicc
DEFINES = -D_MPI
else
CC = cc
endif
GCC = cc
LINKER = $(CC)
@@ -9,9 +15,7 @@ LIBS = # -lomp
endif
VERSION = --version
# CFLAGS = -O3 -std=c17 $(OPENMP)
CFLAGS = -Ofast -std=c17
#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG
LFLAGS = $(OPENMP) -lm
DEFINES = -D_GNU_SOURCE# -DDEBUG
INCLUDES =
DEFINES += -D_GNU_SOURCE# -DDEBUG
INCLUDES = -I/opt/homebrew/include

8
BasicSolver/3D-mpi/include_GCC.mk
View File

@@ -1,4 +1,10 @@
ifeq ($(ENABLE_MPI),true)
CC = mpicc
DEFINES = -D_MPI
else
CC = gcc
endif
GCC = gcc
LINKER = $(CC)
@@ -9,6 +15,6 @@ endif
VERSION = --version
CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP)
LFLAGS = $(OPENMP)
DEFINES = -D_GNU_SOURCE
DEFINES += -D_GNU_SOURCE
INCLUDES =
LIBS =

14
BasicSolver/3D-mpi/include_ICC.mk
View File

@@ -1,5 +1,11 @@
CC = mpiicc
GCC = icc
ifeq ($(ENABLE_MPI),true)
CC = mpiicc
DEFINES = -D_MPI
else
CC = icc
endif
GCC = gcc
LINKER = $(CC)
ifeq ($(ENABLE_OPENMP),true)
@@ -7,8 +13,8 @@ OPENMP = -qopenmp
endif
VERSION = --version
CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP)
CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP)
LFLAGS = $(OPENMP)
DEFINES = -D_GNU_SOURCE
DEFINES += -D_GNU_SOURCE# -DDEBUG
INCLUDES =
LIBS =

82
BasicSolver/3D-mpi/karman.par
View File

@@ -1,82 +0,0 @@
#==============================================================================
# Turbulent Canal Flow
#==============================================================================
# Problem specific Data:
# ---------------------
name karman # name of flow setup
bcLeft 3 # flags for boundary conditions
bcRight 3 # 1 = no-slip 3 = outflow
bcBottom 1 # 2 = free-slip 4 = periodic
bcTop 1 #
bcFront 1 #
bcBack 1 #
gx 0.0 # Body forces (e.g. gravity)
gy 0.0 #
gz 0.0 #
re 5050.0 # Reynolds number
u_init 1.0 # initial value for velocity in x-direction
v_init 0.0 # initial value for velocity in y-direction
w_init 0.0 # initial value for velocity in z-direction
p_init 0.0 # initial value for pressure
# Geometry Data:
# -------------
xlength 30.0 # domain size in x-direction
ylength 8.0 # domain size in y-direction
zlength 8.0 # domain size in z-direction
imax 200 # number of interior cells in x-direction
jmax 80 # number of interior cells in y-direction
kmax 80 # number of interior cells in z-direction
# Time Data:
# ---------
te 250.0 # final time
dt 0.02 # time stepsize
tau 0.5 # safety factor for time stepsize control (<0 constant delt)
# Pressure Iteration Data:
# -----------------------
itermax 200 # maximal number of pressure iteration in one time step
eps 0.001 # stopping tolerance for pressure iteration
rho 0.52
omg 1.75 # relaxation parameter for SOR iteration
gamma 0.9 # upwind differencing factor gamma
# Particle Tracing Data:
# -----------------------
numberOfParticles 4000
startTime 50
injectTimePeriod 1.0
writeTimePeriod 5.0
x1 0.0
y1 3.6
z1 3.6
x2 0.0
y2 4.7
z2 4.7
# Obstacle Geometry Data:
# -----------------------
# Shape 0 disable, 1 Rectangle/Square, 2 Circle
shape 2
xCenter 5.0
yCenter 4.0
zCenter 4.0
xRectLength 8.0
yRectLength 2.0
zRectLength 2.0
circleRadius 2.0
#===============================================================================

7
BasicSolver/3D-mpi/src/allocate.c
View File

@@ -1,14 +1,17 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
*/
#include <errno.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
void* allocate(int alignment, size_t bytesize)
#include "allocate.h"
void* allocate(size_t alignment, size_t bytesize)
{
int errorCode;
void* ptr;

4
BasicSolver/3D-mpi/src/allocate.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
@@ -8,6 +8,6 @@
#define __ALLOCATE_H_
#include <stdlib.h>
extern void* allocate(int alignment, size_t bytesize);
extern void* allocate(size_t alignment, size_t bytesize);
#endif

155
BasicSolver/3D-mpi/src/comm.c
View File

@@ -1,10 +1,9 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <mpi.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
@@ -12,6 +11,7 @@
#include "allocate.h"
#include "comm.h"
#if defined(_MPI)
// subroutines local to this module
static int sizeOfRank(int rank, int size, int N)
{
@@ -101,14 +101,6 @@ static void setupCommunication(Comm* c, Direction direction, int layer)
MPI_DOUBLE,
&c->rbufferTypes[direction]);
MPI_Type_commit(&c->rbufferTypes[direction]);
MPI_Type_create_subarray(NDIMS,
sizes,
subSizes,
starts,
MPI_ORDER_C,
MPI_INT,
&c->rbufferTypesInt[direction]);
MPI_Type_commit(&c->rbufferTypesInt[direction]);
} else if (layer == BULK) {
MPI_Type_create_subarray(NDIMS,
sizes,
@@ -118,14 +110,6 @@ static void setupCommunication(Comm* c, Direction direction, int layer)
MPI_DOUBLE,
&c->sbufferTypes[direction]);
MPI_Type_commit(&c->sbufferTypes[direction]);
MPI_Type_create_subarray(NDIMS,
sizes,
subSizes,
starts,
MPI_ORDER_C,
MPI_INT,
&c->sbufferTypesInt[direction]);
MPI_Type_commit(&c->sbufferTypesInt[direction]);
}
}
@@ -193,19 +177,23 @@ static int sum(int* sizes, int position)
return sum;
}
#endif // defined _MPI
// exported subroutines
void commReduction(double* v, int op)
{
#if defined(_MPI)
if (op == MAX) {
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
} else if (op == SUM) {
MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
}
#endif
}
int commIsBoundary(Comm* c, Direction direction)
{
#if defined(_MPI)
switch (direction) {
case LEFT:
return c->coords[ICORD] == 0;
@@ -229,12 +217,14 @@ int commIsBoundary(Comm* c, Direction direction)
printf("ERROR!\n");
break;
}
#endif
return 0;
return 1;
}
void commExchange(Comm* c, double* grid)
{
#if defined(_MPI)
int counts[6] = { 1, 1, 1, 1, 1, 1 };
MPI_Aint displs[6] = { 0, 0, 0, 0, 0, 0 };
@@ -247,26 +237,12 @@ void commExchange(Comm* c, double* grid)
displs,
c->rbufferTypes,
c->comm);
}
void commExchangeInt(Comm* c, int* grid)
{
int counts[6] = { 1, 1, 1, 1, 1, 1 };
MPI_Aint displs[6] = { 0, 0, 0, 0, 0, 0 };
MPI_Neighbor_alltoallw(grid,
counts,
displs,
c->sbufferTypesInt,
grid,
counts,
displs,
c->rbufferTypesInt,
c->comm);
#endif
}
void commShift(Comm* c, double* f, double* g, double* h)
{
#if defined(_MPI)
MPI_Request requests[6] = { MPI_REQUEST_NULL,
MPI_REQUEST_NULL,
MPI_REQUEST_NULL,
@@ -314,6 +290,31 @@ void commShift(Comm* c, double* f, double* g, double* h)
MPI_Isend(h, 1, c->sbufferTypes[BACK], c->neighbours[BACK], 2, c->comm, &requests[5]);
MPI_Waitall(6, requests, MPI_STATUSES_IGNORE);
#endif
}
void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax)
{
#if defined(_MPI)
int sum = 0;
for (int i = 0; i < c->coords[ICORD]; i++) {
sum += sizeOfRank(i, c->dims[ICORD], imax);
}
offsets[IDIM] = sum;
sum = 0;
for (int i = 0; i < c->coords[JCORD]; i++) {
sum += sizeOfRank(i, c->dims[JCORD], jmax);
}
offsets[JDIM] = sum;
sum = 0;
for (int i = 0; i < c->coords[KCORD]; i++) {
sum += sizeOfRank(i, c->dims[KCORD], kmax);
}
offsets[KDIM] = sum;
#endif
}
#define G(v, i, j, k) \
@@ -336,6 +337,7 @@ void commCollectResult(Comm* c,
int jmaxLocal = c->jmaxLocal;
int kmaxLocal = c->kmaxLocal;
#if defined(_MPI)
int offset[c->size * NDIMS];
int imaxLocalAll[c->size];
int jmaxLocalAll[c->size];
@@ -458,10 +460,52 @@ void commCollectResult(Comm* c,
imax);
free(tmp);
#else
int idx = 0;
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
pg[idx++] = G(p, i, j, k);
}
}
}
idx = 0;
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
ug[idx++] = (G(u, i, j, k) + G(u, i - 1, j, k)) / 2.0;
}
}
}
idx = 0;
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
vg[idx++] = (G(v, i, j, k) + G(v, i, j - 1, k)) / 2.0;
}
}
}
idx = 0;
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
wg[idx++] = (G(w, i, j, k) + G(w, i, j, k - 1)) / 2.0;
}
}
}
#endif
}
void commPrintConfig(Comm* c)
{
#if defined(_MPI)
fflush(stdout);
MPI_Barrier(MPI_COMM_WORLD);
if (commIsMaster(c)) {
@@ -491,13 +535,24 @@ void commPrintConfig(Comm* c)
}
}
MPI_Barrier(MPI_COMM_WORLD);
#endif
}
void commInit(Comm* c, int kmax, int jmax, int imax)
void commInit(Comm* c, int argc, char** argv)
{
/* setup communication */
#if defined(_MPI)
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
#else
c->rank = 0;
c->size = 1;
#endif
}
void commPartition(Comm* c, int kmax, int jmax, int imax)
{
#if defined(_MPI)
int dims[NDIMS] = { 0, 0, 0 };
int periods[NDIMS] = { 0, 0, 0 };
MPI_Dims_create(c->size, NDIMS, dims);
@@ -507,18 +562,9 @@ void commInit(Comm* c, int kmax, int jmax, int imax)
MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]);
MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords);
c->imaxLocal = sizeOfRank(c->coords[KDIM], dims[ICORD], imax);
// printf("\nRank : %d\nimaxLocal : %d -> c->coords[IDIM] : %d , dims[ICORD] : %d,
// imax : %d\n", c->rank, c->imaxLocal, c->coords[IDIM], dims[ICORD], imax);
c->jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JCORD], jmax);
// printf("jmaxLocal : %d -> c->coords[JDIM] : %d , dims[JCORD] : %d, imax : %d\n",
// c->jmaxLocal, c->coords[JDIM], dims[JCORD], jmax);
c->kmaxLocal = sizeOfRank(c->coords[IDIM], dims[KCORD], kmax);
// printf("kmaxLocal : %d -> c->coords[KDIM] : %d , dims[KCORD] : %d, imax : %d\n",
// c->kmaxLocal, c->coords[KDIM], dims[KCORD], kmax);
// sizeOfRank(int rank, int size, int N)
// { return N / size + ((N % size > rank) ? 1 : 0); }
c->imaxLocal = sizeOfRank(c->rank, dims[ICORD], imax);
c->jmaxLocal = sizeOfRank(c->rank, dims[JCORD], jmax);
c->kmaxLocal = sizeOfRank(c->rank, dims[KCORD], kmax);
// setup buffer types for communication
setupCommunication(c, LEFT, BULK);
@@ -533,12 +579,21 @@ void commInit(Comm* c, int kmax, int jmax, int imax)
setupCommunication(c, FRONT, HALO);
setupCommunication(c, BACK, BULK);
setupCommunication(c, BACK, HALO);
#else
c->imaxLocal = imax;
c->jmaxLocal = jmax;
c->kmaxLocal = kmax;
#endif
}
void commFree(Comm* c)
void commFinalize(Comm* c)
{
#if defined(_MPI)
for (int i = 0; i < NDIRS; i++) {
MPI_Type_free(&c->sbufferTypes[i]);
MPI_Type_free(&c->rbufferTypes[i]);
}
MPI_Finalize();
#endif
}

17
BasicSolver/3D-mpi/src/comm.h
View File

@@ -1,12 +1,14 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#ifndef __COMM_H_
#define __COMM_H_
#if defined(_MPI)
#include <mpi.h>
#endif
/*
* Spatial directions:
* ICORD (0) from 0 (LEFT) to imax (RIGHT)
@@ -24,26 +26,25 @@ enum op { MAX = 0, SUM };
typedef struct {
int rank;
int size;
#if defined(_MPI)
MPI_Comm comm;
MPI_Datatype sbufferTypes[NDIRS];
MPI_Datatype rbufferTypes[NDIRS];
MPI_Datatype sbufferTypesInt[NDIRS];
MPI_Datatype rbufferTypesInt[NDIRS];
#endif
int neighbours[NDIRS];
int coords[NDIMS], dims[NDIMS];
int imaxLocal, jmaxLocal, kmaxLocal;
MPI_File fh;
} Comm;
extern void commInit(Comm* comm, int kmax, int jmax, int imax);
extern void commFree(Comm* comm);
extern void commInit(Comm* c, int argc, char** argv);
extern void commPartition(Comm* c, int kmax, int jmax, int imax);
extern void commFinalize(Comm* comm);
extern void commPrintConfig(Comm*);
extern void commExchange(Comm*, double*);
extern void commExchangeInt(Comm*, int*);
extern void commShift(Comm* c, double* f, double* g, double* h);
extern void commReduction(double* v, int op);
extern int commIsBoundary(Comm* c, Direction direction);
extern void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax);
extern void commCollectResult(Comm* c,
double* ug,
double* vg,

2
BasicSolver/3D-mpi/src/grid.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.

148
BasicSolver/3D-mpi/src/main.c
View File

@@ -1,122 +1,88 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
*/
#include <float.h>
#include <limits.h>
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include "allocate.h"
#include "comm.h"
#include "parameter.h"
#include "particletracing.h"
#include "progress.h"
#include "solver.h"
#include "test.h"
#include "timing.h"
#include "vtkWriter.h"
enum VARIANT { SOR = 1, RB, RBA };
int main(int argc, char** argv)
{
int rank;
int variant = RB;
double timeStart, timeStop;
Parameter params;
ParticleTracer particletracer;
Solver solver;
Parameter p;
Solver s;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
initParameter(&params);
commInit(&s.comm, argc, argv);
initParameter(&p);
if (argc < 2) {
if (argc != 2) {
printf("Usage: %s <configFile>\n", argv[0]);
exit(EXIT_SUCCESS);
}
readParameter(&params, argv[1]);
if (argc == 3) {
variant = atoi(argv[2]);
readParameter(&p, argv[1]);
commPartition(&s.comm, p.kmax, p.jmax, p.imax);
if (commIsMaster(&s.comm)) {
printParameter(&p);
}
if (commIsMaster(&solver.comm)) {
printParameter(&params);
}
initSolver(&solver, &params);
initParticleTracer(&particletracer, &params, &solver);
initSolver(&s, &p);
#ifndef VERBOSE
initProgress(solver.te);
initProgress(s.te);
#endif
// printf("Rank : %d, imaxLocal : %d, jmaxLocal : %d, kmaxLocal : %d\n", solver.comm.rank, solver.comm.imaxLocal, solver.comm.jmaxLocal, solver.comm.kmaxLocal);
// if(solver.comm.rank == 1) printGrid(&solver, solver.seg);
// exit(0);
printf("initSolver done");
printParticleTracerParameters(&particletracer);
double tau = solver.tau;
double te = solver.te;
double tau = s.tau;
double te = s.te;
double t = 0.0;
int nt = 0;
void (*solver_generic[])() = { solve, solveRB, solveRBA };
timeStart = getTimeStamp();
while (t <= te) {
if (tau > 0.0) computeTimestep(&solver);
setBoundaryConditions(&solver);
setSpecialBoundaryCondition(&solver);
setObjectBoundaryCondition(&solver);
computeFG(&solver);
computeRHS(&solver);
// if (nt % 100 == 0) normalizePressure(&solver);
solver_generic[variant - 1](&solver);
adaptUV(&solver);
trace(&particletracer, solver.u, solver.v, solver.w, solver.seg, t);
t += solver.dt;
nt++;
if (tau > 0.0) computeTimestep(&s);
setBoundaryConditions(&s);
setSpecialBoundaryCondition(&s);
computeFG(&s);
computeRHS(&s);
solve(&s);
adaptUV(&s);
t += s.dt;
#ifdef VERBOSE
if (rank == 0) {
printf("TIME %f , TIMESTEP %f\n", t, solver.dt);
if (commIsMaster(&s.comm)) {
printf("TIME %f , TIMESTEP %f\n", t, s.dt);
}
#else
printProgress(t);
#endif
}
timeStop = getTimeStamp();
#ifndef VERBOSE
stopProgress();
if (commIsMaster(&solver.comm)) {
#endif
if (commIsMaster(&s.comm)) {
printf("Solution took %.2fs\n", timeStop - timeStart);
}
// testInit(&solver);
// commExchange(&solver.comm, solver.p);
// testPrintHalo(&solver, solver.p);
timeStart = getTimeStamp();
#ifdef _VTK_WRITER_MPI
VtkOptions opts = { .grid = s.grid, .comm = s.comm };
vtkOpen(&opts, s.problem);
vtkScalar(&opts, "pressure", s.p);
vtkVector(&opts, "velocity", (VtkVector) { s.u, s.v, s.w });
vtkClose(&opts);
#else
double *pg, *ug, *vg, *wg;
if (commIsMaster(&solver.comm)) {
size_t bytesize = solver.grid.imax * solver.grid.jmax * solver.grid.kmax *
sizeof(double);
if (commIsMaster(&s.comm)) {
size_t bytesize = s.grid.imax * s.grid.jmax * s.grid.kmax * sizeof(double);
pg = allocate(64, bytesize);
ug = allocate(64, bytesize);
@@ -124,26 +90,34 @@ int main(int argc, char** argv)
wg = allocate(64, bytesize);
}
commCollectResult(&solver.comm,
commCollectResult(&s.comm,
ug,
vg,
wg,
pg,
solver.u,
solver.v,
solver.w,
solver.p,
solver.grid.kmax,
solver.grid.jmax,
solver.grid.imax);
s.u,
s.v,
s.w,
s.p,
s.grid.kmax,
s.grid.jmax,
s.grid.imax);
VtkOptions opts = { .grid = solver.grid, .comm = solver.comm };
vtkOpen(&opts, solver.problem);
vtkScalar(&opts, "pressure", pg);
vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg });
vtkClose(&opts);
if (commIsMaster(&s.comm)) {
VtkOptions opts = { .grid = s.grid };
vtkOpen(&opts, s.problem);
vtkScalar(&opts, "pressure", pg);
vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg });
vtkClose(&opts);
}
#endif
commFree(&solver.comm);
MPI_Finalize();
timeStop = getTimeStamp();
if (commIsMaster(&s.comm)) {
printf("Result output took %.2fs\n", timeStop - timeStart);
}
commFinalize(&s.comm);
return EXIT_SUCCESS;
}

26
BasicSolver/3D-mpi/src/parameter.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -26,7 +26,6 @@ void initParameter(Parameter* param)
param->re = 100.0;
param->gamma = 0.9;
param->tau = 0.5;
param->rho = 0.99;
}
void readParameter(Parameter* param, const char* filename)
@@ -87,29 +86,6 @@ void readParameter(Parameter* param, const char* filename)
PARSE_REAL(v_init);
PARSE_REAL(w_init);
PARSE_REAL(p_init);
PARSE_REAL(rho);
/* Added new particle tracing parameters */
PARSE_INT(numberOfParticles);
PARSE_REAL(startTime);
PARSE_REAL(injectTimePeriod);
PARSE_REAL(writeTimePeriod);
PARSE_REAL(x1);
PARSE_REAL(y1);
PARSE_REAL(z1);
PARSE_REAL(x2);
PARSE_REAL(y2);
PARSE_REAL(z2);
/* Added obstacle geometry parameters */
PARSE_INT(shape);
PARSE_REAL(xCenter);
PARSE_REAL(yCenter);
PARSE_REAL(zCenter);
PARSE_REAL(xRectLength);
PARSE_REAL(yRectLength);
PARSE_REAL(zRectLength);
PARSE_REAL(circleRadius);
}
}

12
BasicSolver/3D-mpi/src/parameter.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -11,21 +11,13 @@ typedef struct {
int imax, jmax, kmax;
double xlength, ylength, zlength;
int itermax;
double eps, omg, rho;
double eps, omg;
double re, tau, gamma;
double te, dt;
double gx, gy, gz;
char* name;
int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack;
double u_init, v_init, w_init, p_init;
int numberOfParticles;
double startTime, injectTimePeriod, writeTimePeriod;
double x1, y1, z1, x2, y2, z2;
int shape;
double xCenter, yCenter, zCenter, xRectLength, yRectLength, zRectLength, circleRadius;
} Parameter;
void initParameter(Parameter*);

729
BasicSolver/3D-mpi/src/particletracing.c
View File

@@ -1,729 +0,0 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <float.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "particletracing.h"
#include "solver.h"
#define U(i, j, k) \
u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define V(i, j, k) \
v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define W(i, j, k) \
w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
#define S(i, j, k) \
seg[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
static int ts = 0;
#define XOFFSET 0
#define YOFFSET 1
#define ZOFFSET 2
#define XOFFSETEND 3
#define YOFFSETEND 4
#define ZOFFSETEND 5
static double sum(int* sizes, int size)
{
double sum = 0;
for (int i = 0; i < size; ++i) {
sum += sizes[i];
}
return sum;
}
static double sumOffset(double* sizes, int init, int offset, int coord)
{
double sum = 0;
for (int i = init - offset; coord > 0; i -= offset, --coord) {
sum += sizes[i];
}
return sum;
}
void printParticles(ParticleTracer* particletracer)
{
for (int i = 0; i < particletracer->totalParticles; ++i) {
printf("Rank : %d Particle position X : %.2f, Y : %.2f, flag : %d, total pt : "
"%d, pointer : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, "
"yOffsetEnd : %.2f\n",
particletracer->rank,
particletracer->particlePool[i].x,
particletracer->particlePool[i].y,
particletracer->particlePool[i].flag,
particletracer->totalParticles,
particletracer->pointer,
particletracer->xOffset,
particletracer->yOffset,
particletracer->xOffsetEnd,
particletracer->yOffsetEnd);
}
}
void injectParticles(ParticleTracer* particletracer, int* restrict seg)
{
double x, y, z;
int imaxLocal = particletracer->imaxLocal;
int jmaxLocal = particletracer->jmaxLocal;
int kmaxLocal = particletracer->kmaxLocal;
for (int i = 0; i < particletracer->numberOfParticles; ++i) {
x = particletracer->linSpaceLine[i].x;
y = particletracer->linSpaceLine[i].y;
z = particletracer->linSpaceLine[i].z;
if (x >= particletracer->xOffset && y >= particletracer->yOffset &&
z >= particletracer->zOffset && x <= particletracer->xOffsetEnd &&
y <= particletracer->yOffsetEnd && y <= particletracer->zOffsetEnd) {
// printf("\nRank : %d\n", particletracer->rank);
// printf("\t%.2f >= %.2f && %.2f >= %.2f && %.2f <= %.2f && %.2f <= %.2f\n",x
// , particletracer->xOffset ,y , particletracer->yOffset, x ,
// particletracer->xOffsetEnd ,y , particletracer->yOffsetEnd);
particletracer->particlePool[particletracer->pointer].x = x;
particletracer->particlePool[particletracer->pointer].y = y;
particletracer->particlePool[particletracer->pointer].z = z;
int i = particletracer->particlePool[particletracer->pointer].x /
particletracer->dx;
int j = particletracer->particlePool[particletracer->pointer].y /
particletracer->dy;
int k = particletracer->particlePool[particletracer->pointer].z /
particletracer->dz;
if (S(i, j, k) == NONE) {
particletracer->particlePool[particletracer->pointer].flag = true;
++(particletracer->pointer);
++(particletracer->totalParticles);
}
}
}
}
void advanceParticles(ParticleTracer* particletracer,
double* restrict u,
double* restrict v,
double* restrict w,
int* restrict seg,
double time)
{
int imax = particletracer->imax;
int jmax = particletracer->jmax;
int kmax = particletracer->kmax;
int imaxLocal = particletracer->imaxLocal;
int jmaxLocal = particletracer->jmaxLocal;
int kmaxLocal = particletracer->kmaxLocal;
double dx = particletracer->dx;
double dy = particletracer->dy;
double dz = particletracer->dz;
double xlength = particletracer->xlength;
double ylength = particletracer->ylength;
double zlength = particletracer->zlength;
Particle buff[particletracer->size][4000];
for (int i = 0; i < particletracer->size; ++i) {
for (int j = 0; j < 4000; ++j) {
buff[i][j].x = 0.0;
buff[i][j].y = 0.0;
buff[i][j].z = 0.0;
buff[i][j].flag = false;
}
}
int particleBufIndex[particletracer->size];
memset(particleBufIndex, 0, sizeof(particleBufIndex));
for (int i = 0; i < particletracer->totalParticles; ++i) {
if (particletracer->particlePool[i].flag == true) {
double xTemp = particletracer->particlePool[i].x;
double yTemp = particletracer->particlePool[i].y;
double zTemp = particletracer->particlePool[i].z;
double x = xTemp - particletracer->xOffset;
double y = yTemp - particletracer->yOffset;
double z = zTemp - particletracer->zOffset;
int iCoord = (int)(x / dx) + 1;
int jCoord = (int)((y + 0.5 * dy) / dy) + 1;
int kCoord = (int)((z + 0.5 * dz) / dz) + 1;
double x1 = (double)(iCoord - 1) * dx;
double y1 = ((double)(jCoord - 1) - 0.5) * dy;
double z1 = ((double)(kCoord - 1) - 0.5) * dz;
double x2 = (double)iCoord * dx;
double y2 = ((double)jCoord - 0.5) * dy;
double z2 = ((double)kCoord - 0.5) * dz;
double u_n =
(1.0 / (dx * dy * dz)) *
((x2 - x) * (y2 - y) * (z2 - z) * U(iCoord - 1, jCoord - 1, kCoord - 1) +
(x - x1) * (y2 - y) * (z2 - z) * U(iCoord, jCoord - 1, kCoord - 1) +
(x2 - x) * (y - y1) * (z2 - z) * U(iCoord - 1, jCoord, kCoord - 1) +
(x - x1) * (y - y1) * (z2 - z) * U(iCoord, jCoord, kCoord - 1) +
(x2 - x) * (y2 - y) * (z - z1) * U(iCoord - 1, jCoord - 1, kCoord) +
(x - x1) * (y2 - y) * (z - z1) * U(iCoord, jCoord - 1, kCoord) +
(x2 - x) * (y - y1) * (z - z1) * U(iCoord - 1, jCoord, kCoord) +
(x - x1) * (y - y1) * (z - z1) * U(iCoord, jCoord, kCoord));
double new_x = (x + particletracer->xOffset) + particletracer->dt * u_n;
particletracer->particlePool[i].x = new_x;
iCoord = (int)((x + 0.5 * dx) / dx) + 1;
jCoord = (int)(y / dy) + 1;
kCoord = (int)((z + 0.5 * dz) / dz) + 1;
x1 = ((double)(iCoord - 1) - 0.5) * dx;
y1 = (double)(jCoord - 1) * dy;
z1 = ((double)(kCoord - 1) - 0.5) * dz;
x2 = ((double)iCoord - 0.5) * dx;
y2 = (double)jCoord * dy;
z2 = ((double)kCoord - 0.5) * dz;
double v_n =
(1.0 / (dx * dy * dz)) *
((x2 - x) * (y2 - y) * (z2 - z) * V(iCoord - 1, jCoord - 1, kCoord - 1) +
(x - x1) * (y2 - y) * (z2 - z) * V(iCoord, jCoord - 1, kCoord - 1) +
(x2 - x) * (y - y1) * (z2 - z) * V(iCoord - 1, jCoord, kCoord - 1) +
(x - x1) * (y - y1) * (z2 - z) * V(iCoord, jCoord, kCoord - 1) +
(x2 - x) * (y2 - y) * (z - z1) * V(iCoord - 1, jCoord - 1, kCoord) +
(x - x1) * (y2 - y) * (z - z1) * V(iCoord, jCoord - 1, kCoord) +
(x2 - x) * (y - y1) * (z - z1) * V(iCoord - 1, jCoord, kCoord) +
(x - x1) * (y - y1) * (z - z1) * V(iCoord, jCoord, kCoord));
double new_y = (y + particletracer->yOffset) + particletracer->dt * v_n;
particletracer->particlePool[i].y = new_y;
iCoord = (int)((x + 0.5 * dx) / dx) + 1;
jCoord = (int)((y + 0.5 * dy) / dy) + 1;
kCoord = (int)(z / dz) + 1;
x1 = ((double)(iCoord - 1) - 0.5) * dx;
y1 = ((double)(jCoord - 1) - 0.5) * dy;
z1 = (double)(kCoord - 1) * dz;
x2 = ((double)iCoord - 0.5) * dx;
y2 = ((double)jCoord - 0.5) * dy;
z2 = (double)kCoord * dz;
double w_n =
(1.0 / (dx * dy * dz)) *
((x2 - x) * (y2 - y) * (z2 - z) * W(iCoord - 1, jCoord - 1, kCoord - 1) +
(x - x1) * (y2 - y) * (z2 - z) * W(iCoord, jCoord - 1, kCoord - 1) +
(x2 - x) * (y - y1) * (z2 - z) * W(iCoord - 1, jCoord, kCoord - 1) +
(x - x1) * (y - y1) * (z2 - z) * W(iCoord, jCoord, kCoord - 1) +
(x2 - x) * (y2 - y) * (z - z1) * W(iCoord - 1, jCoord - 1, kCoord) +
(x - x1) * (y2 - y) * (z - z1) * W(iCoord, jCoord - 1, kCoord) +
(x2 - x) * (y - y1) * (z - z1) * W(iCoord - 1, jCoord, kCoord) +
(x - x1) * (y - y1) * (z - z1) * W(iCoord, jCoord, kCoord));
double new_z = (z + particletracer->zOffset) + particletracer->dt * w_n;
particletracer->particlePool[i].z = new_z;
// printf("Rank : %d\n", particletracer->rank);
// printf("\tOld X : %.2f, translated X : %.2f, xOffset : %.2f, New X : %.2f,
// iCoord : %d\n\tOld Y : %.2f, translated X : %.2f, yOffset : %.2f, New Y :
// %.2f, jCoord : %d\n\n",xTemp, x, particletracer->xOffset, new_x, iCoord,
// yTemp, y, particletracer->yOffset , new_y, jCoord); printf("\tU(iCoord - 1,
// jCoord - 1) : %.2f, U(iCoord, jCoord - 1) : %.2f, U(iCoord - 1, jCoord) :
// %.2f, U(iCoord, jCoord) : %.2f\n", U(iCoord - 1, jCoord - 1), U(iCoord,
// jCoord - 1), U(iCoord - 1, jCoord), U(iCoord, jCoord)); printf("\tV(iCoord
// - 1, jCoord - 1) : %.2f, V(iCoord, jCoord - 1) : %.2f, V(iCoord - 1,
// jCoord) : %.2f, V(iCoord, jCoord) : %.2f\n\n", V(iCoord - 1, jCoord - 1),
// V(iCoord, jCoord - 1), V(iCoord - 1, jCoord), V(iCoord, jCoord));
// printf("\t U N : %.2f, V N : %.2f\n\n", u_n, v_n);
if (((new_x < particletracer->xOffset) ||
(new_x >= particletracer->xOffsetEnd) ||
(new_y < particletracer->yOffset) ||
(new_y >= particletracer->yOffsetEnd) ||
(new_z < particletracer->zOffset) ||
(new_z >= particletracer->zOffsetEnd))) {
// New logic to transfer particles to neighbouring ranks or discard the
// particle.
for (int i = 0; i < particletracer->size; ++i) {
if ((new_x >=
particletracer->offset[i + particletracer->size * XOFFSET]) &&
(new_x <= particletracer
->offset[i + particletracer->size * XOFFSETEND]) &&
(new_y >=
particletracer->offset[i + particletracer->size * YOFFSET]) &&
(new_y <= particletracer
->offset[i + particletracer->size * YOFFSETEND]) &&
(new_z >=
particletracer->offset[i + particletracer->size * ZOFFSET]) &&
(new_z <= particletracer
->offset[i + particletracer->size * ZOFFSETEND]) &&
i != particletracer->rank) {
buff[i][particleBufIndex[i]].x = new_x;
buff[i][particleBufIndex[i]].y = new_y;
buff[i][particleBufIndex[i]].z = new_z;
buff[i][particleBufIndex[i]].flag = true;
++particleBufIndex[i];
}
}
particletracer->particlePool[i].flag = false;
int i_new = new_x / dx, j_new = new_y / dy, k_new = new_z / dz;
int iOffset = particletracer->xOffset / dx,
jOffset = particletracer->yOffset / dy,
kOffset = particletracer->zOffset / dz;
if (S(i_new - iOffset, j_new - jOffset, k_new - kOffset) != NONE) {
particletracer->particlePool[i].flag = false;
}
}
}
}
for (int i = 0; i < particletracer->size; ++i) {
if (i != particletracer->rank) {
MPI_Send(buff[i],
4000,
particletracer->mpi_particle,
i,
0,
particletracer->comm);
}
}
for (int i = 0; i < particletracer->size; ++i) {
if (i != particletracer->rank) {
MPI_Recv(buff[i],
4000,
particletracer->mpi_particle,
i,
0,
particletracer->comm,
MPI_STATUS_IGNORE);
}
}
for (int i = 0; i < particletracer->size; ++i) {
if (i != particletracer->rank) {
for (int j = 0; j < 4000; ++j) {
if (buff[i][j].flag == true) {
particletracer->particlePool[particletracer->pointer].x = buff[i][j]
.x;
particletracer->particlePool[particletracer->pointer].y = buff[i][j]
.y;
particletracer->particlePool[particletracer->pointer].z = buff[i][j]
.z;
particletracer->particlePool[particletracer->pointer].flag = true;
++(particletracer->pointer);
++(particletracer->totalParticles);
}
}
}
}
}
void freeParticles(ParticleTracer* particletracer)
{
free(particletracer->particlePool);
free(particletracer->linSpaceLine);
free(particletracer->offset);
}
void writeParticles(ParticleTracer* particletracer)
{
int collectedBuffIndex[particletracer->size];
MPI_Gather(&particletracer->totalParticles,
1,
MPI_INT,
collectedBuffIndex,
1,
MPI_INT,
0,
particletracer->comm);
if (particletracer->rank != 0) {
Particle buff[particletracer->totalParticles];
for (int i = 0; i < particletracer->totalParticles; ++i) {
buff[i].x = particletracer->particlePool[i].x;
buff[i].y = particletracer->particlePool[i].y;
buff[i].z = particletracer->particlePool[i].z;
buff[i].flag = particletracer->particlePool[i].flag;
// printf("Rank : %d sending to rank 0 X : %.2f, Y : %.2f with totalpt :
// %d\n", particletracer->rank, buff[i].x, buff[i].y,
// particletracer->totalParticles);
}
MPI_Send(buff,
particletracer->totalParticles,
particletracer->mpi_particle,
0,
1,
particletracer->comm);
}
if (particletracer->rank == 0) {
char filename[50];
FILE* fp;
snprintf(filename, 50, "vtk_files/particles_%d.vtk", ts);
fp = fopen(filename, "w");
if (fp == NULL) {
printf("Error!\n");
exit(EXIT_FAILURE);
}
fprintf(fp, "# vtk DataFile Version 3.0\n");
fprintf(fp, "PAMPI cfd solver particle tracing file\n");
fprintf(fp, "ASCII\n");
fprintf(fp, "DATASET UNSTRUCTURED_GRID\n");
fprintf(fp, "FIELD FieldData 2\n");
fprintf(fp, "TIME 1 1 double\n");
fprintf(fp, "%d\n", ts);
fprintf(fp, "CYCLE 1 1 int\n");
fprintf(fp, "1\n");
int overallTotalParticles = sum(collectedBuffIndex, particletracer->size);
fprintf(fp, "POINTS %d float\n", overallTotalParticles);
printf("Total particles : %d\n", overallTotalParticles);
for (int i = 1; i < particletracer->size; ++i) {
Particle recvBuff[collectedBuffIndex[i]];
MPI_Recv(&recvBuff,
collectedBuffIndex[i],
particletracer->mpi_particle,
i,
1,
particletracer->comm,
MPI_STATUS_IGNORE);
for (int j = 0; j < collectedBuffIndex[i]; ++j) {
double x = recvBuff[j].x;
double y = recvBuff[j].y;
double z = recvBuff[j].z;
fprintf(fp, "%f %f %f\n", x, y, z);
// printf("Rank : 0 receiving from rank %d X : %.2f, Y : %.2f with totalpt
// : %d\n", i, x, y, particletracer->totalParticles);
}
}
for (int i = 0; i < particletracer->totalParticles; ++i) {
double x = particletracer->particlePool[i].x;
double y = particletracer->particlePool[i].y;
double z = particletracer->particlePool[i].z;
fprintf(fp, "%f %f %f\n", x, y, z);
}
fprintf(fp, "CELLS %d %d\n", overallTotalParticles, 2 * overallTotalParticles);
for (int i = 0; i < overallTotalParticles; ++i) {
fprintf(fp, "1 %d\n", i);
}
fprintf(fp, "CELL_TYPES %d\n", overallTotalParticles);
for (int i = 0; i < overallTotalParticles; ++i) {
fprintf(fp, "1\n");
}
fclose(fp);
}
++ts;
}
void initParticleTracer(ParticleTracer* particletracer, Parameter* params, Solver* solver)
{
/* initializing local properties from params */
particletracer->numberOfParticles = params->numberOfParticles;
particletracer->startTime = params->startTime;
particletracer->injectTimePeriod = params->injectTimePeriod;
particletracer->writeTimePeriod = params->writeTimePeriod;
particletracer->dt = params->dt;
particletracer->dx = params->xlength / params->imax;
particletracer->dy = params->ylength / params->jmax;
particletracer->dz = params->zlength / params->kmax;
particletracer->xlength = params->xlength;
particletracer->ylength = params->ylength;
particletracer->zlength = params->zlength;
particletracer->x1 = params->x1;
particletracer->y1 = params->y1;
particletracer->z1 = params->z1;
particletracer->x2 = params->x2;
particletracer->y2 = params->y2;
particletracer->z2 = params->z2;
particletracer->lastInjectTime = params->startTime;
particletracer->lastUpdateTime = params->startTime;
particletracer->lastWriteTime = params->startTime;
particletracer->pointer = 0;
particletracer->totalParticles = 0;
particletracer->imax = params->imax;
particletracer->jmax = params->jmax;
particletracer->kmax = params->kmax;
particletracer->imaxLocal = solver->comm.imaxLocal;
particletracer->jmaxLocal = solver->comm.jmaxLocal;
particletracer->kmaxLocal = solver->comm.kmaxLocal;
particletracer->estimatedNumParticles = (particletracer->imaxLocal *
particletracer->jmaxLocal *
particletracer->kmaxLocal);
particletracer->particlePool = malloc(
sizeof(Particle) * particletracer->estimatedNumParticles);
for (int i = 0; i < particletracer->estimatedNumParticles; ++i) {
particletracer->particlePool[i].x = 0.0;
particletracer->particlePool[i].y = 0.0;
particletracer->particlePool[i].z = 0.0;
particletracer->particlePool[i].flag = false;
}
particletracer->linSpaceLine = malloc(
sizeof(Particle) * particletracer->numberOfParticles);
/* duplicating communication from solver */
MPI_Comm_dup(solver->comm.comm, &particletracer->comm);
particletracer->rank = solver->comm.rank;
particletracer->size = solver->comm.size;
particletracer->offset = (double*)malloc(sizeof(double) * 4 * particletracer->size);
memcpy(particletracer->dims, solver->comm.dims, sizeof(solver->comm.dims));
memcpy(particletracer->coords, solver->comm.coords, sizeof(solver->comm.coords));
double offset[6][particletracer->size];
particletracer->xOffset = solver->xOffset;
particletracer->yOffset = solver->yOffset;
particletracer->zOffset = solver->zOffset;
particletracer->xOffsetEnd = solver->xOffsetEnd;
particletracer->yOffsetEnd = solver->yOffsetEnd;
particletracer->zOffsetEnd = solver->zOffsetEnd;
// printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, zOffset : %.2f, xOffsetEnd :
// %.2f, yOffsetEnd : %.2f, zOffsetEnd : %.2f\n", particletracer->rank,
// particletracer->xOffset, particletracer->yOffset, particletracer->zOffset,
// particletracer->xOffsetEnd, particletracer->yOffsetEnd,
// particletracer->zOffsetEnd);
MPI_Allgather(&particletracer->xOffset,
1,
MPI_DOUBLE,
offset[0],
1,
MPI_DOUBLE,
particletracer->comm);
MPI_Allgather(&particletracer->yOffset,
1,
MPI_DOUBLE,
offset[1],
1,
MPI_DOUBLE,
particletracer->comm);
MPI_Allgather(&particletracer->zOffset,
1,
MPI_DOUBLE,
offset[2],
1,
MPI_DOUBLE,
particletracer->comm);
MPI_Allgather(&particletracer->xOffsetEnd,
1,
MPI_DOUBLE,
offset[3],
1,
MPI_DOUBLE,
particletracer->comm);
MPI_Allgather(&particletracer->yOffsetEnd,
1,
MPI_DOUBLE,
offset[4],
1,
MPI_DOUBLE,
particletracer->comm);
MPI_Allgather(&particletracer->zOffsetEnd,
1,
MPI_DOUBLE,
offset[5],
1,
MPI_DOUBLE,
particletracer->comm);
memcpy(particletracer->offset, offset, sizeof(offset));
// if(particletracer->rank == 0)
// {
// for(int i = 0;i < particletracer->size; ++i)
// {
// printf("Rank : %d, xLocal : %.2f, yLocal : %.2f, zLocal : %.2f\n", i,
// xLocal[i], yLocal[i], zLocal[i]);
// }
// for(int i = 0;i < particletracer->size; ++i)
// {
// printf("Rank : %d and its xOffset : %.2f, yOffset : %.2f, zOffset :
// %.2f,xOffsetEnd : %.2f, yOffsetEnd : %.2f, zOffsetEnd : %.2f\n", i,
// particletracer->offset[i + particletracer->size * XOFFSET],
// particletracer->offset[i + particletracer->size * YOFFSET],
// particletracer->offset[i + particletracer->size * ZOFFSET] ,
// particletracer->offset[i + particletracer->size * XOFFSETEND],
// particletracer->offset[i + particletracer->size * YOFFSETEND],
// particletracer->offset[i + particletracer->size * ZOFFSETEND]);
// }
// }
for (int i = 0; i < particletracer->numberOfParticles; ++i) {
// double spacing = (double)i /
// (double)(particletracer->numberOfParticles - 1);
// particletracer->linSpaceLine[i].x = spacing * particletracer->x1 + (1.0 -
// spacing) * particletracer->x2; particletracer->linSpaceLine[i].y = spacing *
// particletracer->y1 + (1.0 - spacing) * particletracer->y2;
// particletracer->linSpaceLine[i].z = spacing * particletracer->z1 + (1.0 -
// spacing) * particletracer->z2;
particletracer->linSpaceLine[i].x = particletracer->x1;
particletracer->linSpaceLine[i].y = (((double)rand() /
RAND_MAX) *
(particletracer->y2 - particletracer->y1)) +
particletracer->y1;
particletracer->linSpaceLine[i].z = (((double)rand() /
RAND_MAX) *
(particletracer->z2 - particletracer->z1)) +
particletracer->z1;
particletracer->linSpaceLine[i].flag = true;
}
// Create the mpi_particle datatype
MPI_Datatype mpi_particle;
int lengths[4] = { 1, 1, 1, 1 };
MPI_Aint displacements[4];
Particle dummy_particle;
MPI_Aint base_address;
MPI_Get_address(&dummy_particle, &base_address);
MPI_Get_address(&dummy_particle.x, &displacements[0]);
MPI_Get_address(&dummy_particle.y, &displacements[1]);
MPI_Get_address(&dummy_particle.z, &displacements[2]);
MPI_Get_address(&dummy_particle.flag, &displacements[3]);
displacements[0] = MPI_Aint_diff(displacements[0], base_address);
displacements[1] = MPI_Aint_diff(displacements[1], base_address);
displacements[2] = MPI_Aint_diff(displacements[2], base_address);
displacements[3] = MPI_Aint_diff(displacements[3], base_address);
MPI_Datatype types[4] = { MPI_DOUBLE, MPI_DOUBLE, MPI_DOUBLE, MPI_C_BOOL };
MPI_Type_create_struct(4,
lengths,
displacements,
types,
&particletracer->mpi_particle);
MPI_Type_commit(&particletracer->mpi_particle);
}
void printParticleTracerParameters(ParticleTracer* particletracer)
{
printf("Particle Tracing data:\n");
printf("Rank : %d\n", particletracer->rank);
printf("\tNumber of particles : %d being injected for every period of %.2f\n",
particletracer->numberOfParticles,
particletracer->injectTimePeriod);
printf("\tstartTime : %.2f\n", particletracer->startTime);
printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : "
"%.2f, z1 : %.2f, x2 : %.2f, y2 : %.2f, z2 : %.2f\n",
particletracer->x1,
particletracer->y1,
particletracer->z1,
particletracer->x2,
particletracer->y2,
particletracer->z2);
printf("\tPointer : %d, TotalParticles : %d\n",
particletracer->pointer,
particletracer->totalParticles);
printf("\tdt : %.2f, dx : %.2f, dy : %.2f, dz : %.2f\n",
particletracer->dt,
particletracer->dx,
particletracer->dy,
particletracer->dz);
printf("\tcoord[0] : %d, coord[1] : %d, coord[2] : %d\n",
particletracer->coords[IDIM],
particletracer->coords[JDIM],
particletracer->coords[KDIM]);
printf("\txOffset : %.2f, yOffset : %.2f, zOffset : %.2f\n",
particletracer->xOffset,
particletracer->yOffset,
particletracer->zOffset);
printf("\txOffsetEnd : %.2f, yOffsetEnd : %.2f, zOffsetEnd : %.2f\n",
particletracer->xOffsetEnd,
particletracer->yOffsetEnd,
particletracer->zOffsetEnd);
printf("\txLocal : %.2f, yLocal : %.2f, zLocal : %.2f\n",
particletracer->xLocal,
particletracer->yLocal,
particletracer->zLocal);
}
void trace(ParticleTracer* particletracer,
double* restrict u,
double* restrict v,
double* restrict w,
int* restrict seg,
double time)
{
if (time >= particletracer->startTime) {
if ((time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod) {
injectParticles(particletracer, seg);
particletracer->lastInjectTime = time;
}
if ((time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod) {
writeParticles(particletracer);
particletracer->lastWriteTime = time;
}
advanceParticles(particletracer, u, v, w, seg, time);
compress(particletracer);
particletracer->lastUpdateTime = time;
}
}
void compress(ParticleTracer* particletracer)
{
Particle* memPool = particletracer->particlePool;
Particle tempPool[particletracer->totalParticles];
for (int i = 0; i < particletracer->totalParticles; ++i) {
tempPool[i].x = 0.0;
tempPool[i].y = 0.0;
tempPool[i].z = 0.0;
tempPool[i].flag = false;
}
int totalParticles = 0;
for (int i = 0; i < particletracer->totalParticles; ++i) {
if (memPool[i].flag == true) {
tempPool[totalParticles].x = memPool[i].x;
tempPool[totalParticles].y = memPool[i].y;
tempPool[totalParticles].z = memPool[i].z;
tempPool[totalParticles].flag = memPool[i].flag;
++totalParticles;
}
}
particletracer->totalParticles = totalParticles;
particletracer->pointer = totalParticles;
memcpy(particletracer->particlePool, tempPool, totalParticles * sizeof(Particle));
}

62
BasicSolver/3D-mpi/src/particletracing.h
View File

@@ -1,62 +0,0 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#ifndef __PARTICLETRACING_H_
#define __PARTICLETRACING_H_
#include "allocate.h"
#include "parameter.h"
#include "solver.h"
#include <mpi.h>
#include <stdbool.h>
#define NDIMS 3
typedef enum COORD { X = 0, Y, NCOORD } COORD;
typedef struct {
double x, y, z;
bool flag;
} Particle;
typedef struct {
int numberOfParticles, totalParticles;
double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime,
lastWriteTime;
int estimatedNumParticles;
double dx, dy, dz, dt;
Particle* linSpaceLine;
Particle* particlePool;
int pointer;
double imax, jmax, kmax, xlength, ylength, zlength, imaxLocal, jmaxLocal, kmaxLocal;
double x1, y1, z1, x2, y2, z2;
MPI_Comm comm;
MPI_Datatype mpi_particle;
int rank, size;
int coords[NDIMS], dims[NDIMS];
double xLocal, yLocal, zLocal, xOffset, yOffset, zOffset, xOffsetEnd, yOffsetEnd,
zOffsetEnd;
double* offset;
} ParticleTracer;
void initParticleTracer(ParticleTracer*, Parameter*, Solver*);
void injectParticles(ParticleTracer*, int*);
void advanceParticles(ParticleTracer*, double*, double*, double*, int*, double);
void freeParticles(ParticleTracer*);
void writeParticles(ParticleTracer*);
void printParticleTracerParameters(ParticleTracer*);
void printParticles(ParticleTracer*);
void trace(ParticleTracer*, double*, double*, double*, int*, double);
void compress(ParticleTracer*);
#endif

3
BasicSolver/3D-mpi/src/progress.c
View File

@@ -1,12 +1,11 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "progress.h"

4
BasicSolver/3D-mpi/src/progress.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
@@ -9,6 +9,6 @@
extern void initProgress(double);
extern void printProgress(double);
extern void stopProgress();
extern void stopProgress(void);
#endif

1241
BasicSolver/3D-mpi/src/solver.c
View File

File diff suppressed because it is too large Load Diff

51
BasicSolver/3D-mpi/src/solver.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -10,46 +10,7 @@
#include "grid.h"
#include "parameter.h"
enum OBJECTBOUNDARY {
NONE = 0,
/* Front Corners */
FRONTTOPLEFTCORNER,
FRONTTOPRIGHTCORNER,
FRONTBOTTOMLEFTCORNER,
FRONTBOTTOMRIGHTCORNER,
/* Back Corners */
BACKTOPLEFTCORNER,
BACKTOPRIGHTCORNER,
BACKBOTTOMLEFTCORNER,
BACKBOTTOMRIGHTCORNER,
/* Faces */
FRONTFACE,
BACKFACE,
LEFTFACE,
RIGHTFACE,
TOPFACE,
BOTTOMFACE,
/* Front Lines remaining after Corners and Faces */
FRONTLEFTLINE,
FRONTRIGHTLINE,
FRONTTOPLINE,
FRONTBOTTOMLINE,
/* Bottom Lines remaining after Corners and Faces */
BACKLEFTLINE,
BACKRIGHTLINE,
BACKTOPLINE,
BACKBOTTOMLINE,
/* Mid Lines remaining after Corners and Faces */
MIDTOPLEFTLINE,
MIDTOPRIGHTLINE,
MIDBOTTOMLEFTLINE,
MIDBOTTOMRIGHTLINE,
/* Local where its an object but not a boundary */
LOCAL
};
enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC };
enum SHAPE { NOSHAPE = 0, RECT, CIRCLE };
typedef struct {
/* geometry and grid information */
@@ -58,11 +19,8 @@ typedef struct {
double *p, *rhs;
double *f, *g, *h;
double *u, *v, *w;
int* seg;
double xLocal, yLocal, zLocal, xOffset, yOffset, zOffset, xOffsetEnd, yOffsetEnd,
zOffsetEnd;
/* parameters */
double eps, omega, rho;
double eps, omega;
double re, tau, gamma;
double gx, gy, gz;
/* time stepping */
@@ -78,15 +36,10 @@ typedef struct {
extern void initSolver(Solver*, Parameter*);
extern void computeRHS(Solver*);
extern void solve(Solver*);
extern void solveRB(Solver*);
extern void solveRBA(Solver*);
extern void normalizePressure(Solver*);
extern void computeTimestep(Solver*);
extern void setBoundaryConditions(Solver*);
extern void setSpecialBoundaryCondition(Solver*);
extern void setObjectBoundaryCondition(Solver*);
extern void setObjectPBoundaryCondition(Solver*);
extern void printGrid(Solver*, int*);
extern void computeFG(Solver*);
extern void adaptUV(Solver*);
#endif

21
BasicSolver/3D-mpi/src/test.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
@@ -26,10 +26,21 @@ void testInit(Solver* s)
for (int k = 0; k < kmaxLocal + 2; k++) {
for (int j = 0; j < jmaxLocal + 2; j++) {
for (int i = 0; i < imaxLocal + 2; i++) {
G(p, i, j, k) = myrank;
G(f, i, j, k) = myrank;
G(g, i, j, k) = myrank;
G(h, i, j, k) = myrank;
G(p, i, j, k) = 10.0;
G(f, i, j, k) = myrank + 1.0;
G(g, i, j, k) = myrank + 1.0;
G(h, i, j, k) = myrank + 1.0;
}
}
}
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
G(p, i, j, k) = myrank + 1.0;
G(f, i, j, k) = myrank + 1.0;
G(g, i, j, k) = myrank + 1.0;
G(h, i, j, k) = myrank + 1.0;
}
}
}

2
BasicSolver/3D-mpi/src/test.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.

8
BasicSolver/3D-mpi/src/timing.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
@@ -7,18 +7,16 @@
#include <stdlib.h>
#include <time.h>
double getTimeStamp()
double getTimeStamp(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9;
}
double getTimeResolution()
double getTimeResolution(void)
{
struct timespec ts;
clock_getres(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9;
}
double getTimeStamp_() { return getTimeStamp(); }

7
BasicSolver/3D-mpi/src/timing.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.
@@ -7,8 +7,7 @@
#ifndef __TIMING_H_
#define __TIMING_H_
extern double getTimeStamp();
extern double getTimeResolution();
extern double getTimeStamp_();
extern double getTimeStamp(void);
extern double getTimeResolution(void);
#endif // __TIMING_H_

2
BasicSolver/3D-mpi/src/util.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved.
* Use of this source code is governed by a MIT-style
* license that can be found in the LICENSE file.

228
BasicSolver/3D-mpi/src/vtkWriter-mpi.c Normal file
View File

@@ -0,0 +1,228 @@
/*
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
*/
#include <mpi.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "allocate.h"
#include "comm.h"
#include "vtkWriter.h"
// reset fileview for output of string headers
static void resetFileview(VtkOptions* o)
{
MPI_Offset disp;
MPI_File_sync(o->fh);
MPI_Barrier(o->comm.comm);
MPI_File_get_size(o->fh, &disp);
MPI_File_set_view(o->fh, disp, MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL);
}
static void writeVersion(VtkOptions* o)
{
char header[50] = "# vtk DataFile Version 3.0\n";
// always overwrite exiting files
MPI_File_set_view(o->fh, 0, MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
}
}
static void writeHeader(VtkOptions* o)
{
char header[400];
char* cursor = header;
cursor += sprintf(cursor, "PAMPI cfd solver output\n");
cursor += sprintf(cursor, "BINARY\n");
cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n");
cursor += sprintf(cursor,
"DIMENSIONS %d %d %d\n",
o->grid.imax,
o->grid.jmax,
o->grid.kmax);
cursor += sprintf(cursor,
"ORIGIN %f %f %f\n",
o->grid.dx * 0.5,
o->grid.dy * 0.5,
o->grid.dz * 0.5);
cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz);
cursor += sprintf(cursor,
"POINT_DATA %d\n",
o->grid.imax * o->grid.jmax * o->grid.kmax);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
}
}
void vtkOpen(VtkOptions* o, char* problem)
{
char filename[50];
snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size);
MPI_File_open(o->comm.comm,
filename,
MPI_MODE_WRONLY | MPI_MODE_CREATE,
MPI_INFO_NULL,
&o->fh);
if (commIsMaster(&o->comm)) {
printf("Writing VTK output for %s\n", problem);
}
writeVersion(o);
writeHeader(o);
}
void vtkScalar(VtkOptions* o, char* name, double* s)
{
resetFileview(o);
if (commIsMaster(&o->comm)) printf("Register scalar %s\n", name);
char header[100];
char* cursor = header;
cursor += sprintf(cursor, "SCALARS %s double\n", name);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
}
int offsets[NDIMS];
commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax);
// set global view in file
MPI_Offset disp;
MPI_Datatype fileViewType;
MPI_File_sync(o->fh);
MPI_Barrier(o->comm.comm);
MPI_File_get_size(o->fh, &disp);
MPI_Type_create_subarray(NDIMS,
(int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax },
(int[NDIMS]) { o->comm.kmaxLocal, o->comm.jmaxLocal, o->comm.imaxLocal },
offsets,
MPI_ORDER_C,
MPI_DOUBLE,
&fileViewType);
MPI_Type_commit(&fileViewType);
MPI_File_set_view(o->fh, disp, MPI_DOUBLE, fileViewType, "external32", MPI_INFO_NULL);
#ifdef VERBOSE
printf("Rank: %d, Disp: %lld, Size(k,j,i): %d %d %d, Offset(k,j,i): %d %d %d\n",
o->comm.rank,
disp,
o->comm.kmaxLocal,
o->comm.jmaxLocal,
o->comm.imaxLocal,
offsets[KDIM],
offsets[JDIM],
offsets[IDIM]);
#endif
// create local bulk type
MPI_Datatype bulkType;
MPI_Type_create_subarray(NDIMS,
(int[NDIMS]) { o->comm.kmaxLocal + 2,
o->comm.jmaxLocal + 2,
o->comm.imaxLocal + 2 }, // oldsizes
(int[NDIMS]) { o->comm.kmaxLocal,
o->comm.jmaxLocal,
o->comm.imaxLocal }, // newsizes
(int[NDIMS]) { 1, 1, 1 }, // offsets
MPI_ORDER_C,
MPI_DOUBLE,
&bulkType);
MPI_Type_commit(&bulkType);
MPI_File_write(o->fh, s, 1, bulkType, MPI_STATUS_IGNORE);
MPI_Type_free(&bulkType);
MPI_Type_free(&fileViewType);
// Binary segment must be terminated with newline character
resetFileview(o);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, "\n", 1, MPI_CHAR, MPI_STATUS_IGNORE);
}
}
#define G(v, i, j, k) \
v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)]
void vtkVector(VtkOptions* o, char* name, VtkVector vec)
{
int imaxLocal = o->comm.imaxLocal;
int jmaxLocal = o->comm.jmaxLocal;
int kmaxLocal = o->comm.kmaxLocal;
if (commIsMaster(&o->comm)) printf("Register vector %s\n", name);
const size_t MAX_HEADER = 100;
char* header = (char*)malloc(MAX_HEADER);
sprintf(header, "VECTORS %s double\n", name);
resetFileview(o);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE);
}
int offsets[NDIMS];
commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax);
// set global view in file
MPI_Offset disp;
MPI_Datatype fileViewType, vectorType;
MPI_File_sync(o->fh);
MPI_Barrier(o->comm.comm);
MPI_File_get_size(o->fh, &disp);
MPI_Type_contiguous(NDIMS, MPI_DOUBLE, &vectorType);
MPI_Type_commit(&vectorType);
MPI_Type_create_subarray(NDIMS,
(int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax },
(int[NDIMS]) { kmaxLocal, jmaxLocal, imaxLocal },
offsets,
MPI_ORDER_C,
vectorType,
&fileViewType);
MPI_Type_commit(&fileViewType);
MPI_File_set_view(o->fh, disp, MPI_DOUBLE, fileViewType, "external32", MPI_INFO_NULL);
size_t cnt = imaxLocal * jmaxLocal * kmaxLocal;
double* tmp = allocate(64, cnt * NDIMS * sizeof(double));
int idx = 0;
for (int k = 1; k < kmaxLocal + 1; k++) {
for (int j = 1; j < jmaxLocal + 1; j++) {
for (int i = 1; i < imaxLocal + 1; i++) {
tmp[idx++] = (G(vec.u, i, j, k) + G(vec.u, i - 1, j, k)) / 2.0;
tmp[idx++] = (G(vec.v, i, j, k) + G(vec.v, i, j - 1, k)) / 2.0;
tmp[idx++] = (G(vec.w, i, j, k) + G(vec.w, i, j, k - 1)) / 2.0;
}
}
}
if (commIsMaster(&o->comm)) printf("Write %d vectors\n", (int)cnt);
MPI_File_write(o->fh, tmp, cnt, vectorType, MPI_STATUS_IGNORE);
MPI_Type_free(&fileViewType);
MPI_Type_free(&vectorType);
// Binary segment must be terminated with newline character
resetFileview(o);
if (commIsMaster(&o->comm)) {
MPI_File_write(o->fh, "\n", 1, MPI_CHAR, MPI_STATUS_IGNORE);
}
}
void vtkClose(VtkOptions* o) { MPI_File_close(&o->fh); }

56
BasicSolver/3D-mpi/src/vtkWriter.c → BasicSolver/3D-mpi/src/vtkWriter-seq.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -9,7 +9,7 @@
#include <string.h>
#include "vtkWriter.h"
#define G(v, i, j, k) v[(k) * imax * jmax + (j) * imax + (i)]
#define G(v, i, j, k) v[(k)*imax * jmax + (j)*imax + (i)]
static double floatSwap(double f)
{
@@ -55,16 +55,11 @@ void vtkOpen(VtkOptions* o, char* problem)
{
char filename[50];
if (o->mode == UNIX) {
if (commIsMaster(&o->comm)) {
snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size);
o->fh = fopen(filename, "w");
writeHeader(o);
}
} else if (o->mode == MPI) {
}
snprintf(filename, 50, "%s.vtk", problem);
o->fh = fopen(filename, "w");
writeHeader(o);
if (commIsMaster(&o->comm)) printf("Writing VTK output for %s\n", problem);
printf("Writing VTK output for %s\n", problem);
}
static void writeScalar(VtkOptions* o, double* s)
@@ -101,17 +96,11 @@ static bool isInitialized(FILE* ptr)
void vtkScalar(VtkOptions* o, char* name, double* s)
{
if (commIsMaster(&o->comm)) printf("Register scalar %s\n", name);
if (o->mode == UNIX) {
if (commIsMaster(&o->comm)) {
if (!isInitialized(o->fh)) return;
fprintf(o->fh, "SCALARS %s double 1\n", name);
fprintf(o->fh, "LOOKUP_TABLE default\n");
writeScalar(o, s);
}
} else if (o->mode == MPI) {
}
printf("Register scalar %s\n", name);
if (!isInitialized(o->fh)) return;
fprintf(o->fh, "SCALARS %s double 1\n", name);
fprintf(o->fh, "LOOKUP_TABLE default\n");
writeScalar(o, s);
}
static void writeVector(VtkOptions* o, VtkVector vec)
@@ -145,25 +134,14 @@ static void writeVector(VtkOptions* o, VtkVector vec)
void vtkVector(VtkOptions* o, char* name, VtkVector vec)
{
if (commIsMaster(&o->comm)) printf("Register vector %s\n", name);
if (o->mode == UNIX) {
if (commIsMaster(&o->comm)) {
if (!isInitialized(o->fh)) return;
fprintf(o->fh, "VECTORS %s double\n", name);
writeVector(o, vec);
}
} else if (o->mode == MPI) {
}
printf("Register vector %s\n", name);
if (!isInitialized(o->fh)) return;
fprintf(o->fh, "VECTORS %s double\n", name);
writeVector(o, vec);
}
void vtkClose(VtkOptions* o)
{
if (o->mode == UNIX) {
if (commIsMaster(&o->comm)) {
fclose(o->fh);
o->fh = NULL;
}
} else if (o->mode == MPI) {
}
fclose(o->fh);
o->fh = NULL;
}

10
BasicSolver/3D-mpi/src/vtkWriter.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
* Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
* All rights reserved. This file is part of nusif-solver.
* Use of this source code is governed by a MIT style
* license that can be found in the LICENSE file.
@@ -12,13 +12,15 @@
#include "grid.h"
typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat;
typedef enum VtkMode { UNIX = 0, MPI } VtkMode;
typedef struct VtkOptions {
VtkFormat fmt;
VtkMode mode;
Grid grid;
#ifdef _VTK_WRITER_MPI
MPI_File fh;
#else
FILE* fh;
VtkFormat fmt;
#endif // _VTK_WRITER_MPI
Comm comm;
} VtkOptions;