From d81313f293135a40739fb869c3bb83d043272804 Mon Sep 17 00:00:00 2001 From: Aditya ujeniya Date: Sat, 27 Jul 2024 02:19:56 +0200 Subject: [PATCH 1/3] EnhancedSolver port complete --- BasicSolver/2D-mpi/Makefile | 16 +- BasicSolver/2D-mpi/canal.par | 19 +- BasicSolver/2D-mpi/config.mk | 6 +- BasicSolver/2D-mpi/dcavity.par | 17 +- BasicSolver/2D-mpi/residual.plot | 9 + BasicSolver/2D-mpi/src/comm-v1.c | 57 +- BasicSolver/2D-mpi/src/comm-v2.c | 71 +- BasicSolver/2D-mpi/src/comm-v3.c | 65 +- BasicSolver/2D-mpi/src/comm.h | 2 + BasicSolver/2D-mpi/src/discretization.c | 7 +- BasicSolver/2D-mpi/src/main.c | 16 +- BasicSolver/2D-mpi/src/parameter.c | 20 +- BasicSolver/2D-mpi/src/parameter.h | 1 + BasicSolver/2D-mpi/src/progress.c | 21 +- BasicSolver/2D-mpi/src/progress.h | 3 +- BasicSolver/2D-mpi/src/solver-mg.c | 302 ++++ BasicSolver/2D-mpi/src/solver-rb.c | 106 ++ BasicSolver/2D-mpi/src/solver.h | 5 +- BasicSolver/2D-seq/Makefile | 17 +- BasicSolver/2D-seq/canal.par | 7 + BasicSolver/2D-seq/config.mk | 6 +- BasicSolver/2D-seq/dcavity.par | 8 +- BasicSolver/2D-seq/residual.plot | 9 + BasicSolver/2D-seq/src/main.c | 17 +- BasicSolver/2D-seq/src/parameter.c | 4 + BasicSolver/2D-seq/src/parameter.h | 1 + BasicSolver/2D-seq/src/progress.c | 19 + BasicSolver/2D-seq/src/progress.h | 2 + BasicSolver/2D-seq/src/solver-mg.c | 51 +- BasicSolver/2D-seq/src/solver-rb.c | 4 +- BasicSolver/2D-seq/src/solver-sor.c | 4 +- BasicSolver/2D-seq/src/solver.h | 3 +- BasicSolver/3D-mpi/Makefile | 15 +- BasicSolver/3D-mpi/canal.par | 15 +- BasicSolver/3D-mpi/config.mk | 5 +- BasicSolver/3D-mpi/dcavity.par | 9 +- BasicSolver/3D-mpi/residual.plot | 9 + BasicSolver/3D-mpi/src/comm.c | 66 +- BasicSolver/3D-mpi/src/comm.h | 4 + BasicSolver/3D-mpi/src/discretization.c | 729 +++++++++ BasicSolver/3D-mpi/src/discretization.h | 44 + BasicSolver/3D-mpi/src/main.c | 85 +- BasicSolver/3D-mpi/src/parameter.c | 31 +- BasicSolver/3D-mpi/src/parameter.h | 1 + BasicSolver/3D-mpi/src/progress.c | 19 +- BasicSolver/3D-mpi/src/progress.h | 7 +- BasicSolver/3D-mpi/src/solver-mg.c | 403 +++++ BasicSolver/3D-mpi/src/solver-rb.c | 175 +++ BasicSolver/3D-mpi/src/solver.h | 20 +- BasicSolver/3D-mpi/src/test.c | 22 +- BasicSolver/3D-mpi/src/util.h | 9 + BasicSolver/3D-mpi/src/vtkWriter-mpi.c | 25 +- BasicSolver/3D-mpi/src/vtkWriter-seq.c | 24 +- BasicSolver/3D-mpi/src/vtkWriter.h | 2 +- BasicSolver/3D-seq/Makefile | 12 + BasicSolver/3D-seq/canal.par | 9 +- BasicSolver/3D-seq/config.mk | 6 +- BasicSolver/3D-seq/dcavity.par | 10 +- BasicSolver/3D-seq/residual.plot | 9 + BasicSolver/3D-seq/src/discretization.c | 2 +- BasicSolver/3D-seq/src/main.c | 14 +- BasicSolver/3D-seq/src/parameter.c | 28 +- BasicSolver/3D-seq/src/parameter.h | 1 + BasicSolver/3D-seq/src/progress.c | 19 + BasicSolver/3D-seq/src/progress.h | 3 +- BasicSolver/3D-seq/src/solver-mg.c | 90 +- BasicSolver/3D-seq/src/solver-rb.c | 101 ++ BasicSolver/3D-seq/src/solver.h | 3 +- BasicSolver/3D-seq/src/vtkWriter.c | 1 + EnhancedSolver/2D-mpi/Makefile | 88 ++ EnhancedSolver/2D-mpi/README.md | 48 + EnhancedSolver/2D-mpi/backstep.par | 79 + EnhancedSolver/2D-mpi/canal.par | 78 + EnhancedSolver/2D-mpi/config.mk | 17 + EnhancedSolver/2D-mpi/dcavity.par | 79 + EnhancedSolver/2D-mpi/include_CLANG.mk | 21 + EnhancedSolver/2D-mpi/include_GCC.mk | 20 + EnhancedSolver/2D-mpi/include_ICC.mk | 20 + EnhancedSolver/2D-mpi/karman.par | 79 + EnhancedSolver/2D-mpi/residual.plot | 9 + EnhancedSolver/2D-mpi/src/affinity.c | 61 + EnhancedSolver/2D-mpi/src/affinity.h | 14 + EnhancedSolver/2D-mpi/src/allocate.c | 38 + EnhancedSolver/2D-mpi/src/allocate.h | 13 + EnhancedSolver/2D-mpi/src/comm-v1.c | 233 +++ EnhancedSolver/2D-mpi/src/comm-v2.c | 342 +++++ EnhancedSolver/2D-mpi/src/comm-v3.c | 320 ++++ EnhancedSolver/2D-mpi/src/comm.c | 129 ++ EnhancedSolver/2D-mpi/src/comm.h | 57 + EnhancedSolver/2D-mpi/src/discretization.c | 724 +++++++++ EnhancedSolver/2D-mpi/src/discretization.h | 66 + EnhancedSolver/2D-mpi/src/grid.h | 17 + EnhancedSolver/2D-mpi/src/likwid-marker.h | 54 + EnhancedSolver/2D-mpi/src/main.c | 125 ++ EnhancedSolver/2D-mpi/src/parameter.c | 143 ++ EnhancedSolver/2D-mpi/src/parameter.h | 38 + EnhancedSolver/2D-mpi/src/particletracing.c | 675 ++++++++ EnhancedSolver/2D-mpi/src/particletracing.h | 65 + EnhancedSolver/2D-mpi/src/progress.c | 70 + EnhancedSolver/2D-mpi/src/progress.h | 17 + EnhancedSolver/2D-mpi/src/solver-mg.c | 325 ++++ EnhancedSolver/2D-mpi/src/solver-rb.c | 131 ++ EnhancedSolver/2D-mpi/src/solver.h | 30 + EnhancedSolver/2D-mpi/src/timing.c | 22 + EnhancedSolver/2D-mpi/src/timing.h | 13 + EnhancedSolver/2D-mpi/src/util.h | 30 + EnhancedSolver/2D-mpi/surface.plot | 7 + EnhancedSolver/2D-mpi/vector.plot | 6 + EnhancedSolver/2D-mpi/vis_files/animate.plot | 10 + .../2D-mpi/vis_files/backstep_animate.plot | 14 + .../2D-mpi/vis_files/canal_animate.plot | 10 + .../2D-mpi/vis_files/karman_animate.plot | 13 + EnhancedSolver/2D-seq/Makefile | 17 +- EnhancedSolver/2D-seq/backstep.par | 11 +- EnhancedSolver/2D-seq/canal.par | 16 +- EnhancedSolver/2D-seq/config.mk | 2 +- EnhancedSolver/2D-seq/dcavity.par | 8 +- EnhancedSolver/2D-seq/karman.par | 10 +- EnhancedSolver/2D-seq/residual.plot | 9 + EnhancedSolver/2D-seq/src/main.c | 10 +- EnhancedSolver/2D-seq/src/parameter.c | 28 +- EnhancedSolver/2D-seq/src/parameter.h | 2 +- EnhancedSolver/2D-seq/src/particletracing.c | 10 +- EnhancedSolver/2D-seq/src/progress.c | 18 + EnhancedSolver/2D-seq/src/progress.h | 3 +- EnhancedSolver/2D-seq/src/solver-mg.c | 53 +- EnhancedSolver/2D-seq/src/solver-rb.c | 29 +- EnhancedSolver/2D-seq/src/solver-sor.c | 27 +- EnhancedSolver/2D-seq/src/solver.h | 4 +- .../2D-seq/vis_files/canal_animate.plot | 10 + EnhancedSolver/3D-mpi/Makefile | 96 ++ EnhancedSolver/3D-mpi/README.md | 78 + EnhancedSolver/3D-mpi/backstep.par | 89 ++ EnhancedSolver/3D-mpi/canal.par | 88 ++ EnhancedSolver/3D-mpi/config.mk | 18 + EnhancedSolver/3D-mpi/dcavity.par | 88 ++ EnhancedSolver/3D-mpi/include_CLANG.mk | 21 + EnhancedSolver/3D-mpi/include_GCC.mk | 20 + EnhancedSolver/3D-mpi/include_ICC.mk | 20 + EnhancedSolver/3D-mpi/karman.par | 89 ++ EnhancedSolver/3D-mpi/residual.plot | 9 + EnhancedSolver/3D-mpi/src/allocate.c | 38 + EnhancedSolver/3D-mpi/src/allocate.h | 13 + EnhancedSolver/3D-mpi/src/comm.c | 647 ++++++++ EnhancedSolver/3D-mpi/src/comm.h | 66 + EnhancedSolver/3D-mpi/src/discretization.c | 1356 +++++++++++++++++ EnhancedSolver/3D-mpi/src/discretization.h | 92 ++ EnhancedSolver/3D-mpi/src/grid.h | 16 + EnhancedSolver/3D-mpi/src/likwid-marker.h | 54 + EnhancedSolver/3D-mpi/src/main.c | 151 ++ EnhancedSolver/3D-mpi/src/parameter.c | 154 ++ EnhancedSolver/3D-mpi/src/parameter.h | 35 + EnhancedSolver/3D-mpi/src/particletracing.c | 749 +++++++++ EnhancedSolver/3D-mpi/src/particletracing.h | 65 + EnhancedSolver/3D-mpi/src/progress.c | 65 + EnhancedSolver/3D-mpi/src/progress.h | 17 + EnhancedSolver/3D-mpi/src/solver-mg.c | 389 +++++ EnhancedSolver/3D-mpi/src/solver-rb.c | 175 +++ EnhancedSolver/3D-mpi/src/solver.h | 39 + EnhancedSolver/3D-mpi/src/test.c | 129 ++ EnhancedSolver/3D-mpi/src/test.h | 13 + EnhancedSolver/3D-mpi/src/timing.c | 22 + EnhancedSolver/3D-mpi/src/timing.h | 13 + EnhancedSolver/3D-mpi/src/util.h | 35 + EnhancedSolver/3D-mpi/src/vtkWriter-mpi.c | 227 +++ EnhancedSolver/3D-mpi/src/vtkWriter-seq.c | 147 ++ EnhancedSolver/3D-mpi/src/vtkWriter.h | 35 + EnhancedSolver/3D-seq/Makefile | 88 ++ EnhancedSolver/3D-seq/README.md | 78 + EnhancedSolver/3D-seq/backstep.par | 89 ++ EnhancedSolver/3D-seq/canal.par | 88 ++ EnhancedSolver/3D-seq/config.mk | 12 + EnhancedSolver/3D-seq/dcavity.par | 88 ++ EnhancedSolver/3D-seq/include_CLANG.mk | 19 + EnhancedSolver/3D-seq/include_GCC.mk | 14 + EnhancedSolver/3D-seq/include_ICC.mk | 14 + EnhancedSolver/3D-seq/karman.par | 89 ++ EnhancedSolver/3D-seq/residual.plot | 9 + EnhancedSolver/3D-seq/src/allocate.c | 38 + EnhancedSolver/3D-seq/src/allocate.h | 13 + EnhancedSolver/3D-seq/src/discretization.c | 1128 ++++++++++++++ EnhancedSolver/3D-seq/src/discretization.h | 86 ++ EnhancedSolver/3D-seq/src/grid.h | 16 + EnhancedSolver/3D-seq/src/likwid-marker.h | 54 + EnhancedSolver/3D-seq/src/main.c | 157 ++ EnhancedSolver/3D-seq/src/parameter.c | 153 ++ EnhancedSolver/3D-seq/src/parameter.h | 35 + EnhancedSolver/3D-seq/src/particletracing.c | 313 ++++ EnhancedSolver/3D-seq/src/particletracing.h | 46 + EnhancedSolver/3D-seq/src/progress.c | 70 + EnhancedSolver/3D-seq/src/progress.h | 15 + EnhancedSolver/3D-seq/src/solver-mg.c | 304 ++++ EnhancedSolver/3D-seq/src/solver-rb.c | 101 ++ EnhancedSolver/3D-seq/src/solver.h | 27 + EnhancedSolver/3D-seq/src/timing.c | 22 + EnhancedSolver/3D-seq/src/timing.h | 13 + EnhancedSolver/3D-seq/src/util.h | 32 + EnhancedSolver/3D-seq/src/vtkWriter.c | 198 +++ EnhancedSolver/3D-seq/src/vtkWriter.h | 34 + 199 files changed, 16030 insertions(+), 310 deletions(-) create mode 100644 BasicSolver/2D-mpi/residual.plot create mode 100644 BasicSolver/2D-mpi/src/solver-mg.c create mode 100644 BasicSolver/2D-mpi/src/solver-rb.c create mode 100644 BasicSolver/2D-seq/residual.plot create mode 100644 BasicSolver/3D-mpi/residual.plot create mode 100644 BasicSolver/3D-mpi/src/discretization.c create mode 100644 BasicSolver/3D-mpi/src/discretization.h create mode 100644 BasicSolver/3D-mpi/src/solver-mg.c create mode 100644 BasicSolver/3D-mpi/src/solver-rb.c create mode 100644 BasicSolver/3D-seq/residual.plot create mode 100644 BasicSolver/3D-seq/src/solver-rb.c create mode 100644 EnhancedSolver/2D-mpi/Makefile create mode 100644 EnhancedSolver/2D-mpi/README.md create mode 100644 EnhancedSolver/2D-mpi/backstep.par create mode 100644 EnhancedSolver/2D-mpi/canal.par create mode 100644 EnhancedSolver/2D-mpi/config.mk create mode 100644 EnhancedSolver/2D-mpi/dcavity.par create mode 100644 EnhancedSolver/2D-mpi/include_CLANG.mk create mode 100644 EnhancedSolver/2D-mpi/include_GCC.mk create mode 100644 EnhancedSolver/2D-mpi/include_ICC.mk create mode 100644 EnhancedSolver/2D-mpi/karman.par create mode 100644 EnhancedSolver/2D-mpi/residual.plot create mode 100644 EnhancedSolver/2D-mpi/src/affinity.c create mode 100644 EnhancedSolver/2D-mpi/src/affinity.h create mode 100644 EnhancedSolver/2D-mpi/src/allocate.c create mode 100644 EnhancedSolver/2D-mpi/src/allocate.h create mode 100644 EnhancedSolver/2D-mpi/src/comm-v1.c create mode 100644 EnhancedSolver/2D-mpi/src/comm-v2.c create mode 100644 EnhancedSolver/2D-mpi/src/comm-v3.c create mode 100644 EnhancedSolver/2D-mpi/src/comm.c create mode 100644 EnhancedSolver/2D-mpi/src/comm.h create mode 100644 EnhancedSolver/2D-mpi/src/discretization.c create mode 100644 EnhancedSolver/2D-mpi/src/discretization.h create mode 100644 EnhancedSolver/2D-mpi/src/grid.h create mode 100644 EnhancedSolver/2D-mpi/src/likwid-marker.h create mode 100644 EnhancedSolver/2D-mpi/src/main.c create mode 100644 EnhancedSolver/2D-mpi/src/parameter.c create mode 100644 EnhancedSolver/2D-mpi/src/parameter.h create mode 100644 EnhancedSolver/2D-mpi/src/particletracing.c create mode 100644 EnhancedSolver/2D-mpi/src/particletracing.h create mode 100644 EnhancedSolver/2D-mpi/src/progress.c create mode 100644 EnhancedSolver/2D-mpi/src/progress.h create mode 100644 EnhancedSolver/2D-mpi/src/solver-mg.c create mode 100644 EnhancedSolver/2D-mpi/src/solver-rb.c create mode 100644 EnhancedSolver/2D-mpi/src/solver.h create mode 100644 EnhancedSolver/2D-mpi/src/timing.c create mode 100644 EnhancedSolver/2D-mpi/src/timing.h create mode 100644 EnhancedSolver/2D-mpi/src/util.h create mode 100644 EnhancedSolver/2D-mpi/surface.plot create mode 100644 EnhancedSolver/2D-mpi/vector.plot create mode 100644 EnhancedSolver/2D-mpi/vis_files/animate.plot create mode 100644 EnhancedSolver/2D-mpi/vis_files/backstep_animate.plot create mode 100644 EnhancedSolver/2D-mpi/vis_files/canal_animate.plot create mode 100644 EnhancedSolver/2D-mpi/vis_files/karman_animate.plot create mode 100644 EnhancedSolver/2D-seq/residual.plot create mode 100644 EnhancedSolver/2D-seq/vis_files/canal_animate.plot create mode 100644 EnhancedSolver/3D-mpi/Makefile create mode 100644 EnhancedSolver/3D-mpi/README.md create mode 100644 EnhancedSolver/3D-mpi/backstep.par create mode 100644 EnhancedSolver/3D-mpi/canal.par create mode 100644 EnhancedSolver/3D-mpi/config.mk create mode 100644 EnhancedSolver/3D-mpi/dcavity.par create mode 100644 EnhancedSolver/3D-mpi/include_CLANG.mk create mode 100644 EnhancedSolver/3D-mpi/include_GCC.mk create mode 100644 EnhancedSolver/3D-mpi/include_ICC.mk create mode 100644 EnhancedSolver/3D-mpi/karman.par create mode 100644 EnhancedSolver/3D-mpi/residual.plot create mode 100644 EnhancedSolver/3D-mpi/src/allocate.c create mode 100644 EnhancedSolver/3D-mpi/src/allocate.h create mode 100644 EnhancedSolver/3D-mpi/src/comm.c create mode 100644 EnhancedSolver/3D-mpi/src/comm.h create mode 100644 EnhancedSolver/3D-mpi/src/discretization.c create mode 100644 EnhancedSolver/3D-mpi/src/discretization.h create mode 100644 EnhancedSolver/3D-mpi/src/grid.h create mode 100644 EnhancedSolver/3D-mpi/src/likwid-marker.h create mode 100644 EnhancedSolver/3D-mpi/src/main.c create mode 100644 EnhancedSolver/3D-mpi/src/parameter.c create mode 100644 EnhancedSolver/3D-mpi/src/parameter.h create mode 100644 EnhancedSolver/3D-mpi/src/particletracing.c create mode 100644 EnhancedSolver/3D-mpi/src/particletracing.h create mode 100644 EnhancedSolver/3D-mpi/src/progress.c create mode 100644 EnhancedSolver/3D-mpi/src/progress.h create mode 100644 EnhancedSolver/3D-mpi/src/solver-mg.c create mode 100644 EnhancedSolver/3D-mpi/src/solver-rb.c create mode 100644 EnhancedSolver/3D-mpi/src/solver.h create mode 100644 EnhancedSolver/3D-mpi/src/test.c create mode 100644 EnhancedSolver/3D-mpi/src/test.h create mode 100644 EnhancedSolver/3D-mpi/src/timing.c create mode 100644 EnhancedSolver/3D-mpi/src/timing.h create mode 100644 EnhancedSolver/3D-mpi/src/util.h create mode 100644 EnhancedSolver/3D-mpi/src/vtkWriter-mpi.c create mode 100644 EnhancedSolver/3D-mpi/src/vtkWriter-seq.c create mode 100644 EnhancedSolver/3D-mpi/src/vtkWriter.h create mode 100644 EnhancedSolver/3D-seq/Makefile create mode 100644 EnhancedSolver/3D-seq/README.md create mode 100644 EnhancedSolver/3D-seq/backstep.par create mode 100644 EnhancedSolver/3D-seq/canal.par create mode 100644 EnhancedSolver/3D-seq/config.mk create mode 100644 EnhancedSolver/3D-seq/dcavity.par create mode 100644 EnhancedSolver/3D-seq/include_CLANG.mk create mode 100644 EnhancedSolver/3D-seq/include_GCC.mk create mode 100644 EnhancedSolver/3D-seq/include_ICC.mk create mode 100644 EnhancedSolver/3D-seq/karman.par create mode 100644 EnhancedSolver/3D-seq/residual.plot create mode 100644 EnhancedSolver/3D-seq/src/allocate.c create mode 100644 EnhancedSolver/3D-seq/src/allocate.h create mode 100644 EnhancedSolver/3D-seq/src/discretization.c create mode 100644 EnhancedSolver/3D-seq/src/discretization.h create mode 100644 EnhancedSolver/3D-seq/src/grid.h create mode 100644 EnhancedSolver/3D-seq/src/likwid-marker.h create mode 100644 EnhancedSolver/3D-seq/src/main.c create mode 100644 EnhancedSolver/3D-seq/src/parameter.c create mode 100644 EnhancedSolver/3D-seq/src/parameter.h create mode 100644 EnhancedSolver/3D-seq/src/particletracing.c create mode 100644 EnhancedSolver/3D-seq/src/particletracing.h create mode 100644 EnhancedSolver/3D-seq/src/progress.c create mode 100644 EnhancedSolver/3D-seq/src/progress.h create mode 100644 EnhancedSolver/3D-seq/src/solver-mg.c create mode 100644 EnhancedSolver/3D-seq/src/solver-rb.c create mode 100644 EnhancedSolver/3D-seq/src/solver.h create mode 100644 EnhancedSolver/3D-seq/src/timing.c create mode 100644 EnhancedSolver/3D-seq/src/timing.h create mode 100644 EnhancedSolver/3D-seq/src/util.h create mode 100644 EnhancedSolver/3D-seq/src/vtkWriter.c create mode 100644 EnhancedSolver/3D-seq/src/vtkWriter.h diff --git a/BasicSolver/2D-mpi/Makefile b/BasicSolver/2D-mpi/Makefile index 2b23422..eb5349c 100644 --- a/BasicSolver/2D-mpi/Makefile +++ b/BasicSolver/2D-mpi/Makefile @@ -22,6 +22,7 @@ SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) OBJ += $(BUILD_DIR)/comm-$(COMM_TYPE).o +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) @@ -38,9 +39,20 @@ $(BUILD_DIR)/%.s: %.c $(info ===> GENERATE ASM $@) $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ -.PHONY: clean distclean tags info asm format +.PHONY: clean distclean vis vis_clean tags info asm format -clean: +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='pressure.dat'" ./surface.plot + @gnuplot -e "filename='velocity.dat'" ./vector.plot + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.png + +clean: vis_clean $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @rm -f tags diff --git a/BasicSolver/2D-mpi/canal.par b/BasicSolver/2D-mpi/canal.par index 80dfa19..37b26fa 100644 --- a/BasicSolver/2D-mpi/canal.par +++ b/BasicSolver/2D-mpi/canal.par @@ -7,10 +7,10 @@ name canal # name of flow setup -bcN 1 # flags for boundary conditions -bcE 3 # 1 = no-slip 3 = outflow -bcS 1 # 2 = free-slip 4 = periodic -bcW 3 # +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # gx 0.0 # Body forces (e.g. gravity) gy 0.0 # @@ -27,15 +27,22 @@ p_init 0.0 # initial value for pressure xlength 30.0 # domain size in x-direction ylength 4.0 # domain size in y-direction imax 200 # number of interior cells in x-direction -jmax 50 # number of interior cells in y-direction +jmax 40 # number of interior cells in y-direction # Time Data: # --------- -te 100.0 # final time +te 60.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- diff --git a/BasicSolver/2D-mpi/config.mk b/BasicSolver/2D-mpi/config.mk index a7b7925..eb5546a 100644 --- a/BasicSolver/2D-mpi/config.mk +++ b/BasicSolver/2D-mpi/config.mk @@ -1,7 +1,11 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC +# Supported: true, false ENABLE_MPI ?= true ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg +# Supported: v1, v2, v3 COMM_TYPE ?= v3 #Feature options diff --git a/BasicSolver/2D-mpi/dcavity.par b/BasicSolver/2D-mpi/dcavity.par index 2c81044..8f229ed 100644 --- a/BasicSolver/2D-mpi/dcavity.par +++ b/BasicSolver/2D-mpi/dcavity.par @@ -26,21 +26,28 @@ p_init 0.0 # initial value for pressure xlength 1.0 # domain size in x-direction ylength 1.0 # domain size in y-direction -imax 80 # number of interior cells in x-direction -jmax 80 # number of interior cells in y-direction +imax 128 # number of interior cells in x-direction +jmax 128 # number of interior cells in y-direction # Time Data: # --------- te 10.0 # final time -dt 0.02 # time stepsize -tau 0.5 # safety factor for time stepsize control (<0 constant delt) +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations # Pressure Iteration Data: # ----------------------- itermax 1000 # maximal number of pressure iteration in one time step eps 0.001 # stopping tolerance for pressure iteration -omg 1.9 # relaxation parameter for SOR iteration +omg 1.7 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma #=============================================================================== diff --git a/BasicSolver/2D-mpi/residual.plot b/BasicSolver/2D-mpi/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/BasicSolver/2D-mpi/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/2D-mpi/src/comm-v1.c b/BasicSolver/2D-mpi/src/comm-v1.c index 8f0ea3e..16a0422 100644 --- a/BasicSolver/2D-mpi/src/comm-v1.c +++ b/BasicSolver/2D-mpi/src/comm-v1.c @@ -14,7 +14,7 @@ static int sum(int* sizes, int position) { int sum = 0; - for (int i = 0; i < position; i++) { + for (int i = 0; i < position; i += position) { sum += sizes[i]; } @@ -67,6 +67,7 @@ int commIsBoundary(Comm* c, int direction) void commExchange(Comm* c, double* grid) { + // printf("Rank : %d In exchange \n", c->rank); #ifdef _MPI MPI_Request requests[4] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL, @@ -173,9 +174,61 @@ void commPartition(Comm* c, int jmax, int imax) { #ifdef _MPI c->imaxLocal = imax; - c->jmaxLocal = sizeOfRank(c->rank, c->size, jmax); + c->jmaxLocal = sizeOfRank(c->coords[JDIM], c->size, jmax); + + c->neighbours[BOTTOM] = c->rank == 0 ? -1 : c->rank - 1; + c->neighbours[TOP] = c->rank == (c->size - 1) ? -1 : c->rank + 1; + c->neighbours[LEFT] = -1; + c->neighbours[RIGHT] = -1; + + c->coords[IDIM] = 0; + c->coords[JDIM] = c->rank; + + c->dims[IDIM] = 1; + c->dims[JDIM] = c->size; #else c->imaxLocal = imax; c->jmaxLocal = jmax; #endif } + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ + +#if defined _MPI + newcomm->comm = MPI_COMM_NULL; + int result = MPI_Comm_dup(MPI_COMM_WORLD, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + + newcomm->neighbours[BOTTOM] = newcomm->rank == 0 ? -1 : newcomm->rank - 1; + newcomm->neighbours[TOP] = newcomm->rank == (newcomm->size - 1) ? -1 : newcomm->rank + 1; + newcomm->neighbours[LEFT] = -1; + newcomm->neighbours[RIGHT] = -1; + + newcomm->coords[IDIM] = 0; + newcomm->coords[JDIM] = newcomm->rank; + + newcomm->dims[IDIM] = 1; + newcomm->dims[JDIM] = newcomm->size; + + +#endif + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/BasicSolver/2D-mpi/src/comm-v2.c b/BasicSolver/2D-mpi/src/comm-v2.c index c50a343..5ace203 100644 --- a/BasicSolver/2D-mpi/src/comm-v2.c +++ b/BasicSolver/2D-mpi/src/comm-v2.c @@ -4,18 +4,18 @@ * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ +#include "comm.h" #include #include - -#include "comm.h" +#include #ifdef _MPI // subroutines local to this module -static int sum(int* sizes, int position) +static int sum(int* sizes, int init, int offset, int coord) { int sum = 0; - for (int i = 0; i < position; i++) { + for (int i = init - offset; coord > 0; i -= offset, --coord) { sum += sizes[i]; } @@ -79,8 +79,8 @@ static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] }; int coords[NDIMS]; MPI_Cart_coords(c->comm, i, NDIMS, coords); - int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]), - sum(newSizesI, coords[IDIM]) }; + int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]), + sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) }; printf( "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d " "Starts(i,j): %d %d\n", @@ -252,8 +252,8 @@ void commPartition(Comm* c, int jmax, int imax) MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]); MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords); - int imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax); - int jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax); + int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax); + int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax); c->imaxLocal = imaxLocal; c->jmaxLocal = jmaxLocal; @@ -285,3 +285,58 @@ void commPartition(Comm* c, int jmax, int imax) c->jmaxLocal = jmax; #endif } + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ +#if defined _MPI + newcomm->comm = MPI_COMM_NULL; + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + memcpy(&newcomm->neighbours, &oldcomm->neighbours, sizeof(oldcomm->neighbours)); + memcpy(&newcomm->coords, &oldcomm->coords, sizeof(oldcomm->coords)); + memcpy(&newcomm->dims, &oldcomm->dims, sizeof(oldcomm->dims)); + + newcomm->imaxLocal = imaxLocal/2; + newcomm->jmaxLocal = jmaxLocal/2; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + newcomm->bufferTypes[LEFT] = iBufferType; + newcomm->bufferTypes[RIGHT] = iBufferType; + newcomm->bufferTypes[BOTTOM] = jBufferType; + newcomm->bufferTypes[TOP] = jBufferType; + + newcomm->sdispls[LEFT] = (imaxLocal + 2) + 1; + newcomm->sdispls[RIGHT] = (imaxLocal + 2) + imaxLocal; + newcomm->sdispls[BOTTOM] = (imaxLocal + 2) + 1; + newcomm->sdispls[TOP] = jmaxLocal * (imaxLocal + 2) + 1; + + newcomm->rdispls[LEFT] = (imaxLocal + 2); + newcomm->rdispls[RIGHT] = (imaxLocal + 2) + (imaxLocal + 1); + newcomm->rdispls[BOTTOM] = 1; + newcomm->rdispls[TOP] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/BasicSolver/2D-mpi/src/comm-v3.c b/BasicSolver/2D-mpi/src/comm-v3.c index 6f0d3b7..a1e6fb2 100644 --- a/BasicSolver/2D-mpi/src/comm-v3.c +++ b/BasicSolver/2D-mpi/src/comm-v3.c @@ -11,11 +11,11 @@ #ifdef _MPI // subroutines local to this module -static int sum(int* sizes, int position) +static int sum(int* sizes, int init, int offset, int coord) { int sum = 0; - for (int i = 0; i < position; i++) { + for (int i = init - offset; coord > 0; i -= offset, --coord) { sum += sizes[i]; } @@ -79,8 +79,8 @@ static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] }; int coords[NDIMS]; MPI_Cart_coords(c->comm, i, NDIMS, coords); - int starts[NDIMS] = { sum(newSizesJ, coords[JDIM]), - sum(newSizesI, coords[IDIM]) }; + int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]), + sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) }; printf( "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d " "Starts(i,j): %d %d\n", @@ -139,7 +139,6 @@ void commExchange(Comm* c, double* grid) { #ifdef _MPI int counts[NDIRS] = { 1, 1, 1, 1 }; - MPI_Neighbor_alltoallw(grid, counts, c->sdispls, @@ -233,8 +232,8 @@ void commPartition(Comm* c, int jmax, int imax) MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]); MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords); - int imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax); - int jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax); + int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax); + int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax); c->imaxLocal = imaxLocal; c->jmaxLocal = jmaxLocal; @@ -267,3 +266,55 @@ void commPartition(Comm* c, int jmax, int imax) c->jmaxLocal = jmax; #endif } + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ +#if defined _MPI + + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + newcomm->bufferTypes[LEFT] = iBufferType; + newcomm->bufferTypes[RIGHT] = iBufferType; + newcomm->bufferTypes[BOTTOM] = jBufferType; + newcomm->bufferTypes[TOP] = jBufferType; + + newcomm->sdispls[LEFT] = (imaxLocal + 2) + 1; + newcomm->sdispls[RIGHT] = (imaxLocal + 2) + imaxLocal; + newcomm->sdispls[BOTTOM] = (imaxLocal + 2) + 1; + newcomm->sdispls[TOP] = jmaxLocal * (imaxLocal + 2) + 1; + + newcomm->rdispls[LEFT] = (imaxLocal + 2); + newcomm->rdispls[RIGHT] = (imaxLocal + 2) + (imaxLocal + 1); + newcomm->rdispls[BOTTOM] = 1; + newcomm->rdispls[TOP] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/BasicSolver/2D-mpi/src/comm.h b/BasicSolver/2D-mpi/src/comm.h index 85b4f70..30eaf7b 100644 --- a/BasicSolver/2D-mpi/src/comm.h +++ b/BasicSolver/2D-mpi/src/comm.h @@ -41,6 +41,8 @@ extern void commExchange(Comm*, double*); extern void commShift(Comm* c, double* f, double* g); extern void commReduction(double* v, int op); extern int commIsBoundary(Comm* c, int direction); +extern void commUpdateDatatypes(Comm*, Comm*, int, int); +extern void commFreeCommunicator(Comm*); extern void commCollectResult(Comm* c, double* ug, double* vg, diff --git a/BasicSolver/2D-mpi/src/discretization.c b/BasicSolver/2D-mpi/src/discretization.c index e9fc421..69d3582 100644 --- a/BasicSolver/2D-mpi/src/discretization.c +++ b/BasicSolver/2D-mpi/src/discretization.c @@ -273,17 +273,18 @@ void setSpecialBoundaryCondition(Discretization* s) if (commIsBoundary(&s->comm, LEFT)) { double ylength = s->grid.ylength; double dy = s->grid.dy; - int rest = s->grid.jmax % s->comm.size; - int yc = s->comm.rank * (s->grid.jmax / s->comm.size) + + int rest = s->grid.jmax % s->comm.dims[JDIM]; + int yc = s->comm.rank * (s->grid.jmax / s->comm.dims[JDIM]) + MIN(rest, s->comm.rank); double ys = dy * (yc + 0.5); double y; - /* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */ + // printf("RANK %d yc: %d ys: %f\n", s->comm.rank, yc, ys); for (int j = 1; j < jmaxLocal + 1; j++) { y = ys + dy * (j - 0.5); U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } } } diff --git a/BasicSolver/2D-mpi/src/main.c b/BasicSolver/2D-mpi/src/main.c index c7f2d2f..8f93767 100644 --- a/BasicSolver/2D-mpi/src/main.c +++ b/BasicSolver/2D-mpi/src/main.c @@ -26,7 +26,9 @@ static void writeResults(Discretization* s) double* pg = allocate(64, bytesize); commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->grid.imax, s->grid.jmax); - writeResult(s, ug, vg, pg); + if (commIsMaster(&s->comm)) { + writeResult(s, ug, vg, pg); + } free(ug); free(vg); @@ -47,6 +49,9 @@ int main(int argc, char** argv) commInit(&d.comm, argc, argv); initParameter(&p); + FILE* fp; + if (commIsMaster(&d.comm)) fp = initResidualWriter(); + if (argc != 2) { printf("Usage: %s \n", argv[0]); exit(EXIT_SUCCESS); @@ -77,16 +82,21 @@ int main(int argc, char** argv) double tau = d.tau; double te = d.te; double t = 0.0; + double res = 0.0; timeStart = getTimeStamp(); while (t <= te) { + if (tau > 0.0) computeTimestep(&d); setBoundaryConditions(&d); setSpecialBoundaryCondition(&d); computeFG(&d); computeRHS(&d); - solve(&s, d.p, d.rhs); + res = solve(&s, d.p, d.rhs); adaptUV(&d); + + if (commIsMaster(&d.comm)) writeResidual(fp, t, res); + t += d.dt; #ifdef VERBOSE @@ -104,7 +114,7 @@ int main(int argc, char** argv) if (commIsMaster(s.comm)) { printf("Solution took %.2fs\n", timeStop - timeStart); } - + if (commIsMaster(&d.comm)) fclose(fp); writeResults(&d); commFinalize(s.comm); return EXIT_SUCCESS; diff --git a/BasicSolver/2D-mpi/src/parameter.c b/BasicSolver/2D-mpi/src/parameter.c index 7bac1e3..e5e7029 100644 --- a/BasicSolver/2D-mpi/src/parameter.c +++ b/BasicSolver/2D-mpi/src/parameter.c @@ -14,13 +14,16 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->imax = 100; - param->jmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.8; + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.8; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) @@ -72,6 +75,9 @@ void readParameter(Parameter* param, const char* filename) PARSE_INT(bcRight); PARSE_INT(bcBottom); PARSE_INT(bcTop); + PARSE_INT(levels); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); PARSE_REAL(u_init); PARSE_REAL(v_init); PARSE_REAL(p_init); diff --git a/BasicSolver/2D-mpi/src/parameter.h b/BasicSolver/2D-mpi/src/parameter.h index be28e82..ed765eb 100644 --- a/BasicSolver/2D-mpi/src/parameter.h +++ b/BasicSolver/2D-mpi/src/parameter.h @@ -18,6 +18,7 @@ typedef struct { char* name; int bcLeft, bcRight, bcBottom, bcTop; double u_init, v_init, p_init; + int levels, presmooth, postsmooth; } Parameter; void initParameter(Parameter*); diff --git a/BasicSolver/2D-mpi/src/progress.c b/BasicSolver/2D-mpi/src/progress.c index be98fce..4f3b22e 100644 --- a/BasicSolver/2D-mpi/src/progress.c +++ b/BasicSolver/2D-mpi/src/progress.c @@ -8,7 +8,7 @@ #include #include #include - +#include #include "progress.h" static double _end; @@ -49,3 +49,22 @@ void stopProgress() printf("\n"); fflush(stdout); } + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} diff --git a/BasicSolver/2D-mpi/src/progress.h b/BasicSolver/2D-mpi/src/progress.h index 1cdb9a3..6803315 100644 --- a/BasicSolver/2D-mpi/src/progress.h +++ b/BasicSolver/2D-mpi/src/progress.h @@ -10,5 +10,6 @@ extern void initProgress(double); extern void printProgress(double); extern void stopProgress(); - +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/BasicSolver/2D-mpi/src/solver-mg.c b/BasicSolver/2D-mpi/src/solver-mg.c new file mode 100644 index 0000000..23d64f3 --- /dev/null +++ b/BasicSolver/2D-mpi/src/solver-mg.c @@ -0,0 +1,302 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +// #define S(i, j) s[(j) * (imaxLocal + 2) + (i)] +#define E(i, j) e[(j) * (imaxLocal + 2) + (i)] +#define R(i, j) r[(j) * (imaxLocal + 2) + (i)] +#define OLD(i, j) old[(j) * (imaxLocal + 2) + (i)] + +static void restrictMG(Solver* s, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double* r = s->r[level + 1]; + double* old = s->r[level]; + +#ifdef _MPI + commExchange(comm, old); +#endif + + for (int j = 1; j < (jmaxLocal / 2) + 1; j++) { + for (int i = 1; i < (imaxLocal / 2) + 1; i++) { + R(i, j) = (OLD(2 * i - 1, 2 * j - 1) + OLD(2 * i, 2 * j - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1) + OLD(2 * i - 1, 2 * j) * 2 + + OLD(2 * i, 2 * j) * 4 + OLD(2 * i + 1, 2 * j) * 2 + + OLD(2 * i - 1, 2 * j + 1) + OLD(2 * i, 2 * j + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1)) / + 16.0; + } + } +} + +static void prolongate(Solver* s, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double* old = s->r[level + 1]; + double* e = s->r[level]; + + for (int j = 2; j < jmaxLocal + 1; j += 2) { + for (int i = 2; i < imaxLocal + 1; i += 2) { + E(i, j) = OLD(i / 2, j / 2); + } + } +} + +static void correct(Solver* s, double* p, int level, Comm* comm) +{ + double* e = s->e[level]; + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + for (int j = 1; j < jmaxLocal + 1; ++j) { + for (int i = 1; i < imaxLocal + 1; ++i) { + P(i, j) += E(i, j); + } + } +} + +static void setBoundaryCondition(Solver* s, double* p, int imaxLocal, int jmaxLocal) +{ +#ifdef _MPI + if (commIsBoundary(s->comm, BOTTOM)) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (commIsBoundary(s->comm, TOP)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (commIsBoundary(s->comm, LEFT)) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (commIsBoundary(s->comm, RIGHT)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } +#else + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + P(imaxLocal + 1, j) = P(imaxLocal, j); + } +#endif +} + +static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + +#ifdef _MPI + commExchange(comm, p); +#endif + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + P(i, j) -= factor * + (RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + +#ifdef _MPI + commExchange(comm, p); +#endif + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + R(i, j) = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + res += (R(i, j) * R(i, j)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + +#ifdef _MPI + commReduction(&res, SUM); +#endif + + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + return res; +} + +static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, comm); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + } + + // calculate residuals + res = calculateResidual(s, p, rhs, level, comm); + + // restrict + restrictMG(s, level, comm); + + Comm newcomm; + commUpdateDatatypes(s->comm, &newcomm, comm->imaxLocal, comm->jmaxLocal); + + + // MGSolver on residual and error. + multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm); + + commFreeCommunicator(&newcomm); + + // prolongate + prolongate(s, level, comm); + + // correct p on finer level using residual + correct(s, p, level, comm); + + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->comm = &d->comm; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int levels = s->levels; + printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * sizeof(double); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size); + s->e[j] = allocate(64, size); + + for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->comm); + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Residuum: %.6f\n", res); + } +#endif + + return res; +} diff --git a/BasicSolver/2D-mpi/src/solver-rb.c b/BasicSolver/2D-mpi/src/solver-rb.c new file mode 100644 index 0000000..622a11e --- /dev/null +++ b/BasicSolver/2D-mpi/src/solver-rb.c @@ -0,0 +1,106 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "discretization.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->comm = &d->comm; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double epssq = eps * eps; + int pass, jsw, isw; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + jsw = 1; + for (pass = 0; pass < 2; pass++) { + isw = jsw; + commExchange(s->comm, p); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + + if (commIsBoundary(s->comm, BOTTOM)) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (commIsBoundary(s->comm, TOP)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (commIsBoundary(s->comm, LEFT)) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (commIsBoundary(s->comm, RIGHT)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } + + commReduction(&res, SUM); + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + it++; + } + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + + return res; +} diff --git a/BasicSolver/2D-mpi/src/solver.h b/BasicSolver/2D-mpi/src/solver.h index 4ceb40b..dcca9bd 100644 --- a/BasicSolver/2D-mpi/src/solver.h +++ b/BasicSolver/2D-mpi/src/solver.h @@ -9,6 +9,7 @@ #include "comm.h" #include "discretization.h" #include "grid.h" +#include "mpi.h" #include "parameter.h" typedef struct { @@ -17,10 +18,12 @@ typedef struct { /* parameters */ double eps, omega; int itermax; + int levels, presmooth, postsmooth; + double **r, **e; /* communication */ Comm* comm; } Solver; void initSolver(Solver*, Discretization*, Parameter*); -void solve(Solver*, double*, double*); +double solve(Solver*, double*, double*); #endif diff --git a/BasicSolver/2D-seq/Makefile b/BasicSolver/2D-seq/Makefile index 979def5..d33dd17 100644 --- a/BasicSolver/2D-seq/Makefile +++ b/BasicSolver/2D-seq/Makefile @@ -38,9 +38,22 @@ $(BUILD_DIR)/%.s: %.c $(info ===> GENERATE ASM $@) $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ -.PHONY: clean distclean tags info asm format +.PHONY: clean distclean vis vis_clean tags info asm format -clean: +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='pressure.dat'" ./surface.plot + @gnuplot -e "filename='velocity.dat'" ./vector.plot + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.png + @rm -f ./vis_files/*.dat + @rm -f ./vis_files/*.gif + +clean: vis_clean $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @rm -f tags diff --git a/BasicSolver/2D-seq/canal.par b/BasicSolver/2D-seq/canal.par index 8b9fd9c..6ed28e6 100644 --- a/BasicSolver/2D-seq/canal.par +++ b/BasicSolver/2D-seq/canal.par @@ -36,6 +36,13 @@ te 100.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- diff --git a/BasicSolver/2D-seq/config.mk b/BasicSolver/2D-seq/config.mk index dc874ee..0b2c509 100644 --- a/BasicSolver/2D-seq/config.mk +++ b/BasicSolver/2D-seq/config.mk @@ -1,12 +1,12 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC ENABLE_OPENMP ?= false # Supported: sor, rb, mg -SOLVER ?= rb +SOLVER ?= mg # Run in debug settings DEBUG ?= false #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 -#OPTIONS += -DVERBOSE +OPTIONS += -DVERBOSE #OPTIONS += -DDEBUG diff --git a/BasicSolver/2D-seq/dcavity.par b/BasicSolver/2D-seq/dcavity.par index 798819f..c4366a1 100644 --- a/BasicSolver/2D-seq/dcavity.par +++ b/BasicSolver/2D-seq/dcavity.par @@ -36,6 +36,13 @@ te 10.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Solver Data: # ----------------------- @@ -44,5 +51,4 @@ eps 0.001 # stopping tolerance for pressure iteration rho 0.5 omg 1.7 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma -levels 5 # Multigrid levels #=============================================================================== diff --git a/BasicSolver/2D-seq/residual.plot b/BasicSolver/2D-seq/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/BasicSolver/2D-seq/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/2D-seq/src/main.c b/BasicSolver/2D-seq/src/main.c index 822e32a..5f617fa 100644 --- a/BasicSolver/2D-seq/src/main.c +++ b/BasicSolver/2D-seq/src/main.c @@ -20,8 +20,10 @@ int main(int argc, char** argv) Parameter p; Discretization d; Solver s; - + initParameter(&p); + FILE* fp; + fp = initResidualWriter(); if (argc != 2) { printf("Usage: %s \n", argv[0]); @@ -32,7 +34,7 @@ int main(int argc, char** argv) printParameter(&p); initDiscretization(&d, &p); initSolver(&s, &d, &p); - + #ifndef VERBOSE initProgress(d.te); #endif @@ -41,8 +43,9 @@ int main(int argc, char** argv) double te = d.te; double t = 0.0; int nt = 0; + double res = 0.0; - timeStart = getTimeStamp(); + timeStart = getTimeStamp(); while (t <= te) { if (tau > 0.0) computeTimestep(&d); setBoundaryConditions(&d); @@ -50,17 +53,21 @@ int main(int argc, char** argv) computeFG(&d); computeRHS(&d); if (nt % 100 == 0) normalizePressure(&d); - solve(&s, d.p, d.rhs); + res = solve(&s, d.p, d.rhs); adaptUV(&d); + + writeResidual(fp, t, res); + t += d.dt; nt++; #ifdef VERBOSE - printf("TIME %f , TIMESTEP %f\n", t, solver.dt); + printf("TIME %f , TIMESTEP %f\n", t, d.dt); #else printProgress(t); #endif } + fclose(fp); timeStop = getTimeStamp(); stopProgress(); printf("Solution took %.2fs\n", timeStop - timeStart); diff --git a/BasicSolver/2D-seq/src/parameter.c b/BasicSolver/2D-seq/src/parameter.c index e559545..b7c32ca 100644 --- a/BasicSolver/2D-seq/src/parameter.c +++ b/BasicSolver/2D-seq/src/parameter.c @@ -24,6 +24,8 @@ void initParameter(Parameter* param) param->gamma = 0.9; param->tau = 0.5; param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) @@ -79,6 +81,8 @@ void readParameter(Parameter* param, const char* filename) PARSE_REAL(u_init); PARSE_REAL(v_init); PARSE_REAL(p_init); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); } } diff --git a/BasicSolver/2D-seq/src/parameter.h b/BasicSolver/2D-seq/src/parameter.h index 5098794..3714f96 100644 --- a/BasicSolver/2D-seq/src/parameter.h +++ b/BasicSolver/2D-seq/src/parameter.h @@ -18,6 +18,7 @@ typedef struct { char* name; int bcLeft, bcRight, bcBottom, bcTop; double u_init, v_init, p_init; + int presmooth, postsmooth; } Parameter; void initParameter(Parameter*); diff --git a/BasicSolver/2D-seq/src/progress.c b/BasicSolver/2D-seq/src/progress.c index d5393c4..bcf6e5f 100644 --- a/BasicSolver/2D-seq/src/progress.c +++ b/BasicSolver/2D-seq/src/progress.c @@ -49,3 +49,22 @@ void stopProgress() printf("\n"); fflush(stdout); } + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} diff --git a/BasicSolver/2D-seq/src/progress.h b/BasicSolver/2D-seq/src/progress.h index 314e921..a79ecfe 100644 --- a/BasicSolver/2D-seq/src/progress.h +++ b/BasicSolver/2D-seq/src/progress.h @@ -10,5 +10,7 @@ extern void initProgress(double); extern void printProgress(double); extern void stopProgress(void); +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/BasicSolver/2D-seq/src/solver-mg.c b/BasicSolver/2D-seq/src/solver-mg.c index d1e56b4..acd3d3b 100644 --- a/BasicSolver/2D-seq/src/solver-mg.c +++ b/BasicSolver/2D-seq/src/solver-mg.c @@ -84,6 +84,37 @@ static double smooth(Solver* s, double* p, double* rhs, int level, int imax, int jsw = 1; + for (pass = 0; pass < 2; pass++) { + isw = jsw; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + P(i, j) -= factor * (RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2)); + + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, int imax, int jmax) +{ + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + for (pass = 0; pass < 2; pass++) { isw = jsw; @@ -94,7 +125,6 @@ static double smooth(Solver* s, double* p, double* rhs, int level, int imax, int ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - P(i, j) -= (factor * R(i, j)); res += (R(i, j) * R(i, j)); } isw = 3 - isw; @@ -110,7 +140,7 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax, { double res = 0.0; - // coarsest level TODO: Use direct solver? + // coarsest level if (level == COARSEST_LEVEL) { for (int i = 0; i < 5; i++) { smooth(s, p, rhs, level, imax, jmax); @@ -118,17 +148,18 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax, return res; } - // pre-smoothing TODO: Make smoothing steps configurable? - for (int i = 0; i < 5; i++) { + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { smooth(s, p, rhs, level, imax, jmax); if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); } + res = calculateResidual(s, p, rhs, level, imax, jmax); + // restrict restrictMG(s, level, imax, jmax); // MGSolver on residual and error. - // TODO: What if there is a rest? multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, imax / 2, jmax / 2); // prolongate @@ -139,8 +170,8 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax, if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); // post-smoothing - for (int i = 0; i < 5; i++) { - res = smooth(s, p, rhs, level, imax, jmax); + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, imax, jmax); if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); } @@ -154,6 +185,8 @@ void initSolver(Solver* s, Discretization* d, Parameter* p) s->itermax = p->itermax; s->levels = p->levels; s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; int imax = s->grid->imax; int jmax = s->grid->jmax; @@ -176,11 +209,13 @@ void initSolver(Solver* s, Discretization* d, Parameter* p) } } -void solve(Solver* s, double* p, double* rhs) +double solve(Solver* s, double* p, double* rhs) { double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax); #ifdef VERBOSE printf("Residuum: %.6f\n", res); #endif + +return res; } diff --git a/BasicSolver/2D-seq/src/solver-rb.c b/BasicSolver/2D-seq/src/solver-rb.c index 265456a..a9facf4 100644 --- a/BasicSolver/2D-seq/src/solver-rb.c +++ b/BasicSolver/2D-seq/src/solver-rb.c @@ -15,7 +15,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p) s->omega = p->omg; } -void solve(Solver* solver, double* p, double* rhs) +double solve(Solver* solver, double* p, double* rhs) { int imax = solver->grid->imax; int jmax = solver->grid->jmax; @@ -73,4 +73,6 @@ void solve(Solver* solver, double* p, double* rhs) #ifdef VERBOSE printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); #endif + + return res; } diff --git a/BasicSolver/2D-seq/src/solver-sor.c b/BasicSolver/2D-seq/src/solver-sor.c index 3d21ea5..21af4eb 100644 --- a/BasicSolver/2D-seq/src/solver-sor.c +++ b/BasicSolver/2D-seq/src/solver-sor.c @@ -15,7 +15,7 @@ void initSolver(Solver* s, Discretization* d, Parameter* p) s->omega = p->omg; } -void solve(Solver* solver, double* p, double* rhs) +double solve(Solver* solver, double* p, double* rhs) { int imax = solver->grid->imax; int jmax = solver->grid->jmax; @@ -65,4 +65,6 @@ void solve(Solver* solver, double* p, double* rhs) #ifdef VERBOSE printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); #endif + + return res; } diff --git a/BasicSolver/2D-seq/src/solver.h b/BasicSolver/2D-seq/src/solver.h index d7ce513..0343f6e 100644 --- a/BasicSolver/2D-seq/src/solver.h +++ b/BasicSolver/2D-seq/src/solver.h @@ -18,9 +18,10 @@ typedef struct { int itermax; int levels; double **r, **e; + int presmooth, postsmooth; } Solver; extern void initSolver(Solver*, Discretization*, Parameter*); -extern void solve(Solver*, double*, double*); +extern double solve(Solver*, double*, double*); #endif diff --git a/BasicSolver/3D-mpi/Makefile b/BasicSolver/3D-mpi/Makefile index f61416c..883bb88 100644 --- a/BasicSolver/3D-mpi/Makefile +++ b/BasicSolver/3D-mpi/Makefile @@ -22,6 +22,7 @@ SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) OBJ += $(BUILD_DIR)/vtkWriter-$(VTK_OUTPUT_FMT).o +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) ifeq ($(VTK_OUTPUT_FMT),mpi) DEFINES += -D_VTK_WRITER_MPI @@ -42,9 +43,19 @@ $(BUILD_DIR)/%.s: %.c $(info ===> GENERATE ASM $@) $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ -.PHONY: clean distclean tags info asm format +.PHONY: clean distclean vis vis_clean tags info asm format -clean: +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.vtk + @rm -f *.png + +clean: vis_clean $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @rm -f tags diff --git a/BasicSolver/3D-mpi/canal.par b/BasicSolver/3D-mpi/canal.par index 3ff4a5f..b8d35c0 100644 --- a/BasicSolver/3D-mpi/canal.par +++ b/BasicSolver/3D-mpi/canal.par @@ -32,21 +32,28 @@ xlength 30.0 # domain size in x-direction ylength 4.0 # domain size in y-direction zlength 4.0 # domain size in z-direction imax 200 # number of interior cells in x-direction -jmax 50 # number of interior cells in y-direction -kmax 50 # number of interior cells in z-direction +jmax 40 # number of interior cells in y-direction +kmax 40 # number of interior cells in z-direction # Time Data: # --------- -te 100.0 # final time +te 60.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- itermax 500 # maximal number of pressure iteration in one time step eps 0.0001 # stopping tolerance for pressure iteration -omg 1.3 # relaxation parameter for SOR iteration +omg 1.7 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma #=============================================================================== diff --git a/BasicSolver/3D-mpi/config.mk b/BasicSolver/3D-mpi/config.mk index 52154f4..0dbe440 100644 --- a/BasicSolver/3D-mpi/config.mk +++ b/BasicSolver/3D-mpi/config.mk @@ -1,7 +1,10 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC +# Supported: true, false ENABLE_MPI ?= true ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg # Supported: seq, mpi VTK_OUTPUT_FMT ?= seq diff --git a/BasicSolver/3D-mpi/dcavity.par b/BasicSolver/3D-mpi/dcavity.par index c5cb201..77f1991 100644 --- a/BasicSolver/3D-mpi/dcavity.par +++ b/BasicSolver/3D-mpi/dcavity.par @@ -18,7 +18,7 @@ gx 0.0 # Body forces (e.g. gravity) gy 0.0 # gz 0.0 # -re 1000.0 # Reynolds number +re 1000.0 # Reynolds number u_init 0.0 # initial value for velocity in x-direction v_init 0.0 # initial value for velocity in y-direction @@ -42,6 +42,13 @@ te 10.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- diff --git a/BasicSolver/3D-mpi/residual.plot b/BasicSolver/3D-mpi/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/BasicSolver/3D-mpi/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/3D-mpi/src/comm.c b/BasicSolver/3D-mpi/src/comm.c index 327bbbe..df9f795 100644 --- a/BasicSolver/3D-mpi/src/comm.c +++ b/BasicSolver/3D-mpi/src/comm.c @@ -167,11 +167,12 @@ static void assembleResult(Comm* c, MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); } -static int sum(int* sizes, int position) +// subroutines local to this module +static int sum(int* sizes, int init, int offset, int coord) { int sum = 0; - for (int i = 0; i < position; i++) { + for (int i = init - offset; coord > 0; i -= offset, --coord) { sum += sizes[i]; } @@ -336,7 +337,6 @@ void commCollectResult(Comm* c, int imaxLocal = c->imaxLocal; int jmaxLocal = c->jmaxLocal; int kmaxLocal = c->kmaxLocal; - #if defined(_MPI) int offset[c->size * NDIMS]; int imaxLocalAll[c->size]; @@ -351,9 +351,13 @@ void commCollectResult(Comm* c, for (int i = 0; i < c->size; i++) { int coords[NCORDS]; MPI_Cart_coords(c->comm, i, NDIMS, coords); - offset[i * NDIMS + IDIM] = sum(imaxLocalAll, coords[ICORD]); - offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, coords[JCORD]); - offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, coords[KCORD]); + offset[i * NDIMS + IDIM] = sum(imaxLocalAll, + i, + c->dims[IDIM] * c->dims[JDIM], + coords[ICORD]); + offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, i, c->dims[IDIM], coords[JCORD]); + offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, i, 1, coords[KCORD]); + printf("Rank: %d, Coords(k,j,i): %d %d %d, Size(k,j,i): %d %d %d, " "Offset(k,j,i): %d %d %d\n", i, @@ -562,9 +566,9 @@ void commPartition(Comm* c, int kmax, int jmax, int imax) MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]); MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords); - c->imaxLocal = sizeOfRank(c->rank, dims[ICORD], imax); - c->jmaxLocal = sizeOfRank(c->rank, dims[JCORD], jmax); - c->kmaxLocal = sizeOfRank(c->rank, dims[KCORD], kmax); + c->imaxLocal = sizeOfRank(c->coords[KDIM], dims[ICORD], imax); + c->jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JCORD], jmax); + c->kmaxLocal = sizeOfRank(c->coords[IDIM], dims[KCORD], kmax); // setup buffer types for communication setupCommunication(c, LEFT, BULK); @@ -597,3 +601,47 @@ void commFinalize(Comm* c) MPI_Finalize(); #endif } + +void commUpdateDatatypes( + Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal) +{ +#if defined _MPI + + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + newcomm->kmaxLocal = kmaxLocal / 2; + + setupCommunication(newcomm, LEFT, BULK); + setupCommunication(newcomm, LEFT, HALO); + setupCommunication(newcomm, RIGHT, BULK); + setupCommunication(newcomm, RIGHT, HALO); + setupCommunication(newcomm, BOTTOM, BULK); + setupCommunication(newcomm, BOTTOM, HALO); + setupCommunication(newcomm, TOP, BULK); + setupCommunication(newcomm, TOP, HALO); + setupCommunication(newcomm, FRONT, BULK); + setupCommunication(newcomm, FRONT, HALO); + setupCommunication(newcomm, BACK, BULK); + setupCommunication(newcomm, BACK, HALO); +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; + newcomm->kmaxLocal = kmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/BasicSolver/3D-mpi/src/comm.h b/BasicSolver/3D-mpi/src/comm.h index 48102fb..58ed4d7 100644 --- a/BasicSolver/3D-mpi/src/comm.h +++ b/BasicSolver/3D-mpi/src/comm.h @@ -23,6 +23,7 @@ typedef enum dimension { KDIM = 0, JDIM, IDIM, NDIMS } Dimension; enum layer { HALO = 0, BULK }; enum op { MAX = 0, SUM }; + typedef struct { int rank; int size; @@ -45,6 +46,9 @@ extern void commShift(Comm* c, double* f, double* g, double* h); extern void commReduction(double* v, int op); extern int commIsBoundary(Comm* c, Direction direction); extern void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax); +extern void commFreeCommunicator(Comm* comm); +extern void commUpdateDatatypes( + Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal); extern void commCollectResult(Comm* c, double* ug, double* vg, diff --git a/BasicSolver/3D-mpi/src/discretization.c b/BasicSolver/3D-mpi/src/discretization.c new file mode 100644 index 0000000..138a037 --- /dev/null +++ b/BasicSolver/3D-mpi/src/discretization.c @@ -0,0 +1,729 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "allocate.h" +#include "discretization.h" +#include "parameter.h" +#include "util.h" + +#define P(i, j, k) \ + p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define F(i, j, k) \ + f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define G(i, j, k) \ + g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define H(i, j, k) \ + h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define U(i, j, k) \ + u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define V(i, j, k) \ + v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define W(i, j, k) \ + w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define RHS(i, j, k) \ + rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +static void printConfig(Discretization* s) +{ + if (commIsMaster(&s->comm)) { + printf("Parameters for #%s#\n", s->problem); + printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n", + s->bcLeft, + s->bcRight, + s->bcBottom, + s->bcTop, + s->bcFront, + s->bcBack); + printf("\tReynolds number: %.2f\n", s->re); + printf("\tGx Gy: %.2f %.2f %.2f\n", s->gx, s->gy, s->gz); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + s->grid.xlength, + s->grid.ylength, + s->grid.zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", + s->grid.imax, + s->grid.jmax, + s->grid.kmax); + printf("\tCell size (dx, dy, dz): %f, %f, %f\n", + s->grid.dx, + s->grid.dy, + s->grid.dz); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te); + printf("\tdt bound: %.6f\n", s->dtBound); + printf("\tTau factor: %.2f\n", s->tau); + printf("Iterative parameters:\n"); + printf("\tMax iterations: %d\n", s->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", s->eps); + printf("\tgamma factor: %f\n", s->gamma); + printf("\tomega (SOR relaxation): %f\n", s->omega); + } + commPrintConfig(&s->comm); +} + +void initDiscretization(Discretization* s, Parameter* params) +{ + s->problem = params->name; + s->bcLeft = params->bcLeft; + s->bcRight = params->bcRight; + s->bcBottom = params->bcBottom; + s->bcTop = params->bcTop; + s->bcFront = params->bcFront; + s->bcBack = params->bcBack; + + s->grid.imax = params->imax; + s->grid.jmax = params->jmax; + s->grid.kmax = params->kmax; + s->grid.xlength = params->xlength; + s->grid.ylength = params->ylength; + s->grid.zlength = params->zlength; + s->grid.dx = params->xlength / params->imax; + s->grid.dy = params->ylength / params->jmax; + s->grid.dz = params->zlength / params->kmax; + + s->eps = params->eps; + s->omega = params->omg; + s->itermax = params->itermax; + s->re = params->re; + s->gx = params->gx; + s->gy = params->gy; + s->gz = params->gz; + s->dt = params->dt; + s->te = params->te; + s->tau = params->tau; + s->gamma = params->gamma; + + /* allocate arrays */ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + size_t size = (imaxLocal + 2) * (jmaxLocal + 2) * (kmaxLocal + 2); + + s->u = allocate(64, size * sizeof(double)); + s->v = allocate(64, size * sizeof(double)); + s->w = allocate(64, size * sizeof(double)); + s->p = allocate(64, size * sizeof(double)); + s->rhs = allocate(64, size * sizeof(double)); + s->f = allocate(64, size * sizeof(double)); + s->g = allocate(64, size * sizeof(double)); + s->h = allocate(64, size * sizeof(double)); + + for (int i = 0; i < size; i++) { + s->u[i] = params->u_init; + s->v[i] = params->v_init; + s->w[i] = params->w_init; + s->p[i] = params->p_init; + s->rhs[i] = 0.0; + s->f[i] = 0.0; + s->g[i] = 0.0; + s->h[i] = 0.0; + } + + double dx = s->grid.dx; + double dy = s->grid.dy; + double dz = s->grid.dz; + + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz); + s->dtBound = 0.5 * s->re * 1.0 / invSqrSum; + +#ifdef VERBOSE + printConfig(s); +#endif /* VERBOSE */ +} + +void setBoundaryConditions(Discretization* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* u = s->u; + double* v = s->v; + double* w = s->w; + + if (commIsBoundary(&s->comm, TOP)) { + switch (s->bcTop) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = -U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = 0.0; + W(i, jmaxLocal + 1, k) = -W(i, jmaxLocal, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = 0.0; + W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = V(i, jmaxLocal - 1, k); + W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, BOTTOM)) { + switch (s->bcBottom) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = -U(i, 1, k); + V(i, 0, k) = 0.0; + W(i, 0, k) = -W(i, 1, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = 0.0; + W(i, 0, k) = W(i, 1, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = V(i, 1, k); + W(i, 0, k) = W(i, 1, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, LEFT)) { + switch (s->bcLeft) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = -V(1, j, k); + W(0, j, k) = -W(1, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = U(1, j, k); + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, RIGHT)) { + switch (s->bcRight) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = 0.0; + V(imaxLocal + 1, j, k) = -V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = -W(imaxLocal, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = 0.0; + V(imaxLocal + 1, j, k) = V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = W(imaxLocal, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = U(imaxLocal - 1, j, k); + V(imaxLocal + 1, j, k) = V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = W(imaxLocal, j, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, FRONT)) { + switch (s->bcFront) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = -U(i, j, 1); + V(i, j, 0) = -V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = W(i, j, 1); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, BACK)) { + switch (s->bcBack) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = -U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = -V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = W(i, j, kmaxLocal - 1); + } + } + break; + case PERIODIC: + break; + } + } +} + +void computeRHS(Discretization* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double idx = 1.0 / s->grid.dx; + double idy = 1.0 / s->grid.dy; + double idz = 1.0 / s->grid.dz; + double idt = 1.0 / s->dt; + + double* rhs = s->rhs; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + commShift(&s->comm, f, g, h); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx + + (G(i, j, k) - G(i, j - 1, k)) * idy + + (H(i, j, k) - H(i, j, k - 1)) * idz) * + idt; + } + } + } +} + +void setSpecialBoundaryCondition(Discretization* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* u = s->u; + + if (strcmp(s->problem, "dcavity") == 0) { + if (commIsBoundary(&s->comm, TOP)) { + for (int k = 1; k < kmaxLocal; k++) { + for (int i = 1; i < imaxLocal; i++) { + U(i, jmaxLocal + 1, k) = 2.0 - U(i, jmaxLocal, k); + } + } + } + } else if (strcmp(s->problem, "canal") == 0) { + if (commIsBoundary(&s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 2.0; + } + } + } + } +} + +static double maxElement(Discretization* s, double* m) +{ + int size = (s->comm.imaxLocal + 2) * (s->comm.jmaxLocal + 2) * + (s->comm.kmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + commReduction(&maxval, MAX); + return maxval; +} + +void normalizePressure(Discretization* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* p = s->p; + double avgP = 0.0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + avgP += P(i, j, k); + } + } + } + commReduction(&avgP, SUM); + avgP /= (s->grid.imax * s->grid.jmax * s->grid.kmax); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, k) = P(i, j, k) - avgP; + } + } + } +} + +void computeTimestep(Discretization* s) +{ + double dt = s->dtBound; + double dx = s->grid.dx; + double dy = s->grid.dy; + double dz = s->grid.dz; + + double umax = maxElement(s, s->u); + double vmax = maxElement(s, s->v); + double wmax = maxElement(s, s->w); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + if (wmax > 0) { + dt = (dt > dz / wmax) ? dz / wmax : dt; + } + + s->dt = dt * s->tau; +} + + + +void computeFG(Discretization* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* u = s->u; + double* v = s->v; + double* w = s->w; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + double gx = s->gx; + double gy = s->gy; + double gz = s->gz; + double dt = s->dt; + + double gamma = s->gamma; + double inverseRe = 1.0 / s->re; + double inverseDx = 1.0 / s->grid.dx; + double inverseDy = 1.0 / s->grid.dy; + double inverseDz = 1.0 / s->grid.dz; + double du2dx, dv2dy, dw2dz; + double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz; + double du2dx2, du2dy2, du2dz2; + double dv2dx2, dv2dy2, dv2dz2; + double dw2dx2, dw2dy2, dw2dz2; + + commExchange(&s->comm, u); + commExchange(&s->comm, v); + commExchange(&s->comm, w); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + du2dx = inverseDx * 0.25 * + ((U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) + U(i + 1, j, k)) - + (U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) + U(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) - U(i + 1, j, k)) + + fabs(U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) - U(i - 1, j, k))); + + duvdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) + U(i, j + 1, k)) - + (V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) + U(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) - U(i, j + 1, k)) + + fabs(V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) - U(i, j - 1, k))); + + duwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) + U(i, j, k + 1)) - + (W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) + U(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) - U(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) - U(i, j, k - 1))); + + du2dx2 = inverseDx * inverseDx * + (U(i + 1, j, k) - 2.0 * U(i, j, k) + U(i - 1, j, k)); + du2dy2 = inverseDy * inverseDy * + (U(i, j + 1, k) - 2.0 * U(i, j, k) + U(i, j - 1, k)); + du2dz2 = inverseDz * inverseDz * + (U(i, j, k + 1) - 2.0 * U(i, j, k) + U(i, j, k - 1)); + F(i, j, k) = U(i, j, k) + dt * (inverseRe * (du2dx2 + du2dy2 + du2dz2) - + du2dx - duvdy - duwdz + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) + V(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) + V(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) - V(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) - V(i - 1, j, k))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) + V(i, j + 1, k)) - + (V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) + V(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) - V(i, j + 1, k)) + + fabs(V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) - V(i, j - 1, k))); + + dvwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) + V(i, j, k + 1)) - + (W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) + V(i, j, k + 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) - V(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) - V(i, j, k + 1))); + + dv2dx2 = inverseDx * inverseDx * + (V(i + 1, j, k) - 2.0 * V(i, j, k) + V(i - 1, j, k)); + dv2dy2 = inverseDy * inverseDy * + (V(i, j + 1, k) - 2.0 * V(i, j, k) + V(i, j - 1, k)); + dv2dz2 = inverseDz * inverseDz * + (V(i, j, k + 1) - 2.0 * V(i, j, k) + V(i, j, k - 1)); + G(i, j, k) = V(i, j, k) + dt * (inverseRe * (dv2dx2 + dv2dy2 + dv2dz2) - + duvdx - dv2dy - dvwdz + gy); + + duwdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) + W(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) + W(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) - W(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) - W(i - 1, j, k))); + + dvwdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) + W(i, j + 1, k)) - + (V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) + W(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) - W(i, j + 1, k)) + + fabs(V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) - W(i, j - 1, k))); + + dw2dz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) + W(i, j, k + 1)) - + (W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) + W(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) - W(i, j, k + 1)) + + fabs(W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) - W(i, j, k - 1))); + + dw2dx2 = inverseDx * inverseDx * + (W(i + 1, j, k) - 2.0 * W(i, j, k) + W(i - 1, j, k)); + dw2dy2 = inverseDy * inverseDy * + (W(i, j + 1, k) - 2.0 * W(i, j, k) + W(i, j - 1, k)); + dw2dz2 = inverseDz * inverseDz * + (W(i, j, k + 1) - 2.0 * W(i, j, k) + W(i, j, k - 1)); + H(i, j, k) = W(i, j, k) + dt * (inverseRe * (dw2dx2 + dw2dy2 + dw2dz2) - + duwdx - dvwdy - dw2dz + gz); + } + } + } + + /* ----------------------------- boundary of F --------------------------- + */ + if (commIsBoundary(&s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(0, j, k) = U(0, j, k); + } + } + } + + if (commIsBoundary(&s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(imaxLocal, j, k) = U(imaxLocal, j, k); + } + } + } + + /* ----------------------------- boundary of G --------------------------- + */ + if (commIsBoundary(&s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, 0, k) = V(i, 0, k); + } + } + } + + if (commIsBoundary(&s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, jmaxLocal, k) = V(i, jmaxLocal, k); + } + } + } + + /* ----------------------------- boundary of H --------------------------- + */ + if (commIsBoundary(&s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + H(i, j, 0) = W(i, j, 0); + } + } + } + + if (commIsBoundary(&s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + H(i, j, kmaxLocal) = W(i, j, kmaxLocal); + } + } + } +} + +void adaptUV(Discretization* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* p = s->p; + double* u = s->u; + double* v = s->v; + double* w = s->w; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + double factorX = s->dt / s->grid.dx; + double factorY = s->dt / s->grid.dy; + double factorZ = s->dt / s->grid.dz; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, k) = F(i, j, k) - (P(i + 1, j, k) - P(i, j, k)) * factorX; + V(i, j, k) = G(i, j, k) - (P(i, j + 1, k) - P(i, j, k)) * factorY; + W(i, j, k) = H(i, j, k) - (P(i, j, k + 1) - P(i, j, k)) * factorZ; + } + } + } +} \ No newline at end of file diff --git a/BasicSolver/3D-mpi/src/discretization.h b/BasicSolver/3D-mpi/src/discretization.h new file mode 100644 index 0000000..2c04800 --- /dev/null +++ b/BasicSolver/3D-mpi/src/discretization.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ + +#include "grid.h" +#include "parameter.h" +#include "comm.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g, *h; + double *u, *v, *w; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy, gz; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + Comm comm; +} Discretization; + +extern void initDiscretization(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +#endif diff --git a/BasicSolver/3D-mpi/src/main.c b/BasicSolver/3D-mpi/src/main.c index 153abbf..3b68f70 100644 --- a/BasicSolver/3D-mpi/src/main.c +++ b/BasicSolver/3D-mpi/src/main.c @@ -9,7 +9,7 @@ #include #include "allocate.h" -#include "comm.h" +#include "discretization.h" #include "parameter.h" #include "progress.h" #include "solver.h" @@ -21,9 +21,12 @@ int main(int argc, char** argv) double timeStart, timeStop; Parameter p; Solver s; + Discretization d; - commInit(&s.comm, argc, argv); + commInit(&d.comm, argc, argv); initParameter(&p); + FILE* fp; + if (commIsMaster(&d.comm)) fp = initResidualWriter(); if (argc != 2) { printf("Usage: %s \n", argv[0]); @@ -31,33 +34,44 @@ int main(int argc, char** argv) } readParameter(&p, argv[1]); - commPartition(&s.comm, p.kmax, p.jmax, p.imax); - if (commIsMaster(&s.comm)) { + commPartition(&d.comm, p.kmax, p.jmax, p.imax); + + if (commIsMaster(&d.comm)) { printParameter(&p); } - initSolver(&s, &p); + + initDiscretization(&d, &p); + initSolver(&s, &d, &p); + #ifndef VERBOSE - initProgress(s.te); + initProgress(d.te); #endif - double tau = s.tau; - double te = s.te; + double tau = d.tau; + double te = d.te; double t = 0.0; + int nt = 0; + double res = 0.0; timeStart = getTimeStamp(); while (t <= te) { - if (tau > 0.0) computeTimestep(&s); - setBoundaryConditions(&s); - setSpecialBoundaryCondition(&s); - computeFG(&s); - computeRHS(&s); - solve(&s); - adaptUV(&s); - t += s.dt; + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + computeFG(&d); + computeRHS(&d); + if (nt % 100 == 0) normalizePressure(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + + if (commIsMaster(&d.comm)) writeResidual(fp, t, res); + + t += d.dt; + nt++; #ifdef VERBOSE - if (commIsMaster(&s.comm)) { - printf("TIME %f , TIMESTEP %f\n", t, s.dt); + if (commIsMaster(s.comm)) { + printf("TIME %f , TIMESTEP %f\n", t, d.dt); } #else printProgress(t); @@ -67,7 +81,7 @@ int main(int argc, char** argv) #ifndef VERBOSE stopProgress(); #endif - if (commIsMaster(&s.comm)) { + if (commIsMaster(s.comm)) { printf("Solution took %.2fs\n", timeStop - timeStart); } @@ -75,14 +89,16 @@ int main(int argc, char** argv) #ifdef _VTK_WRITER_MPI VtkOptions opts = { .grid = s.grid, .comm = s.comm }; vtkOpen(&opts, s.problem); - vtkScalar(&opts, "pressure", s.p); - vtkVector(&opts, "velocity", (VtkVector) { s.u, s.v, s.w }); + vtkScalar(&opts, "pressure", d.p); + vtkVector(&opts, "velocity", (VtkVector) { d.u, d.v, d.w }); vtkClose(&opts); #else + if (commIsMaster(&d.comm)) fclose(fp); + double *pg, *ug, *vg, *wg; - if (commIsMaster(&s.comm)) { - size_t bytesize = s.grid.imax * s.grid.jmax * s.grid.kmax * sizeof(double); + if (commIsMaster(s.comm)) { + size_t bytesize = s.grid->imax * s.grid->jmax * s.grid->kmax * sizeof(double); pg = allocate(64, bytesize); ug = allocate(64, bytesize); @@ -90,34 +106,35 @@ int main(int argc, char** argv) wg = allocate(64, bytesize); } - commCollectResult(&s.comm, + commCollectResult(s.comm, ug, vg, wg, pg, - s.u, - s.v, - s.w, - s.p, - s.grid.kmax, - s.grid.jmax, - s.grid.imax); + d.u, + d.v, + d.w, + d.p, + s.grid->kmax, + s.grid->jmax, + s.grid->imax); - if (commIsMaster(&s.comm)) { + if (commIsMaster(s.comm)) { VtkOptions opts = { .grid = s.grid }; vtkOpen(&opts, s.problem); vtkScalar(&opts, "pressure", pg); vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); vtkClose(&opts); } + #endif timeStop = getTimeStamp(); - if (commIsMaster(&s.comm)) { + if (commIsMaster(s.comm)) { printf("Result output took %.2fs\n", timeStop - timeStart); } - commFinalize(&s.comm); + commFinalize(s.comm); return EXIT_SUCCESS; } diff --git a/BasicSolver/3D-mpi/src/parameter.c b/BasicSolver/3D-mpi/src/parameter.c index 2128c2a..3f98561 100644 --- a/BasicSolver/3D-mpi/src/parameter.c +++ b/BasicSolver/3D-mpi/src/parameter.c @@ -14,18 +14,22 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->zlength = 1.0; - param->imax = 100; - param->jmax = 100; - param->kmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.7; - param->re = 100.0; - param->gamma = 0.9; - param->tau = 0.5; + param->xlength = 1.0; + param->ylength = 1.0; + param->zlength = 1.0; + param->imax = 100; + param->jmax = 100; + param->kmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; + } void readParameter(Parameter* param, const char* filename) @@ -65,6 +69,9 @@ void readParameter(Parameter* param, const char* filename) PARSE_INT(jmax); PARSE_INT(kmax); PARSE_INT(itermax); + PARSE_INT(levels); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); PARSE_REAL(eps); PARSE_REAL(omg); PARSE_REAL(re); diff --git a/BasicSolver/3D-mpi/src/parameter.h b/BasicSolver/3D-mpi/src/parameter.h index d946d5c..e40b9a4 100644 --- a/BasicSolver/3D-mpi/src/parameter.h +++ b/BasicSolver/3D-mpi/src/parameter.h @@ -18,6 +18,7 @@ typedef struct { char* name; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; double u_init, v_init, w_init, p_init; + int levels, presmooth, postsmooth; } Parameter; void initParameter(Parameter*); diff --git a/BasicSolver/3D-mpi/src/progress.c b/BasicSolver/3D-mpi/src/progress.c index f03dc02..8249ed1 100644 --- a/BasicSolver/3D-mpi/src/progress.c +++ b/BasicSolver/3D-mpi/src/progress.c @@ -4,12 +4,12 @@ * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ +#include "progress.h" #include #include +#include #include -#include "progress.h" - static double _end; static int _current; @@ -48,3 +48,18 @@ void stopProgress() printf("\n"); fflush(stdout); } + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; +} + +void writeResidual(FILE* fp, double ts, double res) { fprintf(fp, "%f, %f\n", ts, res); } \ No newline at end of file diff --git a/BasicSolver/3D-mpi/src/progress.h b/BasicSolver/3D-mpi/src/progress.h index 314e921..d612ebe 100644 --- a/BasicSolver/3D-mpi/src/progress.h +++ b/BasicSolver/3D-mpi/src/progress.h @@ -4,11 +4,14 @@ * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ + +#include + #ifndef __PROGRESS_H_ #define __PROGRESS_H_ - extern void initProgress(double); extern void printProgress(double); extern void stopProgress(void); - +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/BasicSolver/3D-mpi/src/solver-mg.c b/BasicSolver/3D-mpi/src/solver-mg.c new file mode 100644 index 0000000..27b1b18 --- /dev/null +++ b/BasicSolver/3D-mpi/src/solver-mg.c @@ -0,0 +1,403 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +#define S(i, j, k) \ + s[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define E(i, j, k) \ + e[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define R(i, j, k) \ + r[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define OLD(i, j, k) \ + old[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +static void restrictMG(Solver* s, int level, Comm* comm) +{ + double* r = s->r[level + 1]; + double* old = s->r[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + commExchange(comm, old); + + for (int k = 1; k < comm->kmaxLocal + 1; k++) { + for (int j = 1; j < comm->jmaxLocal + 1; j++) { + for (int i = 1; i < comm->imaxLocal + 1; ++i) { + R(i, j, k) = (OLD(2 * i - 1, 2 * j - 1, 2 * k) + + OLD(2 * i, 2 * j - 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k) + + OLD(2 * i - 1, 2 * j, 2 * k) * 2 + + OLD(2 * i, 2 * j, 2 * k) * 8 + + OLD(2 * i + 1, 2 * j, 2 * k) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k) + + OLD(2 * i, 2 * j + 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i, 2 * j - 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i - 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i, 2 * j, 2 * k - 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k - 1) + + OLD(2 * i, 2 * j + 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k - 1) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i, 2 * j - 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i - 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i, 2 * j, 2 * k + 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k + 1) + + OLD(2 * i, 2 * j + 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k + 1)) / + 64.0; + } + } + } +} + +static void prolongate(Solver* s, int level, Comm* comm) +{ + double* old = s->r[level + 1]; + double* e = s->r[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + for (int k = 2; k < kmaxLocal + 1; k += 2) { + for (int j = 2; j < jmaxLocal + 1; j += 2) { + for (int i = 2; i < imaxLocal + 1; i += 2) { + E(i, j, k) = OLD(i / 2, j / 2, k / 2); + } + } + } +} + +static void correct(Solver* s, double* p, int level, Comm* comm) +{ + double* e = s->e[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + for (int k = 1; k < kmaxLocal + 1; ++k) { + for (int j = 1; j < jmaxLocal + 1; ++j) { + for (int i = 1; i < imaxLocal + 1; ++i) { + P(i, j, k) += E(i, j, k); + } + } + } +} + +static void setBoundaryCondition( + Solver* s, double* p, int imaxLocal, int jmaxLocal, int kmaxLocal) +{ +#ifdef _MPI + if (commIsBoundary(s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + } + } + } + + if (commIsBoundary(s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + } + + if (commIsBoundary(s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + } + } + } + + if (commIsBoundary(s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + } + + if (commIsBoundary(s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + } + } + } + + if (commIsBoundary(s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } + } +#else + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } +#endif +} + +static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + commExchange(comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + P(i, j, k) -= + factor * + (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + commExchange(comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + R(i, + j, + k) = (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * + idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + + res += (R(i, j, k) * R(i, j, k)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + commReduction(&res, SUM); + + res = res / (double)(imaxLocal * jmaxLocal * kmaxLocal); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + return res; +} + +static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, comm); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + } + + res = calculateResidual(s, p, rhs, level, comm); + + // restrict + restrictMG(s, level, comm); + + Comm newcomm; + commUpdateDatatypes(s->comm, + &newcomm, + imaxLocal, + jmaxLocal, + kmaxLocal); + + // MGSolver on residual and error. + multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm); + + commFreeCommunicator(&newcomm); + + // prolongate + prolongate(s, level, comm); + + // correct p on finer level using residual + correct(s, p, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + s->comm = &d->comm; + s->problem = p->name; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + int levels = s->levels; + if (commIsMaster(s->comm)) printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * (kmax + 2); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size * sizeof(double)); + s->e[j] = allocate(64, size * sizeof(double)); + + for (size_t i = 0; i < size; i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->comm); + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Residuum: %.6f\n", res); + } +#endif + + return res; +} diff --git a/BasicSolver/3D-mpi/src/solver-rb.c b/BasicSolver/3D-mpi/src/solver-rb.c new file mode 100644 index 0000000..d64fecd --- /dev/null +++ b/BasicSolver/3D-mpi/src/solver-rb.c @@ -0,0 +1,175 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + s->comm = &d->comm; + s->problem = p->name; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double epssq = eps * eps; + int it = 0; + double res = 1.0; + int pass, ksw, jsw, isw; + + while ((res >= epssq) && (it < itermax)) { + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + commExchange(s->comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + double r = + RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2); + + P(i, j, k) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +#ifdef _MPI + if (commIsBoundary(s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + } + } + } + + if (commIsBoundary(s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + } + + if (commIsBoundary(s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + } + } + } + + if (commIsBoundary(s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + } + + if (commIsBoundary(s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + } + } + } + + if (commIsBoundary(s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } + } +#else + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } +#endif + commReduction(&res, SUM); + res = res / (double)(imax * jmax * kmax); +#ifdef DEBUG + if (commIsMaster(&s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + commExchange(s->comm, p); + it++; + } + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + +return res; +} \ No newline at end of file diff --git a/BasicSolver/3D-mpi/src/solver.h b/BasicSolver/3D-mpi/src/solver.h index ae734f7..d9a0c39 100644 --- a/BasicSolver/3D-mpi/src/solver.h +++ b/BasicSolver/3D-mpi/src/solver.h @@ -9,12 +9,11 @@ #include "comm.h" #include "grid.h" #include "parameter.h" - -enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; +#include "discretization.h" typedef struct { /* geometry and grid information */ - Grid grid; + Grid* grid; /* arrays */ double *p, *rhs; double *f, *g, *h; @@ -30,16 +29,11 @@ typedef struct { char* problem; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; /* communication */ - Comm comm; + double **r, **e; + int levels, presmooth, postsmooth; + Comm* comm; } Solver; -extern void initSolver(Solver*, Parameter*); -extern void computeRHS(Solver*); -extern void solve(Solver*); -extern void normalizePressure(Solver*); -extern void computeTimestep(Solver*); -extern void setBoundaryConditions(Solver*); -extern void setSpecialBoundaryCondition(Solver*); -extern void computeFG(Solver*); -extern void adaptUV(Solver*); +extern double solve(Solver* , double* , double* ); +extern void initSolver(Solver*, Discretization*, Parameter*); #endif diff --git a/BasicSolver/3D-mpi/src/test.c b/BasicSolver/3D-mpi/src/test.c index 4f27776..ba6bc78 100644 --- a/BasicSolver/3D-mpi/src/test.c +++ b/BasicSolver/3D-mpi/src/test.c @@ -14,10 +14,10 @@ void testInit(Solver* s) { - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - int myrank = s->comm.rank; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; + int myrank = s->comm->rank; double* p = s->p; double* f = s->f; double* g = s->g; @@ -76,11 +76,11 @@ static char* direction2String(Direction dir) static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir) { - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; char filename[50]; - snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm.rank); + snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm->rank); FILE* fh = fopen(filename, "w"); for (int y = 0; y < ymax; y++) { @@ -116,9 +116,9 @@ static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir) void testPrintHalo(Solver* s, double* a) { - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, BOTTOM); printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, TOP); diff --git a/BasicSolver/3D-mpi/src/util.h b/BasicSolver/3D-mpi/src/util.h index c27b2ba..68b87f0 100644 --- a/BasicSolver/3D-mpi/src/util.h +++ b/BasicSolver/3D-mpi/src/util.h @@ -19,4 +19,13 @@ #define ABS(a) ((a) >= 0 ? (a) : -(a)) #endif +#define P(i, j, k) p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define F(i, j, k) f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define G(i, j, k) g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define H(i, j, k) h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define U(i, j, k) u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define V(i, j, k) v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define W(i, j, k) w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define RHS(i, j, k) rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + #endif // __UTIL_H_ diff --git a/BasicSolver/3D-mpi/src/vtkWriter-mpi.c b/BasicSolver/3D-mpi/src/vtkWriter-mpi.c index 8b29c56..d9670d5 100644 --- a/BasicSolver/3D-mpi/src/vtkWriter-mpi.c +++ b/BasicSolver/3D-mpi/src/vtkWriter-mpi.c @@ -46,18 +46,18 @@ static void writeHeader(VtkOptions* o) cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n"); cursor += sprintf(cursor, "DIMENSIONS %d %d %d\n", - o->grid.imax, - o->grid.jmax, - o->grid.kmax); + o->grid->imax, + o->grid->jmax, + o->grid->kmax); cursor += sprintf(cursor, "ORIGIN %f %f %f\n", - o->grid.dx * 0.5, - o->grid.dy * 0.5, - o->grid.dz * 0.5); - cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz); + o->grid->dx * 0.5, + o->grid->dy * 0.5, + o->grid->dz * 0.5); + cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz); cursor += sprintf(cursor, "POINT_DATA %d\n", - o->grid.imax * o->grid.jmax * o->grid.kmax); + o->grid->imax * o->grid->jmax * o->grid->kmax); if (commIsMaster(&o->comm)) { MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); @@ -67,7 +67,6 @@ static void writeHeader(VtkOptions* o) void vtkOpen(VtkOptions* o, char* problem) { char filename[50]; - snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size); MPI_File_open(o->comm.comm, filename, @@ -98,7 +97,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s) } int offsets[NDIMS]; - commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax); + commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax); // set global view in file MPI_Offset disp; @@ -108,7 +107,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s) MPI_File_get_size(o->fh, &disp); MPI_Type_create_subarray(NDIMS, - (int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax }, + (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax }, (int[NDIMS]) { o->comm.kmaxLocal, o->comm.jmaxLocal, o->comm.imaxLocal }, offsets, MPI_ORDER_C, @@ -177,7 +176,7 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec) } int offsets[NDIMS]; - commGetOffsets(&o->comm, offsets, o->grid.kmax, o->grid.jmax, o->grid.imax); + commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax); // set global view in file MPI_Offset disp; @@ -190,7 +189,7 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec) MPI_Type_commit(&vectorType); MPI_Type_create_subarray(NDIMS, - (int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax }, + (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax }, (int[NDIMS]) { kmaxLocal, jmaxLocal, imaxLocal }, offsets, MPI_ORDER_C, diff --git a/BasicSolver/3D-mpi/src/vtkWriter-seq.c b/BasicSolver/3D-mpi/src/vtkWriter-seq.c index 7bd8a14..4bb4fbb 100644 --- a/BasicSolver/3D-mpi/src/vtkWriter-seq.c +++ b/BasicSolver/3D-mpi/src/vtkWriter-seq.c @@ -41,14 +41,14 @@ static void writeHeader(VtkOptions* o) } fprintf(o->fh, "DATASET STRUCTURED_POINTS\n"); - fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid.imax, o->grid.jmax, o->grid.kmax); + fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid->imax, o->grid->jmax, o->grid->kmax); fprintf(o->fh, "ORIGIN %f %f %f\n", - o->grid.dx * 0.5, - o->grid.dy * 0.5, - o->grid.dz * 0.5); - fprintf(o->fh, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz); - fprintf(o->fh, "POINT_DATA %d\n", o->grid.imax * o->grid.jmax * o->grid.kmax); + o->grid->dx * 0.5, + o->grid->dy * 0.5, + o->grid->dz * 0.5); + fprintf(o->fh, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz); + fprintf(o->fh, "POINT_DATA %d\n", o->grid->imax * o->grid->jmax * o->grid->kmax); } void vtkOpen(VtkOptions* o, char* problem) @@ -64,9 +64,9 @@ void vtkOpen(VtkOptions* o, char* problem) static void writeScalar(VtkOptions* o, double* s) { - int imax = o->grid.imax; - int jmax = o->grid.jmax; - int kmax = o->grid.kmax; + int imax = o->grid->imax; + int jmax = o->grid->jmax; + int kmax = o->grid->kmax; for (int k = 0; k < kmax; k++) { for (int j = 0; j < jmax; j++) { @@ -105,9 +105,9 @@ void vtkScalar(VtkOptions* o, char* name, double* s) static void writeVector(VtkOptions* o, VtkVector vec) { - int imax = o->grid.imax; - int jmax = o->grid.jmax; - int kmax = o->grid.kmax; + int imax = o->grid->imax; + int jmax = o->grid->jmax; + int kmax = o->grid->kmax; for (int k = 0; k < kmax; k++) { for (int j = 0; j < jmax; j++) { diff --git a/BasicSolver/3D-mpi/src/vtkWriter.h b/BasicSolver/3D-mpi/src/vtkWriter.h index 4b4c2fe..e93835c 100644 --- a/BasicSolver/3D-mpi/src/vtkWriter.h +++ b/BasicSolver/3D-mpi/src/vtkWriter.h @@ -14,7 +14,7 @@ typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat; typedef struct VtkOptions { - Grid grid; + Grid* grid; #ifdef _VTK_WRITER_MPI MPI_File fh; #else diff --git a/BasicSolver/3D-seq/Makefile b/BasicSolver/3D-seq/Makefile index 64735f6..65e6d8d 100644 --- a/BasicSolver/3D-seq/Makefile +++ b/BasicSolver/3D-seq/Makefile @@ -40,6 +40,18 @@ $(BUILD_DIR)/%.s: %.c .PHONY: clean distclean tags info asm format +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.vtk + @rm -f *.png + +clean: vis_clean + clean: $(info ===> CLEAN) @rm -rf $(BUILD_DIR) diff --git a/BasicSolver/3D-seq/canal.par b/BasicSolver/3D-seq/canal.par index 30e1f41..597c5b5 100644 --- a/BasicSolver/3D-seq/canal.par +++ b/BasicSolver/3D-seq/canal.par @@ -38,10 +38,17 @@ kmax 50 # number of interior cells in z-direction # Time Data: # --------- -te 100.0 # final time +te 60.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- diff --git a/BasicSolver/3D-seq/config.mk b/BasicSolver/3D-seq/config.mk index ccdfa2c..2cf33f9 100644 --- a/BasicSolver/3D-seq/config.mk +++ b/BasicSolver/3D-seq/config.mk @@ -1,12 +1,12 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC ENABLE_OPENMP ?= false -# Supported: sor, mg +# Supported: rb, mg SOLVER ?= mg # Run in debug settings DEBUG ?= false #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 -#OPTIONS += -DVERBOSE +OPTIONS += -DVERBOSE #OPTIONS += -DDEBUG diff --git a/BasicSolver/3D-seq/dcavity.par b/BasicSolver/3D-seq/dcavity.par index 8224887..b668f49 100644 --- a/BasicSolver/3D-seq/dcavity.par +++ b/BasicSolver/3D-seq/dcavity.par @@ -38,10 +38,17 @@ kmax 128 # number of interior cells in z-direction # Time Data: # --------- -te 2.0 # final time +te 10.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- @@ -50,5 +57,4 @@ eps 0.001 # stopping tolerance for pressure iteration rho 0.5 omg 1.7 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma -levels 5 # Multigrid levels #=============================================================================== diff --git a/BasicSolver/3D-seq/residual.plot b/BasicSolver/3D-seq/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/BasicSolver/3D-seq/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/3D-seq/src/discretization.c b/BasicSolver/3D-seq/src/discretization.c index eacac17..8821e7e 100644 --- a/BasicSolver/3D-seq/src/discretization.c +++ b/BasicSolver/3D-seq/src/discretization.c @@ -106,7 +106,7 @@ void initDiscretization(Discretization* d, Parameter* p) d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; #ifdef VERBOSE - printConfig(s); + printConfig(d); #endif /* VERBOSE */ } diff --git a/BasicSolver/3D-seq/src/main.c b/BasicSolver/3D-seq/src/main.c index c289631..f453fdd 100644 --- a/BasicSolver/3D-seq/src/main.c +++ b/BasicSolver/3D-seq/src/main.c @@ -73,6 +73,9 @@ int main(int argc, char** argv) Solver s; initParameter(&p); + FILE* fp; + fp = initResidualWriter(); + if (argc != 2) { printf("Usage: %s \n", argv[0]); exit(EXIT_SUCCESS); @@ -90,6 +93,7 @@ int main(int argc, char** argv) double te = d.te; double t = 0.0; int nt = 0; + double res = 0.0; timeStart = getTimeStamp(); while (t <= te) { @@ -99,13 +103,16 @@ int main(int argc, char** argv) computeFG(&d); computeRHS(&d); if (nt % 100 == 0) normalizePressure(&d); - solve(&s, d.p, d.rhs); + res = solve(&s, d.p, d.rhs); adaptUV(&d); + + writeResidual(fp, t, res); + t += d.dt; nt++; #ifdef VERBOSE - printf("TIME %f , TIMESTEP %f\n", t, solver.dt); + printf("TIME %f , TIMESTEP %f\n", t, d.dt); #else printProgress(t); #endif @@ -123,7 +130,8 @@ int main(int argc, char** argv) ug = allocate(64, bytesize); vg = allocate(64, bytesize); wg = allocate(64, bytesize); - + + fclose(fp); createBulkArrays(&d, pg, ug, vg, wg); VtkOptions opts = { .grid = d.grid }; vtkOpen(&opts, d.problem); diff --git a/BasicSolver/3D-seq/src/parameter.c b/BasicSolver/3D-seq/src/parameter.c index 005b69a..13a2d8a 100644 --- a/BasicSolver/3D-seq/src/parameter.c +++ b/BasicSolver/3D-seq/src/parameter.c @@ -14,19 +14,21 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->zlength = 1.0; - param->imax = 100; - param->jmax = 100; - param->kmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.7; - param->re = 100.0; - param->gamma = 0.9; - param->tau = 0.5; - param->levels = 5; + param->xlength = 1.0; + param->ylength = 1.0; + param->zlength = 1.0; + param->imax = 100; + param->jmax = 100; + param->kmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) diff --git a/BasicSolver/3D-seq/src/parameter.h b/BasicSolver/3D-seq/src/parameter.h index 83917c0..eb2b753 100644 --- a/BasicSolver/3D-seq/src/parameter.h +++ b/BasicSolver/3D-seq/src/parameter.h @@ -18,6 +18,7 @@ typedef struct { char* name; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; double u_init, v_init, w_init, p_init; + int presmooth, postsmooth; } Parameter; void initParameter(Parameter*); diff --git a/BasicSolver/3D-seq/src/progress.c b/BasicSolver/3D-seq/src/progress.c index d5393c4..3523426 100644 --- a/BasicSolver/3D-seq/src/progress.c +++ b/BasicSolver/3D-seq/src/progress.c @@ -49,3 +49,22 @@ void stopProgress() printf("\n"); fflush(stdout); } + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} \ No newline at end of file diff --git a/BasicSolver/3D-seq/src/progress.h b/BasicSolver/3D-seq/src/progress.h index 314e921..240c279 100644 --- a/BasicSolver/3D-seq/src/progress.h +++ b/BasicSolver/3D-seq/src/progress.h @@ -10,5 +10,6 @@ extern void initProgress(double); extern void printProgress(double); extern void stopProgress(void); - +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/BasicSolver/3D-seq/src/solver-mg.c b/BasicSolver/3D-seq/src/solver-mg.c index 9644f59..743292c 100644 --- a/BasicSolver/3D-seq/src/solver-mg.c +++ b/BasicSolver/3D-seq/src/solver-mg.c @@ -141,13 +141,62 @@ static double smooth( for (int j = 1; j < jmax + 1; j++) { for (int i = isw; i < imax + 1; i += 2) { - R(i, j, k) = - RHS(i, j, k) - - ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + - (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * idy2 + - (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * idz2); + P(i, j, k) -= + factor * + (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +} + +static double calculateResidual( + Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax) +{ + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + R(i, + j, + k) = (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * + idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); - P(i, j, k) -= (factor * R(i, j, k)); res += (R(i, j, k) * R(i, j, k)); } isw = 3 - isw; @@ -167,7 +216,7 @@ static double multiGrid( { double res = 0.0; - // coarsest level TODO: Use direct solver? + // coarsest level if (level == COARSEST_LEVEL) { for (int i = 0; i < 5; i++) { smooth(s, p, rhs, level, imax, jmax, kmax); @@ -175,17 +224,18 @@ static double multiGrid( return res; } - // pre-smoothing TODO: Make smoothing steps configurable? - for (int i = 0; i < 5; i++) { + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { smooth(s, p, rhs, level, imax, jmax, kmax); if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); } + res = calculateResidual(s, p, rhs, level, imax, jmax, kmax); + // restrict restrictMG(s, level, imax, jmax, kmax); // MGSolver on residual and error. - // TODO: What if there is a rest? multiGrid(s, s->e[level + 1], s->r[level + 1], @@ -202,8 +252,8 @@ static double multiGrid( if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); // post-smoothing - for (int i = 0; i < 5; i++) { - res = smooth(s, p, rhs, level, imax, jmax, kmax); + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, imax, jmax, kmax); if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); } @@ -212,11 +262,13 @@ static double multiGrid( void initSolver(Solver* s, Discretization* d, Parameter* p) { - s->eps = p->eps; - s->omega = p->omg; - s->itermax = p->itermax; - s->levels = p->levels; - s->grid = &d->grid; + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; int imax = s->grid->imax; int jmax = s->grid->jmax; @@ -240,11 +292,13 @@ void initSolver(Solver* s, Discretization* d, Parameter* p) } } -void solve(Solver* s, double* p, double* rhs) +double solve(Solver* s, double* p, double* rhs) { double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax, s->grid->kmax); #ifdef VERBOSE printf("Residuum: %.6f\n", res); #endif + +return res; } diff --git a/BasicSolver/3D-seq/src/solver-rb.c b/BasicSolver/3D-seq/src/solver-rb.c new file mode 100644 index 0000000..6332cce --- /dev/null +++ b/BasicSolver/3D-seq/src/solver-rb.c @@ -0,0 +1,101 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->itermax = p->itermax; + s->eps = p->eps; + s->omega = p->omg; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double epssq = eps * eps; + int it = 0; + double res = 1.0; + int pass, ksw, jsw, isw; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + double r = + RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2); + + P(i, j, k) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } + + res = res / (double)(imax * jmax * kmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif + +return res; +} diff --git a/BasicSolver/3D-seq/src/solver.h b/BasicSolver/3D-seq/src/solver.h index d7ce513..e5f1b38 100644 --- a/BasicSolver/3D-seq/src/solver.h +++ b/BasicSolver/3D-seq/src/solver.h @@ -18,9 +18,10 @@ typedef struct { int itermax; int levels; double **r, **e; + int presmooth, postsmooth; } Solver; extern void initSolver(Solver*, Discretization*, Parameter*); -extern void solve(Solver*, double*, double*); +extern double solve(Solver*, double*, double*); #endif diff --git a/BasicSolver/3D-seq/src/vtkWriter.c b/BasicSolver/3D-seq/src/vtkWriter.c index 8668b44..2b8f92f 100644 --- a/BasicSolver/3D-seq/src/vtkWriter.c +++ b/BasicSolver/3D-seq/src/vtkWriter.c @@ -70,6 +70,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s) exit(EXIT_FAILURE); } fprintf(o->fh, "SCALARS %s float\n", name); + fprintf(o->fh, "LOOKUP_TABLE default\n"); for (int k = 0; k < kmax; k++) { for (int j = 0; j < jmax; j++) { diff --git a/EnhancedSolver/2D-mpi/Makefile b/EnhancedSolver/2D-mpi/Makefile new file mode 100644 index 0000000..22ed824 --- /dev/null +++ b/EnhancedSolver/2D-mpi/Makefile @@ -0,0 +1,88 @@ +#======================================================================================= +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/comm-$(COMM_TYPE).o +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean vis_clean vis tags info asm format + +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='pressure.dat'" ./surface.plot + @gnuplot -e "filename='velocity.dat'" ./vector.plot + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.png + @rm -f ./vis_files/*.dat + @rm -f ./vis_files/*.gif + +clean: vis_clean + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + @rm -f *.dat + @rm -f *.png + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/EnhancedSolver/2D-mpi/README.md b/EnhancedSolver/2D-mpi/README.md new file mode 100644 index 0000000..b0a80a6 --- /dev/null +++ b/EnhancedSolver/2D-mpi/README.md @@ -0,0 +1,48 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution. + + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. diff --git a/EnhancedSolver/2D-mpi/backstep.par b/EnhancedSolver/2D-mpi/backstep.par new file mode 100644 index 0000000..7063358 --- /dev/null +++ b/EnhancedSolver/2D-mpi/backstep.par @@ -0,0 +1,79 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name backstep # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 36000.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 1.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 7.0 # domain size in x-direction +ylength 1.5 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 40 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 60.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 55 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 500 +startTime 0 #if you want to see particles trapped in recirculation zone, startTime should be set to 0 +injectTimePeriod 1.0 +writeTimePeriod 0.5 + +x1 0.0 +y1 0.5 +x2 0.0 +y2 1.5 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 1 +xCenter 0.0 +yCenter 0.0 +xRectLength 2.0 +yRectLength 1.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/EnhancedSolver/2D-mpi/canal.par b/EnhancedSolver/2D-mpi/canal.par new file mode 100644 index 0000000..7642ace --- /dev/null +++ b/EnhancedSolver/2D-mpi/canal.par @@ -0,0 +1,78 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 100.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 40 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 60.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.00001 # stopping tolerance for pressure iteration +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 60 +startTime 10.0 +injectTimePeriod 4.0 +writeTimePeriod 1.0 + +x1 1.0 +y1 0.0 +x2 1.0 +y2 4.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 10.0 +yCenter 2 +xRectLength 6.0 +yRectLength 1.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/EnhancedSolver/2D-mpi/config.mk b/EnhancedSolver/2D-mpi/config.mk new file mode 100644 index 0000000..0b7938e --- /dev/null +++ b/EnhancedSolver/2D-mpi/config.mk @@ -0,0 +1,17 @@ +# Supported: GCC, CLANG, ICC +TAG ?= ICC +# Supported: true, false +ENABLE_MPI ?= true +ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg +# Run in debug settings ?= mg +COMM_TYPE ?= v3 + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +# OPTIONS += -DTEST +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/EnhancedSolver/2D-mpi/dcavity.par b/EnhancedSolver/2D-mpi/dcavity.par new file mode 100644 index 0000000..bc91616 --- /dev/null +++ b/EnhancedSolver/2D-mpi/dcavity.par @@ -0,0 +1,79 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 1 # 2 = free-slip 4 = periodic +bcRight 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 10.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +imax 128 # number of interior cells in x-direction +jmax 128 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 10.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.5 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 2.0 +injectTimePeriod 0.5 +writeTimePeriod 0.2 + +x1 0.1 +y1 0.9 +x2 0.9 +y2 0.9 + + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 0.5 +yCenter 0.5 +xRectLength 0.5 +yRectLength 0.5 +circleRadius 0.5 +#=============================================================================== diff --git a/EnhancedSolver/2D-mpi/include_CLANG.mk b/EnhancedSolver/2D-mpi/include_CLANG.mk new file mode 100644 index 0000000..3641cad --- /dev/null +++ b/EnhancedSolver/2D-mpi/include_CLANG.mk @@ -0,0 +1,21 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpicc +DEFINES = -D_MPI +else +CC = cc +endif + +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +CFLAGS = -Ofast -std=c17 +LFLAGS = $(OPENMP) -lm +DEFINES += -D_GNU_SOURCE# -DDEBUG +INCLUDES = -I/opt/homebrew/include diff --git a/EnhancedSolver/2D-mpi/include_GCC.mk b/EnhancedSolver/2D-mpi/include_GCC.mk new file mode 100644 index 0000000..cf1e2aa --- /dev/null +++ b/EnhancedSolver/2D-mpi/include_GCC.mk @@ -0,0 +1,20 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpicc +DEFINES = -D_MPI +else +CC = gcc +endif + +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES += -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/EnhancedSolver/2D-mpi/include_ICC.mk b/EnhancedSolver/2D-mpi/include_ICC.mk new file mode 100644 index 0000000..6bedf55 --- /dev/null +++ b/EnhancedSolver/2D-mpi/include_ICC.mk @@ -0,0 +1,20 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpiicc +DEFINES = -D_MPI +else +CC = icc +endif + +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES += -D_GNU_SOURCE# -DDEBUG +INCLUDES = +LIBS = diff --git a/EnhancedSolver/2D-mpi/karman.par b/EnhancedSolver/2D-mpi/karman.par new file mode 100644 index 0000000..a22eb34 --- /dev/null +++ b/EnhancedSolver/2D-mpi/karman.par @@ -0,0 +1,79 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name karman # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 5050.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 8.0 # domain size in y-direction +imax 400 # number of interior cells in x-direction +jmax 200 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 150.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 200 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.75 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 15 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 50 +injectTimePeriod 1.0 +writeTimePeriod 0.5 + +x1 0.0 +y1 3.8 +x2 0.0 +y2 4.1 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 2 +xCenter 5.0 +yCenter 4.0 +xRectLength 2.0 +yRectLength 1.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/EnhancedSolver/2D-mpi/residual.plot b/EnhancedSolver/2D-mpi/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/EnhancedSolver/2D-mpi/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/affinity.c b/EnhancedSolver/2D-mpi/src/affinity.c new file mode 100644 index 0000000..ef8355a --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/affinity.c @@ -0,0 +1,61 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifdef __linux__ +#ifdef _OPENMP +#include +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_THREADS 128 +#define gettid() syscall(SYS_gettid) + +static int getProcessorID(cpu_set_t* cpu_set) +{ + int processorId; + + for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { + if (CPU_ISSET(processorId, cpu_set)) { + break; + } + } + return processorId; +} + +int affinity_getProcessorId() +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); + + return getProcessorID(&cpu_set); +} + +void affinity_pinThread(int processorId) +{ + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); +} + +void affinity_pinProcess(int processorId) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} +#endif /*_OPENMP*/ +#endif /*__linux__*/ diff --git a/EnhancedSolver/2D-mpi/src/affinity.h b/EnhancedSolver/2D-mpi/src/affinity.h new file mode 100644 index 0000000..84bf733 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/affinity.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef AFFINITY_H +#define AFFINITY_H + +extern int affinity_getProcessorId(); +extern void affinity_pinProcess(int); +extern void affinity_pinThread(int); + +#endif /*AFFINITY_H*/ diff --git a/EnhancedSolver/2D-mpi/src/allocate.c b/EnhancedSolver/2D-mpi/src/allocate.c new file mode 100644 index 0000000..cf2efd6 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/allocate.c @@ -0,0 +1,38 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/EnhancedSolver/2D-mpi/src/allocate.h b/EnhancedSolver/2D-mpi/src/allocate.h new file mode 100644 index 0000000..77f4ba0 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(size_t alignment, size_t bytesize); + +#endif diff --git a/EnhancedSolver/2D-mpi/src/comm-v1.c b/EnhancedSolver/2D-mpi/src/comm-v1.c new file mode 100644 index 0000000..9be7e88 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/comm-v1.c @@ -0,0 +1,233 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include + +#include "comm.h" + +#ifdef _MPI +// subroutines local to this module +static int sum(int* sizes, int position) +{ + int sum = 0; + + for (int i = 0; i < position; i += position) { + sum += sizes[i]; + } + + return sum; +} + +static void gatherArray( + Comm* c, int cnt, int* rcvCounts, int* displs, double* src, double* dst) +{ + double* sendbuffer = src + (c->imaxLocal + 2); + + if (c->rank == 0) { + sendbuffer = src; + } + + MPI_Gatherv(sendbuffer, + cnt, + MPI_DOUBLE, + dst, + rcvCounts, + displs, + MPI_DOUBLE, + 0, + MPI_COMM_WORLD); +} +#endif // defined _MPI + +// exported subroutines +int commIsBoundary(Comm* c, int direction) +{ +#ifdef _MPI + switch (direction) { + case L: + return 1; + break; + case R: + return 1; + break; + case B: + return c->rank == 0; + break; + case T: + return c->rank == (c->size - 1); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ +#ifdef _MPI + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* exchange ghost cells with top neighbor */ + if (c->rank + 1 < c->size) { + int top = c->rank + 1; + double* src = grid + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + double* dst = grid + (c->jmaxLocal + 1) * (c->imaxLocal + 2) + 1; + + MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, top, 1, MPI_COMM_WORLD, &requests[0]); + MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, top, 2, MPI_COMM_WORLD, &requests[1]); + } + + /* exchange ghost cells with bottom neighbor */ + if (c->rank > 0) { + int bottom = c->rank - 1; + double* src = grid + (c->imaxLocal + 2) + 1; + double* dst = grid + 1; + + MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]); + MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]); + } + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commShift(Comm* c, double* f, double* g) +{ +#ifdef _MPI + MPI_Request requests[2] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + if (c->rank > 0) { + int bottom = c->rank - 1; + MPI_Irecv(g + 1, + c->imaxLocal, + MPI_DOUBLE, + bottom, + 0, + MPI_COMM_WORLD, + &requests[0]); + } + + if (c->rank + 1 < c->size) { + int top = c->rank + 1; + double* buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + /* send ghost cells to top neighbor */ + MPI_Isend(buf, c->imaxLocal, MPI_DOUBLE, top, 0, MPI_COMM_WORLD, &requests[1]); + } + + MPI_Waitall(2, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int jmax, + int imax) +{ +#ifdef _MPI + int *rcvCounts, *displs; + int cnt = c->jmaxLocal * (imax + 2); + + if (c->rank == 0) { + rcvCounts = (int*)malloc(c->size * sizeof(int)); + displs = (int*)malloc(c->size * sizeof(int)); + } + + if (c->rank == 0 && c->size == 1) { + cnt = (c->jmaxLocal + 2) * (imax + 2); + } else if (c->rank == 0 || c->rank == (c->size - 1)) { + cnt = (c->jmaxLocal + 1) * (imax + 2); + } + + MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD); + + if (c->rank == 0) { + displs[0] = 0; + int cursor = rcvCounts[0]; + + for (int i = 1; i < c->size; i++) { + displs[i] = cursor; + cursor += rcvCounts[i]; + } + } + + gatherArray(c, cnt, rcvCounts, displs, p, pg); + gatherArray(c, cnt, rcvCounts, displs, u, ug); + gatherArray(c, cnt, rcvCounts, displs, v, vg); +#endif +} + +void commPartition(Comm* c, int jmax, int imax) +{ +#ifdef _MPI + c->imaxLocal = imax; + c->jmaxLocal = sizeOfRank(c->coords[JDIM], c->size, jmax); + + c->neighbours[BOTTOM] = c->rank == 0 ? -1 : c->rank - 1; + c->neighbours[TOP] = c->rank == (c->size - 1) ? -1 : c->rank + 1; + c->neighbours[LEFT] = -1; + c->neighbours[RIGHT] = -1; + + c->coords[IDIM] = 0; + c->coords[JDIM] = c->rank; + + c->dims[IDIM] = 1; + c->dims[JDIM] = c->size; +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; +#endif +} + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ + +#if defined _MPI + newcomm->comm = MPI_COMM_NULL; + int result = MPI_Comm_dup(MPI_COMM_WORLD, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + + newcomm->neighbours[BOTTOM] = newcomm->rank == 0 ? -1 : newcomm->rank - 1; + newcomm->neighbours[TOP] = newcomm->rank == (newcomm->size - 1) ? -1 : newcomm->rank + 1; + newcomm->neighbours[LEFT] = -1; + newcomm->neighbours[RIGHT] = -1; + + newcomm->coords[IDIM] = 0; + newcomm->coords[JDIM] = newcomm->rank; + + newcomm->dims[IDIM] = 1; + newcomm->dims[JDIM] = newcomm->size; + + +#endif + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/comm-v2.c b/EnhancedSolver/2D-mpi/src/comm-v2.c new file mode 100644 index 0000000..ef1777c --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/comm-v2.c @@ -0,0 +1,342 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "comm.h" +#include +#include +#include + +#ifdef _MPI +// subroutines local to this module +static int sum(int* sizes, int init, int offset, int coord) +{ + int sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax) +{ + MPI_Request* requests; + int numRequests = 1; + + if (c->rank == 0) { + numRequests = c->size + 1; + } else { + numRequests = 1; + } + + requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); + + /* all ranks send their bulk array, including the external boundary layer */ + MPI_Datatype bulkType; + int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; + int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; + int starts[NDIMS] = { 1, 1 }; + + if (commIsBoundary(c, L)) { + newSizes[CIDIM] += 1; + starts[CIDIM] = 0; + } + if (commIsBoundary(c, R)) { + newSizes[CIDIM] += 1; + } + if (commIsBoundary(c, B)) { + newSizes[CJDIM] += 1; + starts[CJDIM] = 0; + } + if (commIsBoundary(c, T)) { + newSizes[CJDIM] += 1; + } + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]); + + int newSizesI[c->size]; + int newSizesJ[c->size]; + MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD); + + /* rank 0 assembles the subdomains */ + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + MPI_Datatype domainType; + int oldSizes[NDIMS] = { jmax + 2, imax + 2 }; + int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] }; + int coords[NDIMS]; + MPI_Cart_coords(c->comm, i, NDIMS, coords); + int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]), + sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) }; + printf( + "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d " + "Starts(i,j): %d %d\n", + i, + coords[IDIM], + coords[JDIM], + oldSizes[CIDIM], + oldSizes[CJDIM], + newSizes[CIDIM], + newSizes[CJDIM], + starts[CIDIM], + starts[CJDIM]); + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} +#endif // defined _MPI + +// exported subroutines +int commIsBoundary(Comm* c, int direction) +{ +#ifdef _MPI + switch (direction) { + case L: + return c->coords[IDIM] == 0; + break; + case R: + return c->coords[IDIM] == (c->dims[IDIM] - 1); + break; + case B: + return c->coords[JDIM] == 0; + break; + case T: + return c->coords[JDIM] == (c->dims[JDIM] - 1); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ +#ifdef _MPI + MPI_Request requests[8]; + for (int i = 0; i < 8; i++) + requests[i] = MPI_REQUEST_NULL; + + for (int i = 0; i < NDIRS; i++) { + double* sbuf = grid + c->sdispls[i]; + double* rbuf = grid + c->rdispls[i]; + + int tag = 0; + if (c->neighbours[i] != MPI_PROC_NULL) { + // printf("DEBUG: Rank %d - SendRecv with %d\n", c->rank, c->neighbours[i]); + tag = c->neighbours[i]; + } + MPI_Irecv(rbuf, + 1, + c->bufferTypes[i], + c->neighbours[i], + tag, + c->comm, + &requests[i * 2]); + MPI_Isend(sbuf, + 1, + c->bufferTypes[i], + c->neighbours[i], + c->rank, + c->comm, + &requests[i * 2 + 1]); + } + + MPI_Waitall(8, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commShift(Comm* c, double* f, double* g) +{ +#ifdef _MPI + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + double* buf = g + 1; + MPI_Irecv(buf, + 1, + c->bufferTypes[B], + c->neighbours[B], + 0, + c->comm, + &requests[0]); + + /* send ghost cells to top neighbor */ + buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + MPI_Isend(buf, 1, c->bufferTypes[T], c->neighbours[T], 0, c->comm, &requests[1]); + + /* shift F */ + /* receive ghost cells from left neighbor */ + buf = f + (c->imaxLocal + 2); + MPI_Irecv(buf, + 1, + c->bufferTypes[L], + c->neighbours[L], + 1, + c->comm, + &requests[2]); + + /* send ghost cells to right neighbor */ + buf = f + (c->imaxLocal + 2) + (c->imaxLocal); + MPI_Isend(buf, + 1, + c->bufferTypes[R], + c->neighbours[R], + 1, + c->comm, + &requests[3]); + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int imax, + int jmax) +{ +#ifdef _MPI + /* collect P */ + assembleResult(c, p, pg, imax, jmax); + + /* collect U */ + assembleResult(c, u, ug, imax, jmax); + + /* collect V */ + assembleResult(c, v, vg, imax, jmax); +#endif +} + +void commPartition(Comm* c, int jmax, int imax) +{ +#ifdef _MPI + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + MPI_Dims_create(c->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm); + MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[L], &c->neighbours[R]); + MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[B], &c->neighbours[T]); + MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords); + + int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax); + int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax); + + c->imaxLocal = imaxLocal; + c->jmaxLocal = jmaxLocal; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + c->bufferTypes[L] = iBufferType; + c->bufferTypes[R] = iBufferType; + c->bufferTypes[B] = jBufferType; + c->bufferTypes[T] = jBufferType; + + c->sdispls[L] = (imaxLocal + 2) + 1; + c->sdispls[R] = (imaxLocal + 2) + imaxLocal; + c->sdispls[B] = (imaxLocal + 2) + 1; + c->sdispls[T] = jmaxLocal * (imaxLocal + 2) + 1; + + c->rdispls[L] = (imaxLocal + 2); + c->rdispls[R] = (imaxLocal + 2) + (imaxLocal + 1); + c->rdispls[B] = 1; + c->rdispls[T] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; +#endif +} + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ +#if defined _MPI + newcomm->comm = MPI_COMM_NULL; + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + memcpy(&newcomm->neighbours, &oldcomm->neighbours, sizeof(oldcomm->neighbours)); + memcpy(&newcomm->coords, &oldcomm->coords, sizeof(oldcomm->coords)); + memcpy(&newcomm->dims, &oldcomm->dims, sizeof(oldcomm->dims)); + + newcomm->imaxLocal = imaxLocal/2; + newcomm->jmaxLocal = jmaxLocal/2; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + newcomm->bufferTypes[L] = iBufferType; + newcomm->bufferTypes[R] = iBufferType; + newcomm->bufferTypes[B] = jBufferType; + newcomm->bufferTypes[T] = jBufferType; + + newcomm->sdispls[L] = (imaxLocal + 2) + 1; + newcomm->sdispls[R] = (imaxLocal + 2) + imaxLocal; + newcomm->sdispls[B] = (imaxLocal + 2) + 1; + newcomm->sdispls[T] = jmaxLocal * (imaxLocal + 2) + 1; + + newcomm->rdispls[L] = (imaxLocal + 2); + newcomm->rdispls[R] = (imaxLocal + 2) + (imaxLocal + 1); + newcomm->rdispls[B] = 1; + newcomm->rdispls[T] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ + #ifdef _MPI + MPI_Comm_free(&comm->comm); + #endif +} \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/comm-v3.c b/EnhancedSolver/2D-mpi/src/comm-v3.c new file mode 100644 index 0000000..18cd186 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/comm-v3.c @@ -0,0 +1,320 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "comm.h" + +#ifdef _MPI +// subroutines local to this module +static int sum(int* sizes, int init, int offset, int coord) +{ + int sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax) +{ + MPI_Request* requests; + int numRequests = 1; + + if (c->rank == 0) { + numRequests = c->size + 1; + } else { + numRequests = 1; + } + + requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); + + /* all ranks send their bulk array, including the external boundary layer */ + MPI_Datatype bulkType; + int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; + int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; + int starts[NDIMS] = { 1, 1 }; + + if (commIsBoundary(c, L)) { + newSizes[CIDIM] += 1; + starts[CIDIM] = 0; + } + if (commIsBoundary(c, R)) { + newSizes[CIDIM] += 1; + } + if (commIsBoundary(c, B)) { + newSizes[CJDIM] += 1; + starts[CJDIM] = 0; + } + if (commIsBoundary(c, T)) { + newSizes[CJDIM] += 1; + } + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]); + + int newSizesI[c->size]; + int newSizesJ[c->size]; + MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD); + + /* rank 0 assembles the subdomains */ + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + MPI_Datatype domainType; + int oldSizes[NDIMS] = { jmax + 2, imax + 2 }; + int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] }; + int coords[NDIMS]; + MPI_Cart_coords(c->comm, i, NDIMS, coords); + int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]), + sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) }; + printf( + "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d " + "Starts(i,j): %d %d\n", + i, + coords[IDIM], + coords[JDIM], + oldSizes[CIDIM], + oldSizes[CJDIM], + newSizes[CIDIM], + newSizes[CJDIM], + starts[CIDIM], + starts[CJDIM]); + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} +#endif // defined _MPI + +// exported subroutines +int commIsBoundary(Comm* c, int direction) +{ +#ifdef _MPI + switch (direction) { + case L: + return c->coords[IDIM] == 0; + break; + case R: + return c->coords[IDIM] == (c->dims[IDIM] - 1); + break; + case B: + return c->coords[JDIM] == 0; + break; + case T: + return c->coords[JDIM] == (c->dims[JDIM] - 1); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ +#ifdef _MPI + int counts[NDIRS] = { 1, 1, 1, 1 }; + MPI_Neighbor_alltoallw(grid, + counts, + c->sdispls, + c->bufferTypes, + grid, + counts, + c->rdispls, + c->bufferTypes, + c->comm); +#endif +} + +void commShift(Comm* c, double* f, double* g) +{ +#ifdef _MPI + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + double* buf = g + 1; + MPI_Irecv(buf, + 1, + c->bufferTypes[B], + c->neighbours[B], + 0, + c->comm, + &requests[0]); + + /* send ghost cells to top neighbor */ + buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + MPI_Isend(buf, 1, c->bufferTypes[T], c->neighbours[T], 0, c->comm, &requests[1]); + + /* shift F */ + /* receive ghost cells from left neighbor */ + buf = f + (c->imaxLocal + 2); + MPI_Irecv(buf, + 1, + c->bufferTypes[L], + c->neighbours[L], + 1, + c->comm, + &requests[2]); + + /* send ghost cells to right neighbor */ + buf = f + (c->imaxLocal + 2) + (c->imaxLocal); + MPI_Isend(buf, + 1, + c->bufferTypes[R], + c->neighbours[R], + 1, + c->comm, + &requests[3]); + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int imax, + int jmax) +{ +#ifdef _MPI + /* collect P */ + assembleResult(c, p, pg, imax, jmax); + + /* collect U */ + assembleResult(c, u, ug, imax, jmax); + + /* collect V */ + assembleResult(c, v, vg, imax, jmax); +#endif +} + +void commPartition(Comm* c, int jmax, int imax) +{ +#ifdef _MPI + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + MPI_Dims_create(c->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm); + MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[L], &c->neighbours[R]); + MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[B], &c->neighbours[T]); + MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords); + + int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax); + int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax); + + c->imaxLocal = imaxLocal; + c->jmaxLocal = jmaxLocal; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + c->bufferTypes[L] = iBufferType; + c->bufferTypes[R] = iBufferType; + c->bufferTypes[B] = jBufferType; + c->bufferTypes[T] = jBufferType; + + size_t dblsize = sizeof(double); + c->sdispls[L] = ((imaxLocal + 2) + 1) * dblsize; + c->sdispls[R] = ((imaxLocal + 2) + imaxLocal) * dblsize; + c->sdispls[B] = ((imaxLocal + 2) + 1) * dblsize; + c->sdispls[T] = (jmaxLocal * (imaxLocal + 2) + 1) * dblsize; + + c->rdispls[L] = (imaxLocal + 2) * dblsize; + c->rdispls[R] = ((imaxLocal + 2) + (imaxLocal + 1)) * dblsize; + c->rdispls[B] = 1 * dblsize; + c->rdispls[T] = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * dblsize; +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; +#endif +} + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ +#if defined _MPI + + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + newcomm->bufferTypes[L] = iBufferType; + newcomm->bufferTypes[R] = iBufferType; + newcomm->bufferTypes[B] = jBufferType; + newcomm->bufferTypes[T] = jBufferType; + + newcomm->sdispls[L] = (imaxLocal + 2) + 1; + newcomm->sdispls[R] = (imaxLocal + 2) + imaxLocal; + newcomm->sdispls[B] = (imaxLocal + 2) + 1; + newcomm->sdispls[T] = jmaxLocal * (imaxLocal + 2) + 1; + + newcomm->rdispls[L] = (imaxLocal + 2); + newcomm->rdispls[R] = (imaxLocal + 2) + (imaxLocal + 1); + newcomm->rdispls[B] = 1; + newcomm->rdispls[T] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ + #ifdef _MPI + MPI_Comm_free(&comm->comm); + #endif +} \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/comm.c b/EnhancedSolver/2D-mpi/src/comm.c new file mode 100644 index 0000000..7904a9b --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/comm.c @@ -0,0 +1,129 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "comm.h" + +// subroutines local to this module +int sizeOfRank(int rank, int size, int N) +{ + return N / size + ((N % size > rank) ? 1 : 0); +} + +void commReduction(double* v, int op) +{ +#ifdef _MPI + if (op == MAX) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + } else if (op == SUM) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + } +#endif +} + +void commPrintConfig(Comm* c) +{ +#ifdef _MPI + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + if (commIsMaster(c)) { + printf("Communication setup:\n"); + } + + for (int i = 0; i < c->size; i++) { + if (i == c->rank) { + printf("\tRank %d of %d\n", c->rank, c->size); + printf("\tNeighbours (bottom, top, left, right): %d %d, %d, %d\n", + c->neighbours[B], + c->neighbours[T], + c->neighbours[L], + c->neighbours[R]); + printf("\tIs boundary:\n"); + printf("\t\tLEFT: %d\n", commIsBoundary(c, L)); + printf("\t\tRIGHT: %d\n", commIsBoundary(c, R)); + printf("\t\tBOTTOM: %d\n", commIsBoundary(c, B)); + printf("\t\tTOP: %d\n", commIsBoundary(c, T)); + printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]); + printf("\tDims (i,j) %d %d\n", c->dims[IDIM], c->dims[JDIM]); + printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal); + fflush(stdout); + } + MPI_Barrier(MPI_COMM_WORLD); + } +#endif +} + +void commInit(Comm* c, int argc, char** argv) +{ +#ifdef _MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank)); + MPI_Comm_size(MPI_COMM_WORLD, &(c->size)); +#else + c->rank = 0; + c->size = 1; +#endif +} + +void commTestInit(Comm* c, double* p, double* f, double* g) +{ + int imax = c->imaxLocal; + int jmax = c->jmaxLocal; + int rank = c->rank; + + for (int j = 0; j < jmax + 2; j++) { + for (int i = 0; i < imax + 2; i++) { + p[j * (imax + 2) + i] = rank; + f[j * (imax + 2) + i] = rank; + g[j * (imax + 2) + i] = rank; + } + } +} + +static void testWriteFile(char* filename, double* grid, int imax, int jmax) +{ + FILE* fp = fopen(filename, "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 0; j < jmax + 2; j++) { + for (int i = 0; i < imax + 2; i++) { + fprintf(fp, "%.2f ", grid[j * (imax + 2) + i]); + } + fprintf(fp, "\n"); + } + + fclose(fp); +} + +void commTestWrite(Comm* c, double* p, double* f, double* g) +{ + int imax = c->imaxLocal; + int jmax = c->jmaxLocal; + int rank = c->rank; + + char filename[30]; + snprintf(filename, 30, "ptest-%d.dat", rank); + testWriteFile(filename, p, imax, jmax); + + snprintf(filename, 30, "ftest-%d.dat", rank); + testWriteFile(filename, f, imax, jmax); + + snprintf(filename, 30, "gtest-%d.dat", rank); + testWriteFile(filename, g, imax, jmax); +} + +void commFinalize(Comm* c) +{ +#ifdef _MPI + MPI_Finalize(); +#endif +} diff --git a/EnhancedSolver/2D-mpi/src/comm.h b/EnhancedSolver/2D-mpi/src/comm.h new file mode 100644 index 0000000..04d2a8a --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/comm.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __COMM_H_ +#define __COMM_H_ +#if defined(_MPI) +#include +#endif + +enum direction { L = 0, R, B, T, NDIRS }; // L = Left, R = Right, B = Bottom, T =Top +enum dimension { IDIM = 0, JDIM, NDIMS }; +enum cdimension { CJDIM = 0, CIDIM }; +enum layer { HALO = 0, BULK }; +enum op { MAX = 0, SUM }; + +typedef struct { + int rank; + int size; +#if defined(_MPI) + MPI_Comm comm; + MPI_Datatype bufferTypes[NDIRS]; + MPI_Aint sdispls[NDIRS]; + MPI_Aint rdispls[NDIRS]; +#endif + int neighbours[NDIRS]; + int coords[NDIMS], dims[NDIMS]; + int imaxLocal, jmaxLocal; +} Comm; + +extern int sizeOfRank(int rank, int size, int N); +extern void commInit(Comm* c, int argc, char** argv); +extern void commTestInit(Comm* c, double* p, double* f, double* g); +extern void commTestWrite(Comm* c, double* p, double* f, double* g); +extern void commFinalize(Comm* c); +extern void commPartition(Comm* c, int jmax, int imax); +extern void commPrintConfig(Comm*); +extern void commExchange(Comm*, double*); +extern void commShift(Comm* c, double* f, double* g); +extern void commReduction(double* v, int op); +extern int commIsBoundary(Comm* c, int direction); +extern void commUpdateDatatypes(Comm*, Comm*, int, int); +extern void commFreeCommunicator(Comm*); +extern void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int jmax, + int imax); + +static inline int commIsMaster(Comm* c) { return c->rank == 0; } +#endif // __COMM_H_ diff --git a/EnhancedSolver/2D-mpi/src/discretization.c b/EnhancedSolver/2D-mpi/src/discretization.c new file mode 100644 index 0000000..94c6acb --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/discretization.c @@ -0,0 +1,724 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "discretization.h" +#include "parameter.h" +#include "util.h" + +static double distance(double i, double j, double iCenter, double jCenter) +{ + return sqrt(pow(iCenter - i, 2) + pow(jCenter - j, 2) * 1.0); +} + +double sumOffset(double* sizes, int init, int offset, int coord) +{ + double sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +void print(Discretization* d, double* grid) +{ + int imaxLocal = d->comm.imaxLocal; + + for (int i = 0; i < d->comm.size; i++) { + if (i == d->comm.rank) { + sleep(1 * d->comm.rank); + printf("### RANK %d LVL " + "###################################################### #\n ", + d->comm.rank); + for (int j = 0; j < d->comm.jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < d->comm.imaxLocal + 2; i++) { + printf("%2.2f ", grid[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + } +} + +static void printConfig(Discretization* d) +{ + if (commIsMaster(&d->comm)) { + printf("Parameters for #%s#\n", d->problem); + printf("BC Left:%d Right:%d Bottom:%d Top:%d\n", + d->bcLeft, + d->bcRight, + d->bcBottom, + d->bcTop); + printf("\tReynolds number: %.2f\n", d->re); + printf("\tGx Gy: %.2f %.2f\n", d->gx, d->gy); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", + d->grid.xlength, + d->grid.ylength); + printf("\tCells (x, y): %d, %d\n", d->grid.imax, d->grid.jmax); + printf("\tCell size (dx, dy): %f, %f\n", d->grid.dx, d->grid.dy); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te); + printf("\tdt bound: %.6f\n", d->dtBound); + printf("\tTau factor: %.2f\n", d->tau); + printf("Iterative s parameters:\n"); + printf("\tgamma factor: %f\n", d->gamma); + } + commPrintConfig(&d->comm); +} + +void initDiscretiztion(Discretization* d, Parameter* params) +{ + d->problem = params->name; + d->bcLeft = params->bcLeft; + d->bcRight = params->bcRight; + d->bcBottom = params->bcBottom; + d->bcTop = params->bcTop; + d->grid.imax = params->imax; + d->grid.jmax = params->jmax; + d->grid.xlength = params->xlength; + d->grid.ylength = params->ylength; + d->grid.dx = params->xlength / params->imax; + d->grid.dy = params->ylength / params->jmax; + d->re = params->re; + d->gx = params->gx; + d->gy = params->gy; + d->dt = params->dt; + d->te = params->te; + d->tau = params->tau; + d->gamma = params->gamma; + + /* allocate arrays */ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + size_t size = (imaxLocal + 2) * (jmaxLocal + 2); + + d->u = allocate(64, size * sizeof(double)); + d->v = allocate(64, size * sizeof(double)); + d->p = allocate(64, size * sizeof(double)); + d->rhs = allocate(64, size * sizeof(double)); + d->f = allocate(64, size * sizeof(double)); + d->g = allocate(64, size * sizeof(double)); + d->grid.s = allocate(64, size * sizeof(double)); + + for (int i = 0; i < size; i++) { + d->u[i] = params->u_init; + d->v[i] = params->v_init; + d->p[i] = params->p_init; + d->rhs[i] = 0.0; + d->f[i] = 0.0; + d->g[i] = 0.0; + d->grid.s[i] = FLUID; + } + + double dx = d->grid.dx; + double dy = d->grid.dy; + + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy); + d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; + + d->xLocal = d->comm.imaxLocal * d->grid.dx; + d->yLocal = d->comm.jmaxLocal * d->grid.dy; + + double xLocal[d->comm.size]; + double yLocal[d->comm.size]; + +#ifdef _MPI + MPI_Allgather(&d->xLocal, 1, MPI_DOUBLE, xLocal, 1, MPI_DOUBLE, d->comm.comm); + MPI_Allgather(&d->yLocal, 1, MPI_DOUBLE, yLocal, 1, MPI_DOUBLE, d->comm.comm); + + d->xOffset = sumOffset(xLocal, + d->comm.rank, + d->comm.dims[JDIM], + d->comm.coords[IDIM]); + d->yOffset = sumOffset(yLocal, d->comm.rank, 1, d->comm.coords[JDIM]); + d->xOffsetEnd = d->xOffset + d->xLocal; + d->yOffsetEnd = d->yOffset + d->yLocal; +#else + + d->xOffset = 0; + d->yOffset = 0; + d->xOffsetEnd = d->xOffset + d->xLocal; + d->yOffsetEnd = d->yOffset + d->yLocal; +#endif + + printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : " + "%.2f\n", + d->comm.rank, + d->xOffset, + d->yOffset, + d->xOffsetEnd, + d->yOffsetEnd); + + double* s = d->grid.s; + int iOffset = 0, jOffset = 0; + + double xCenter = 0, yCenter = 0, radius = 0; + double x1 = 0, x2 = 0, y1 = 0, y2 = 0; + + switch (params->shape) { + case NOSHAPE: + break; + case RECT: + x1 = params->xCenter - params->xRectLength / 2; + x2 = params->xCenter + params->xRectLength / 2; + y1 = params->yCenter - params->yRectLength / 2; + y2 = params->yCenter + params->yRectLength / 2; + + iOffset = d->xOffset / dx; + jOffset = d->yOffset / dy; + + for (int j = 1; j < jmaxLocal + 1; ++j) { + for (int i = 1; i < imaxLocal + 1; ++i) { + if ((x1 <= ((i + iOffset) * dx)) && (((i + iOffset) * dx) <= x2) && + (y1 <= ((j + jOffset) * dy)) && (((j + jOffset) * dy) <= y2)) { + S(i, j) = OBSTACLE; + } + } + } + + break; + case CIRCLE: + xCenter = params->xCenter; + yCenter = params->yCenter; + radius = params->circleRadius; + + iOffset = d->xOffset / dx; + jOffset = d->yOffset / dy; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if (distance(((i + iOffset) * dx), + ((j + jOffset) * dy), + xCenter, + yCenter) <= radius) { + S(i, j) = OBSTACLE; + } + } + } + + break; + default: + break; + } + +#ifdef _MPI + commExchange(&d->comm, s); +#endif + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if (S(i, j - 1) == FLUID && S(i, j + 1) == OBSTACLE && S(i, j) == OBSTACLE) + S(i, j) = BOTTOM; // BOTTOM + if (S(i - 1, j) == FLUID && S(i + 1, j) == OBSTACLE && S(i, j) == OBSTACLE) + S(i, j) = LEFT; // LEFT + if (S(i + 1, j) == FLUID && S(i - 1, j) == OBSTACLE && S(i, j) == OBSTACLE) + S(i, j) = RIGHT; // RIGHT + if (S(i, j + 1) == FLUID && S(i, j - 1) == OBSTACLE && S(i, j) == OBSTACLE) + S(i, j) = TOP; // TOP + if (S(i - 1, j - 1) == FLUID && S(i, j - 1) == FLUID && + S(i - 1, j) == FLUID && S(i + 1, j + 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == LEFT || S(i, j) == BOTTOM)) + S(i, j) = BOTTOMLEFT; // BOTTOMLEFT + if (S(i + 1, j - 1) == FLUID && S(i, j - 1) == FLUID && + S(i + 1, j) == FLUID && S(i - 1, j + 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == RIGHT || S(i, j) == BOTTOM)) + S(i, j) = BOTTOMRIGHT; // BOTTOMRIGHT + if (S(i - 1, j + 1) == FLUID && S(i - 1, j) == FLUID && + S(i, j + 1) == FLUID && S(i + 1, j - 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == LEFT || S(i, j) == TOP)) + S(i, j) = TOPLEFT; // TOPLEFT + if (S(i + 1, j + 1) == FLUID && S(i + 1, j) == FLUID && + S(i, j + 1) == FLUID && S(i - 1, j - 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == RIGHT || S(i, j) == TOP)) + S(i, j) = TOPRIGHT; // TOPRIGHT + } + } + +#ifdef VERBOSE + printConfig(d); +#endif +} + +void computeRHS(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + double idx = 1.0 / d->grid.dx; + double idy = 1.0 / d->grid.dy; + double idt = 1.0 / d->dt; + double* rhs = d->rhs; + double* f = d->f; + double* g = d->g; + + commShift(&d->comm, f, g); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) * + idt; + } + } +} + +static double maxElement(Discretization* d, double* m) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int size = (imaxLocal + 2) * (jmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + commReduction(&maxval, MAX); + return maxval; +} + +void computeTimestep(Discretization* d) +{ + double dt = d->dtBound; + double dx = d->grid.dx; + double dy = d->grid.dy; + double umax = maxElement(d, d->u); + double vmax = maxElement(d, d->v); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + + d->dt = dt * d->tau; +} + +void setBoundaryConditions(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + double* u = d->u; + double* v = d->v; + + if (commIsBoundary(&d->comm, T)) { + switch (d->bcTop) { + case NOSLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = -U(i, jmaxLocal); + } + break; + case SLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + } + break; + case OUTFLOW: + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + V(i, jmaxLocal) = V(i, jmaxLocal - 1); + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, B)) { + switch (d->bcBottom) { + case NOSLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = -U(i, 1); + } + break; + case SLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = U(i, 1); + } + break; + case OUTFLOW: + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0) = U(i, 1); + V(i, 0) = V(i, 1); + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, R)) { + switch (d->bcRight) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = 0.0; + V(imaxLocal + 1, j) = -V(imaxLocal, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = 0.0; + V(imaxLocal + 1, j) = V(imaxLocal, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = U(imaxLocal - 1, j); + V(imaxLocal + 1, j) = V(imaxLocal, j); + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, L)) { + switch (d->bcLeft) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = -V(1, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = V(1, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = U(1, j); + V(0, j) = V(1, j); + } + break; + case PERIODIC: + break; + } + } +} + +void setSpecialBoundaryCondition(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + double* u = d->u; + double* s = d->grid.s; + + if (strcmp(d->problem, "dcavity") == 0) { + if (commIsBoundary(&d->comm, T)) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal); + } + } + } else if (strcmp(d->problem, "canal") == 0) { + if (commIsBoundary(&d->comm, L)) { + + double ylength = d->grid.ylength; + double dy = d->grid.dy; + int rest = d->grid.jmax % d->comm.dims[JDIM]; + int yc = d->comm.rank * (d->grid.jmax / d->comm.dims[JDIM]) + + MIN(rest, d->comm.rank); + double ys = dy * (yc + 0.5); + double y; + + // printf("RANK %d yc: %d ys: %f\n",d->comm.rank, yc, ys); + + for (int j = 1; j < jmaxLocal + 1; j++) { + y = ys + dy * (j - 0.5); + U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } + } else if (strcmp(d->problem, "backstep") == 0) { + for (int j = 1; j < jmaxLocal + 1; j++) { + if (S(0, j) == FLUID) U(0, j) = 1.0; + } + } else if (strcmp(d->problem, "karman") == 0) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 1.0; + } + } + /* print(solver, solver->u); */ +} + +void setObjectBoundaryCondition(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + double* u = d->u; + double* v = d->v; + double* s = d->grid.s; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + switch ((int)S(i, j)) { + case TOP: + U(i, j) = -U(i, j + 1); + U(i - 1, j) = -U(i - 1, j + 1); + V(i, j) = 0.0; + break; + case BOTTOM: + U(i, j) = -U(i, j - 1); + U(i - 1, j) = -U(i - 1, j - 1); + V(i, j) = 0.0; + break; + case LEFT: + U(i - 1, j) = 0.0; + V(i, j) = -V(i - 1, j); + V(i, j - 1) = -V(i - 1, j - 1); + break; + case RIGHT: + U(i, j) = 0.0; + V(i, j) = -V(i + 1, j); + V(i, j - 1) = -V(i + 1, j - 1); + break; + case TOPLEFT: + U(i, j) = -U(i, j + 1); + U(i - 1, j) = 0.0; + V(i, j) = 0.0; + V(i, j - 1) = -V(i - 1, j - 1); + break; + case TOPRIGHT: + U(i, j) = 0.0; + U(i - 1, j) = -U(i - 1, j + 1); + V(i, j) = 0.0; + V(i, j - 1) = -V(i + 1, j - 1); + break; + case BOTTOMLEFT: + U(i, j) = -U(i, j - 1); + U(i - 1, j) = 0.0; + V(i, j) = -V(i - 1, j); + V(i, j - 1) = 0.0; + break; + case BOTTOMRIGHT: + U(i, j) = 0.0; + U(i - 1, j) = -U(i - 1, j - 1); + V(i, j) = -V(i, j + 1); + V(i, j - 1) = 0.0; + break; + } + } + } +} + +void computeFG(Discretization* d) +{ + double* u = d->u; + double* v = d->v; + double* f = d->f; + double* g = d->g; + double* s = d->grid.s; + + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + + double gx = d->gx; + double gy = d->gy; + double gamma = d->gamma; + double dt = d->dt; + double inverseRe = 1.0 / d->re; + double inverseDx = 1.0 / d->grid.dx; + double inverseDy = 1.0 / d->grid.dy; + double du2dx, dv2dy, duvdx, duvdy; + double du2dx2, du2dy2, dv2dx2, dv2dy2; + + commExchange(&d->comm, u); + commExchange(&d->comm, v); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if (S(i, j) == FLUID) { + du2dx = inverseDx * 0.25 * + ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - + (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + + fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); + + duvdy = inverseDy * 0.25 * + ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - + (V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) + U(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + + fabs(V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) - U(i, j - 1))); + + du2dx2 = inverseDx * inverseDx * + (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); + du2dy2 = inverseDy * inverseDy * + (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); + F(i, j) = U(i, j) + + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - + (U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) + V(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + + fabs(U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) - V(i - 1, j))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - + (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + + fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); + + dv2dx2 = inverseDx * inverseDx * + (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); + dv2dy2 = inverseDy * inverseDy * + (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); + G(i, j) = V(i, j) + + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); + } else { + switch ((int)S(i, j)) { + case TOP: + G(i, j) = V(i, j); + break; + case BOTTOM: + G(i, j - 1) = V(i, j - 1); + break; + case LEFT: + F(i - 1, j) = U(i - 1, j); + break; + case RIGHT: + F(i, j) = U(i, j); + break; + case TOPLEFT: + F(i - 1, j) = U(i - 1, j); + G(i, j) = V(i, j); + break; + case TOPRIGHT: + F(i, j) = U(i, j); + G(i, j) = V(i, j); + break; + case BOTTOMLEFT: + F(i - 1, j) = U(i - 1, j); + G(i, j - 1) = V(i, j - 1); + break; + case BOTTOMRIGHT: + F(i, j) = U(i, j); + G(i, j - 1) = V(i, j - 1); + break; + } + } + } + } + + /* ----------------------------- boundary of F --------------------------- */ + if (commIsBoundary(&d->comm, L)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(0, j) = U(0, j); + } + } + + if (commIsBoundary(&d->comm, R)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(imaxLocal, j) = U(imaxLocal, j); + } + } + + /* ----------------------------- boundary of G --------------------------- */ + if (commIsBoundary(&d->comm, B)) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, 0) = V(i, 0); + } + } + + if (commIsBoundary(&d->comm, T)) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, jmaxLocal) = V(i, jmaxLocal); + } + } +} + +void adaptUV(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + + double* p = d->p; + double* u = d->u; + double* v = d->v; + double* f = d->f; + double* g = d->g; + + double factorX = d->dt / d->grid.dx; + double factorY = d->dt / d->grid.dy; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; + V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; + } + } +} + +void writeResult(Discretization* d, double* u, double* v, double* p) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double dx = d->grid.dx; + double dy = d->grid.dy; + double x = 0.0, y = 0.0; + + FILE* fp; + fp = fopen("pressure.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j <= jmax; j++) { + y = (double)(j - 0.5) * dy; + for (int i = 1; i <= imax; i++) { + x = (double)(i - 0.5) * dx; + fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax + 2) + i]); + } + fprintf(fp, "\n"); + } + + fclose(fp); + + fp = fopen("velocity.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j <= jmax; j++) { + y = dy * (j - 0.5); + for (int i = 1; i <= imax; i++) { + x = dx * (i - 0.5); + double velU = (u[j * (imax + 2) + i] + u[j * (imax + 2) + (i - 1)]) / 2.0; + double velV = (v[j * (imax + 2) + i] + v[(j - 1) * (imax + 2) + i]) / 2.0; + double len = sqrt((velU * velU) + (velV * velV)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, velU, velV, len); + } + } + + fclose(fp); +} diff --git a/EnhancedSolver/2D-mpi/src/discretization.h b/EnhancedSolver/2D-mpi/src/discretization.h new file mode 100644 index 0000000..5765a2e --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/discretization.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ +#include "comm.h" +#include "grid.h" +#include "parameter.h" +#include + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +enum OBJECTBOUNDARY { + FLUID = 0, + TOP, + BOTTOM, + LEFT, + RIGHT, + TOPLEFT, + BOTTOMLEFT, + TOPRIGHT, + BOTTOMRIGHT, + OBSTACLE +}; + +enum SHAPE { NOSHAPE = 0, RECT, CIRCLE }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double re, tau, gamma; + double gx, gy; + /* time stepping */ + double dt, te; + double dtBound; + char* problem; + + double xLocal, yLocal, xOffset, yOffset, xOffsetEnd, yOffsetEnd; + + + int bcLeft, bcRight, bcBottom, bcTop; + /* communication */ + Comm comm; +} Discretization; + +extern void initDiscretiztion(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void setObjectBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +extern void writeResult(Discretization* s, double* u, double* v, double* p); +extern double sumOffset(double* , int , int , int ); +extern void print(Discretization* , double* ); +#endif diff --git a/EnhancedSolver/2D-mpi/src/grid.h b/EnhancedSolver/2D-mpi/src/grid.h new file mode 100644 index 0000000..d326641 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/grid.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __GRID_H_ +#define __GRID_H_ + +typedef struct { + double dx, dy; + int imax, jmax; + double xlength, ylength; + double* s; +} Grid; + +#endif // __GRID_H_ diff --git a/EnhancedSolver/2D-mpi/src/likwid-marker.h b/EnhancedSolver/2D-mpi/src/likwid-marker.h new file mode 100644 index 0000000..c3770c0 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/EnhancedSolver/2D-mpi/src/main.c b/EnhancedSolver/2D-mpi/src/main.c new file mode 100644 index 0000000..1f3f79c --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/main.c @@ -0,0 +1,125 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "discretization.h" +#include "grid.h" +#include "parameter.h" +#include "particletracing.h" +#include "progress.h" +#include "timing.h" + +static void writeResults(Discretization* s) +{ +#ifdef _MPI + size_t bytesize = (s->grid.imax + 2) * (s->grid.jmax + 2) * sizeof(double); + + double* ug = allocate(64, bytesize); + double* vg = allocate(64, bytesize); + double* pg = allocate(64, bytesize); + + commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->grid.imax, s->grid.jmax); + if (commIsMaster(&s->comm)) { + writeResult(s, ug, vg, pg); + } + + free(ug); + free(vg); + free(pg); +#else + writeResult(s, s->u, s->v, s->p); +#endif +} + +int main(int argc, char** argv) +{ + int rank; + double timeStart, timeStop; + Parameter p; + Discretization d; + Solver s; + ParticleTracer particletracer; + + commInit(&d.comm, argc, argv); + initParameter(&p); + FILE* fp; + fp = initResidualWriter(); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(&p, argv[1]); + commPartition(&d.comm, p.jmax, p.imax); + if (commIsMaster(&d.comm)) { + printParameter(&p); + } + + initDiscretiztion(&d, &p); + initSolver(&s, &d, &p); + initParticleTracer(&particletracer, &d, &p); + +#ifdef TEST + commPrintConfig(&d.comm); + commTestInit(&d.comm, d.p, d.f, d.g); + commExchange(&d.comm, d.p); + commShift(&d.comm, d.f, d.g); + commTestWrite(&d.comm, d.p, d.f, d.g); + writeResults(&d); + commFinalize(&d.comm); + exit(EXIT_SUCCESS); +#endif +#ifndef VERBOSE + initProgress(d.te); +#endif + + double tau = d.tau; + double te = d.te; + double t = 0.0; + double res = 0.0; + timeStart = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + setObjectBoundaryCondition(&d); + computeFG(&d); + computeRHS(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + trace(&particletracer, &d, t); + + writeResidual(fp, t, res); + + t += d.dt; + +#ifdef VERBOSE + if (commIsMaster(s.comm)) { + printf("TIME %f , TIMESTEP %f\n", t, d.dt); + } +#else + printProgress(t); +#endif + } + timeStop = getTimeStamp(); +#ifndef VERBOSE + stopProgress(); +#endif + if (commIsMaster(s.comm)) { + printf("Solution took %.2fs\n", timeStop - timeStart); + } + fclose(fp); + freeParticles(&particletracer); + writeResults(&d); + commFinalize(s.comm); + return EXIT_SUCCESS; +} diff --git a/EnhancedSolver/2D-mpi/src/parameter.c b/EnhancedSolver/2D-mpi/src/parameter.c new file mode 100644 index 0000000..3f388c4 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/parameter.c @@ -0,0 +1,143 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.8; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_STRING(name); + PARSE_INT(bcLeft); + PARSE_INT(bcRight); + PARSE_INT(bcBottom); + PARSE_INT(bcTop); + PARSE_INT(levels); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(p_init); + + /* Added new particle tracing parameters */ + PARSE_INT(numberOfParticles); + PARSE_REAL(startTime); + PARSE_REAL(injectTimePeriod); + PARSE_REAL(writeTimePeriod); + PARSE_REAL(x1); + PARSE_REAL(y1); + PARSE_REAL(x2); + PARSE_REAL(y2); + + /* Added obstacle geometry parameters */ + PARSE_INT(shape); + PARSE_REAL(xCenter); + PARSE_REAL(yCenter); + PARSE_REAL(xRectLength); + PARSE_REAL(yRectLength); + PARSE_REAL(circleRadius); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + param->bcLeft, + param->bcRight, + param->bcBottom, + param->bcTop); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength); + printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); + printf("Particle tracer parameters:\n"); + printf("\tNumber of particles: %d\n", param->numberOfParticles); + printf("\tstartTime : %f\n", param->startTime); + printf("\tinjecTimePeriod : %f\n", param->injectTimePeriod); + printf("\twriteTimePeriod: %f\n", param->writeTimePeriod); + printf("\tx1 : %f, x2 : %f, y1 : %f, y2 : %f\n", + param->x1, + param->x2, + param->y1, + param->y2); + printf("\tShape : %d\n", param->shape); +} diff --git a/EnhancedSolver/2D-mpi/src/parameter.h b/EnhancedSolver/2D-mpi/src/parameter.h new file mode 100644 index 0000000..3085db3 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/parameter.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + double xlength, ylength; + int imax, jmax; + int itermax; + double eps, omg; + double re, tau, gamma; + double te, dt; + double gx, gy; + char* name; + int bcLeft, bcRight, bcBottom, bcTop; + double u_init, v_init, p_init; + + int levels, presmooth, postsmooth; + + int numberOfParticles; + double startTime, injectTimePeriod, writeTimePeriod; + + double x1, y1, x2, y2; + + int shape; + double xCenter, yCenter, xRectLength, yRectLength, circleRadius; + + +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/EnhancedSolver/2D-mpi/src/particletracing.c b/EnhancedSolver/2D-mpi/src/particletracing.c new file mode 100644 index 0000000..f4bb874 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/particletracing.c @@ -0,0 +1,675 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "particletracing.h" + +#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] +#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] +#define S(i, j) s[(j) * (imaxLocal + 2) + (i)] + +static int ts = 0; + +#define IDIM 0 +#define JDIM 1 + +#define XOFFSET 0 +#define YOFFSET 1 +#define XOFFSETEND 2 +#define YOFFSETEND 3 + +static double sum(int* sizes, int size) +{ + double sum = 0; + + for (int i = 0; i < size; ++i) { + sum += sizes[i]; + } + + return sum; +} + +void printUV(ParticleTracer* particletracer, double* u, double* v) +{ + int imaxLocal = particletracer->imaxLocal; + + for (int i = 0; i < particletracer->size; i++) { + if (i == particletracer->rank) { + printf( + "\n### RANK %d #######################################################\n", + particletracer->rank); + printf("\nGrid U : \n"); + for (int j = 0; j < particletracer->jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < particletracer->imaxLocal + 2; i++) { + printf("%4.2f ", u[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + printf("\nGrid V : \n"); + for (int j = 0; j < particletracer->jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < particletracer->imaxLocal + 2; i++) { + printf("%4.2f ", v[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + +#ifdef _MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + } +} + +void initParticleTracer( + ParticleTracer* particletracer, Discretization* d, Parameter* params) +{ + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + + /* initializing local properties from params */ + particletracer->rank = d->comm.rank; + particletracer->size = d->comm.size; + + particletracer->numberOfParticles = params->numberOfParticles; + particletracer->startTime = params->startTime; + particletracer->injectTimePeriod = params->injectTimePeriod; + particletracer->writeTimePeriod = params->writeTimePeriod; + + particletracer->dt = params->dt; + particletracer->dx = params->xlength / params->imax; + particletracer->dy = params->ylength / params->jmax; + + particletracer->xlength = params->xlength; + particletracer->ylength = params->ylength; + + particletracer->x1 = params->x1; + particletracer->y1 = params->y1; + particletracer->x2 = params->x2; + particletracer->y2 = params->y2; + + particletracer->lastInjectTime = params->startTime; + particletracer->lastUpdateTime = params->startTime; + particletracer->lastWriteTime = params->startTime; + + particletracer->pointer = 0; + particletracer->totalParticles = 0; + particletracer->removedParticles = 0; + + particletracer->imax = params->imax; + particletracer->jmax = params->jmax; + + particletracer->imaxLocal = d->comm.imaxLocal; + particletracer->jmaxLocal = d->comm.jmaxLocal; + particletracer->removedParticles = 0; + + // Estimating the number of particles over the number of timesteps that could be + // required. + particletracer->estimatedNumParticles = (particletracer->imaxLocal * + particletracer->jmaxLocal); + + // Allocating memory for the estimated particles over the timesteps. + particletracer->particlePool = malloc( + sizeof(Particle) * particletracer->estimatedNumParticles); + + // Initializing the number of particles to 0 and turning OFF all of the particles. + // Contain information in x and y length metric, not i and j discretization metrics. + for (int i = 0; i < particletracer->estimatedNumParticles; ++i) { + particletracer->particlePool[i].x = 0.0; + particletracer->particlePool[i].y = 0.0; + particletracer->particlePool[i].flag = false; + } + + // Creating a linearly spaced particle line. + particletracer->linSpaceLine = malloc( + sizeof(Particle) * particletracer->numberOfParticles); + + // Creating an array for each rank that will hold information about + // offsets from other ranks. Holds each ranks x and y length metrics. + particletracer->offset = (double*)malloc(sizeof(double) * 4 * particletracer->size); + + // Calculating each ranks local x and y length metrics. + + double offset[4][particletracer->size]; + + // Calculating each ranks x and y local lengths. + particletracer->xLocal = d->xLocal; + particletracer->yLocal = d->yLocal; + + double xLocal[particletracer->size]; + double yLocal[particletracer->size]; + + // Calculate own x and y length metric offset based on other ranks offset data. + particletracer->xOffset = d->xOffset; + particletracer->yOffset = d->yOffset; + particletracer->xOffsetEnd = d->xOffsetEnd; + particletracer->yOffsetEnd = d->yOffsetEnd; + + printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : " + "%.2f\n", + particletracer->rank, + particletracer->xOffset, + particletracer->yOffset, + particletracer->xOffsetEnd, + particletracer->yOffsetEnd); + +#ifdef _MPI + // Gather each ranks x and y length metric that marks each ranks own territory. + // Once the boundary leaves local domain, then it needs to know which ranks to send. + // And to know whos boundary it is, we need to know the rank. + MPI_Allgather(&particletracer->xOffset, + 1, + MPI_DOUBLE, + offset[0], + 1, + MPI_DOUBLE, + d->comm.comm); + MPI_Allgather(&particletracer->yOffset, + 1, + MPI_DOUBLE, + offset[1], + 1, + MPI_DOUBLE, + d->comm.comm); + MPI_Allgather(&particletracer->xOffsetEnd, + 1, + MPI_DOUBLE, + offset[2], + 1, + MPI_DOUBLE, + d->comm.comm); + MPI_Allgather(&particletracer->yOffsetEnd, + 1, + MPI_DOUBLE, + offset[3], + 1, + MPI_DOUBLE, + d->comm.comm); +#endif + + memcpy(particletracer->offset, offset, sizeof(offset)); + + particleRandomizer(particletracer); + +#ifdef _MPI + // Create the mpi_particle datatype + MPI_Datatype mpi_particle; + int lengths[3] = { 1, 1, 1 }; + + MPI_Aint displacements[3]; + Particle dummy_particle; + MPI_Aint base_address; + MPI_Get_address(&dummy_particle, &base_address); + MPI_Get_address(&dummy_particle.x, &displacements[0]); + MPI_Get_address(&dummy_particle.y, &displacements[1]); + MPI_Get_address(&dummy_particle.flag, &displacements[2]); + displacements[0] = MPI_Aint_diff(displacements[0], base_address); + displacements[1] = MPI_Aint_diff(displacements[1], base_address); + displacements[2] = MPI_Aint_diff(displacements[2], base_address); + + MPI_Datatype types[3] = { MPI_DOUBLE, MPI_DOUBLE, MPI_C_BOOL }; + MPI_Type_create_struct(3, + lengths, + displacements, + types, + &particletracer->mpi_particle); + MPI_Type_commit(&particletracer->mpi_particle); +#endif +} + +void printParticles(ParticleTracer* particletracer) +{ + for (int i = 0; i < particletracer->totalParticles; ++i) { + printf("Rank : %d Particle position X : %.2f, Y : %.2f, flag : %d, total pt : " + "%d, pointer : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, " + "yOffsetEnd : %.2f\n", + particletracer->rank, + particletracer->particlePool[i].x, + particletracer->particlePool[i].y, + particletracer->particlePool[i].flag, + particletracer->totalParticles, + particletracer->pointer, + particletracer->xOffset, + particletracer->yOffset, + particletracer->xOffsetEnd, + particletracer->yOffsetEnd); + } +} +void injectParticles(ParticleTracer* particletracer, double* s) +{ + double x, y; + compress(particletracer); + particleRandomizer(particletracer); + + int imaxLocal = particletracer->imaxLocal; + int jmaxLocal = particletracer->jmaxLocal; + + for (int i = 0; i < particletracer->numberOfParticles; ++i) { + x = particletracer->linSpaceLine[i].x; + y = particletracer->linSpaceLine[i].y; + if (x >= particletracer->xOffset && y >= particletracer->yOffset && + x <= particletracer->xOffsetEnd && y <= particletracer->yOffsetEnd) { + + particletracer->particlePool[particletracer->pointer].x = x; + particletracer->particlePool[particletracer->pointer].y = y; + + int i = particletracer->particlePool[particletracer->pointer].x / + particletracer->dx; + int j = particletracer->particlePool[particletracer->pointer].y / + particletracer->dy; + + int iOffset = particletracer->xOffset / particletracer->dx, + jOffset = particletracer->yOffset / particletracer->dy; + + if (S(i - iOffset, j - jOffset) == FLUID) { + particletracer->particlePool[particletracer->pointer].flag = true; + ++(particletracer->pointer); + ++(particletracer->totalParticles); + } + } + } +} + +void advanceParticles(ParticleTracer* particletracer, + double* restrict u, + double* restrict v, + double* restrict s, + Comm* comm, + double time) +{ + + int imax = particletracer->imax; + int jmax = particletracer->jmax; + int imaxLocal = particletracer->imaxLocal; + int jmaxLocal = particletracer->jmaxLocal; + + double dx = particletracer->dx; + double dy = particletracer->dy; + + double xlength = particletracer->xlength; + double ylength = particletracer->ylength; + + Particle buff[particletracer->size][(particletracer->estimatedNumParticles)]; + memset(buff, 0, sizeof(buff)); + Particle recvbuff[particletracer->size][(particletracer->estimatedNumParticles)]; + memset(buff, 0, sizeof(recvbuff)); + + int particleBufIndex[particletracer->size], + recvparticleBufIndex[particletracer->size]; + + memset(particleBufIndex, 0, sizeof(particleBufIndex)); + memset(recvparticleBufIndex, 0, sizeof(recvparticleBufIndex)); + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (particletracer->particlePool[i].flag == true) { + double xTemp = particletracer->particlePool[i].x; + double yTemp = particletracer->particlePool[i].y; + + double x = xTemp - particletracer->xOffset; + double y = yTemp - particletracer->yOffset; + + int iCoord = (int)(x / dx) + 1; + int jCoord = (int)((y + 0.5 * dy) / dy) + 1; + + double x1 = (double)(iCoord - 1) * dx; + double y1 = ((double)(jCoord - 1) - 0.5) * dy; + double x2 = (double)iCoord * dx; + double y2 = ((double)jCoord - 0.5) * dy; + + double u_n = (1.0 / (dx * dy)) * + ((x2 - x) * (y2 - y) * U(iCoord - 1, jCoord - 1) + + (x - x1) * (y2 - y) * U(iCoord, jCoord - 1) + + (x2 - x) * (y - y1) * U(iCoord - 1, jCoord) + + (x - x1) * (y - y1) * U(iCoord, jCoord)); + + double new_x = (x + particletracer->xOffset) + particletracer->dt * u_n; + particletracer->particlePool[i].x = new_x; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)(y / dy) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = (double)(jCoord - 1) * dy; + x2 = ((double)iCoord - 0.5) * dx; + y2 = (double)jCoord * dy; + + double v_n = (1.0 / (dx * dy)) * + ((x2 - x) * (y2 - y) * V(iCoord - 1, jCoord - 1) + + (x - x1) * (y2 - y) * V(iCoord, jCoord - 1) + + (x2 - x) * (y - y1) * V(iCoord - 1, jCoord) + + (x - x1) * (y - y1) * V(iCoord, jCoord)); + + double new_y = (y + particletracer->yOffset) + particletracer->dt * v_n; + particletracer->particlePool[i].y = new_y; + + if (((new_x < particletracer->xOffset) || + (new_x > particletracer->xOffsetEnd) || + (new_y < particletracer->yOffset) || + (new_y > particletracer->yOffsetEnd))) { + // New logic to transfer particles to neighbouring ranks or discard the + // particle. +#ifdef _MPI + for (int i = 0; i < particletracer->size; ++i) { + if ((new_x >= + particletracer->offset[i + particletracer->size * XOFFSET]) && + (new_x <= particletracer + ->offset[i + particletracer->size * XOFFSETEND]) && + (new_y >= + particletracer->offset[i + particletracer->size * YOFFSET]) && + (new_y <= particletracer + ->offset[i + particletracer->size * YOFFSETEND]) && + i != particletracer->rank) { + buff[i][particleBufIndex[i]].x = new_x; + buff[i][particleBufIndex[i]].y = new_y; + buff[i][particleBufIndex[i]].flag = true; + ++particleBufIndex[i]; + } + } +#endif + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + + int i_new = new_x / dx, j_new = new_y / dy; + int iOffset = particletracer->xOffset / dx, + jOffset = particletracer->yOffset / dy; + + if (S(i_new - iOffset, j_new - jOffset) != FLUID) { + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + } + } + +#ifdef _MPI + for (int i = 0; i < particletracer->size; ++i) { + if (i != comm->rank) { + MPI_Send(&particleBufIndex[i], 1, MPI_INT, i, 0, comm->comm); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Recv(&recvparticleBufIndex[i], + 1, + MPI_INT, + i, + 0, + comm->comm, + MPI_STATUS_IGNORE); + + // if (0 !=recvparticleBufIndex[i]) { + // printf("Rank %d will receive %d particles from rank %d\n", + // particletracer->rank, + // recvparticleBufIndex[i], + // i); + // } + } + } + + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Send(buff[i], + particleBufIndex[i], + particletracer->mpi_particle, + i, + 0, + comm->comm); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Recv(recvbuff[i], + recvparticleBufIndex[i], + particletracer->mpi_particle, + i, + 0, + comm->comm, + MPI_STATUS_IGNORE); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + for (int j = 0; j < recvparticleBufIndex[i]; ++j) { + particletracer->particlePool[particletracer->pointer].x = recvbuff[i][j] + .x; + particletracer->particlePool[particletracer->pointer].y = recvbuff[i][j] + .y; + particletracer->particlePool[particletracer->pointer].flag = true; + ++(particletracer->pointer); + ++(particletracer->totalParticles); + } + } + } +#endif +} + +void freeParticles(ParticleTracer* particletracer) +{ + free(particletracer->particlePool); + free(particletracer->linSpaceLine); + free(particletracer->offset); +} + +void writeParticles(ParticleTracer* particletracer, Comm* comm) +{ + int collectedBuffIndex[particletracer->size]; + compress(particletracer); +#ifdef _MPI + + MPI_Gather(&particletracer->totalParticles, + 1, + MPI_INT, + collectedBuffIndex, + 1, + MPI_INT, + 0, + comm->comm); + + if (particletracer->rank != 0) { + Particle buff[particletracer->totalParticles]; + for (int i = 0; i < particletracer->totalParticles; ++i) { + buff[i].x = particletracer->particlePool[i].x; + buff[i].y = particletracer->particlePool[i].y; + buff[i].flag = particletracer->particlePool[i].flag; + // printf("Rank : %d sending to rank 0 X : %.2f, Y : %.2f with totalpt : + // %d\n", particletracer->rank, buff[i].x, buff[i].y, + // particletracer->totalParticles); + } + MPI_Send(buff, + particletracer->totalParticles, + particletracer->mpi_particle, + 0, + 1, + comm->comm); + } +#endif + + if (particletracer->rank == 0) { + char filename[50]; + FILE* fp; + + snprintf(filename, 50, "vis_files/particles_%d.dat", ts); + fp = fopen(filename, "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + // fprintf(fp, "# vtk DataFile Version 3.0\n"); + // fprintf(fp, "PAMPI cfd solver particle tracing file\n"); + // fprintf(fp, "ASCII\n"); + + // fprintf(fp, "DATASET UNSTRUCTURED_GRID\n"); + // fprintf(fp, "FIELD FieldData 2\n"); + // fprintf(fp, "TIME 1 1 double\n"); + // fprintf(fp, "%d\n", ts); + // fprintf(fp, "CYCLE 1 1 int\n"); + // fprintf(fp, "1\n"); + +#ifdef _MPI + int overallTotalParticles = sum(collectedBuffIndex, particletracer->size); + + // fprintf(fp, "POINTS %d float\n", overallTotalParticles); + + // printf("Total particles : %d\n", overallTotalParticles); + for (int i = 1; i < particletracer->size; ++i) { + Particle recvBuff[collectedBuffIndex[i]]; + MPI_Recv(&recvBuff, + collectedBuffIndex[i], + particletracer->mpi_particle, + i, + 1, + comm->comm, + MPI_STATUS_IGNORE); + + for (int j = 0; j < collectedBuffIndex[i]; ++j) { + double x = recvBuff[j].x; + double y = recvBuff[j].y; + fprintf(fp, "%f %f\n", x, y); + + // printf("Rank : 0 receiving from rank %d X : %.2f, Y : %.2f with totalpt + // : %d\n", i, x, y, particletracer->totalParticles); + } + } +#else + int overallTotalParticles = particletracer->totalParticles; + + // fprintf(fp, "POINTS %d float\n", overallTotalParticles); + + // printf("Total particles : %d\n", overallTotalParticles); +#endif + for (int i = 0; i < particletracer->totalParticles; ++i) { + double x = particletracer->particlePool[i].x; + double y = particletracer->particlePool[i].y; + fprintf(fp, "%f %f\n", x, y); + } + + // fprintf(fp, "CELLS %d %d\n", overallTotalParticles, 2 * overallTotalParticles); + + // for (int i = 0; i < overallTotalParticles; ++i) + // { + // fprintf(fp, "1 %d\n", i); + // } + + // fprintf(fp, "CELL_TYPES %d\n", overallTotalParticles); + + // for (int i = 0; i < overallTotalParticles; ++i) + // { + // fprintf(fp, "1\n"); + // } + + fclose(fp); + } + + ++ts; +} + +void printParticleTracerParameters(ParticleTracer* particletracer) +{ + printf("Particle Tracing data:\n"); + printf("Rank : %d\n", particletracer->rank); + printf("\tNumber of particles : %d being injected for every period of %.2f\n", + particletracer->numberOfParticles, + particletracer->injectTimePeriod); + printf("\tstartTime : %.2f\n", particletracer->startTime); + printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : " + "%.2f, x2 : %.2f, y2 : %.2f\n", + particletracer->x1, + particletracer->y1, + particletracer->x2, + particletracer->y2); + printf("\tPointer : %d, TotalParticles : %d\n", + particletracer->pointer, + particletracer->totalParticles); + printf("\tdt : %.2f, dx : %.2f, dy : %.2f\n", + particletracer->dt, + particletracer->dx, + particletracer->dy); + printf("\txOffset : %.2f, yOffset : %.2f\n", + particletracer->xOffset, + particletracer->yOffset); + printf("\txOffsetEnd : %.2f, yOffsetEnd : %.2f\n", + particletracer->xOffsetEnd, + particletracer->yOffsetEnd); + printf("\txLocal : %.2f, yLocal : %.2f\n", + particletracer->xLocal, + particletracer->yLocal); +} + +void trace(ParticleTracer* particletracer, Discretization* d, double time) +{ + if (time >= particletracer->startTime) { + if ((time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod) { + injectParticles(particletracer, d->grid.s); + particletracer->lastInjectTime = time; + } + if ((time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod) { + writeParticles(particletracer, &d->comm); + particletracer->lastWriteTime = time; + } + advanceParticles(particletracer, d->u, d->v, d->grid.s, &d->comm, time); + + if (particletracer->removedParticles > (particletracer->totalParticles * 0.2)) { + compress(particletracer); + } + + particletracer->lastUpdateTime = time; + } +} + +void compress(ParticleTracer* particletracer) +{ + Particle* memPool = particletracer->particlePool; + Particle tempPool[particletracer->totalParticles]; + int totalParticles = 0; + + // printf("\nPerforming compression ..."); + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (memPool[i].flag == true) { + tempPool[totalParticles].x = memPool[i].x; + tempPool[totalParticles].y = memPool[i].y; + tempPool[totalParticles].flag = memPool[i].flag; + ++totalParticles; + } + } + + // printf(" remove %d particles\n", particletracer->totalParticles - totalParticles); + + particletracer->totalParticles = totalParticles; + particletracer->removedParticles = 0; + particletracer->pointer = totalParticles; + + memcpy(particletracer->particlePool, tempPool, totalParticles * sizeof(Particle)); +} + +void particleRandomizer(ParticleTracer* particletracer) +{ + memset(particletracer->linSpaceLine, + 0, + sizeof(Particle) * particletracer->numberOfParticles); + + for (int i = 0; i < particletracer->numberOfParticles; ++i) { + + particletracer->linSpaceLine[i].x = (((double)rand() / RAND_MAX) * + (particletracer->x2 - + particletracer->x1)) + + particletracer->x1; + particletracer->linSpaceLine[i].y = (((double)rand() / RAND_MAX) * + (particletracer->y2 - + particletracer->y1)) + + particletracer->y1; + particletracer->linSpaceLine[i].flag = true; + } +} \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/particletracing.h b/EnhancedSolver/2D-mpi/src/particletracing.h new file mode 100644 index 0000000..b771014 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/particletracing.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARTICLETRACING_H_ +#define __PARTICLETRACING_H_ +#include "allocate.h" +#include "parameter.h" + +#include "solver.h" +#include +#include + +#define NDIMS 2 + +typedef enum COORD { X = 0, Y, NCOORD } COORD; + +typedef struct { + double x, y; + bool flag; +} Particle; + +typedef struct { + int numberOfParticles, removedParticles, totalParticles; + double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime, + lastWriteTime; + + int estimatedNumParticles; + + double dx, dy, dt; + Particle* linSpaceLine; + Particle* particlePool; + + int pointer; + + double imax, jmax, xlength, ylength, imaxLocal, jmaxLocal; + + double x1, y1, x2, y2; + + int size, rank; + +#ifdef _MPI + MPI_Datatype mpi_particle; +#endif + + double xLocal, yLocal, xOffset, yOffset, xOffsetEnd, yOffsetEnd; + + double* offset; + +} ParticleTracer; + +extern void initParticleTracer(ParticleTracer*, Discretization*, Parameter*); +extern void injectParticles(ParticleTracer*, double*); +extern void advanceParticles(ParticleTracer*, double*, double*, double*, Comm*, double); +extern void freeParticles(ParticleTracer*); +extern void writeParticles(ParticleTracer*, Comm*); +extern void printParticleTracerParameters(ParticleTracer*); +extern void printParticles(ParticleTracer*); +extern void trace(ParticleTracer*, Discretization*, double); +extern void compress(ParticleTracer*); +extern void particleRandomizer(ParticleTracer*); + +#endif \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/progress.c b/EnhancedSolver/2D-mpi/src/progress.c new file mode 100644 index 0000000..78944bd --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/progress.c @@ -0,0 +1,70 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; + +void initProgress(double end) +{ + _end = end; + _current = 0; + + printf("[ ]"); + fflush(stdout); +} + +void printProgress(double current) +{ + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); +} + +void stopProgress() +{ + printf("\n"); + fflush(stdout); +} + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(-1); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/progress.h b/EnhancedSolver/2D-mpi/src/progress.h new file mode 100644 index 0000000..9508797 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/progress.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include + +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ + +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(); +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); +#endif diff --git a/EnhancedSolver/2D-mpi/src/solver-mg.c b/EnhancedSolver/2D-mpi/src/solver-mg.c new file mode 100644 index 0000000..920c9e7 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/solver-mg.c @@ -0,0 +1,325 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) + +#define E(i, j) e[(j) * (imaxLocal + 2) + (i)] +#define R(i, j) r[(j) * (imaxLocal + 2) + (i)] +#define OLD(i, j) old[(j) * (imaxLocal + 2) + (i)] + +void printSolver(Solver* s, double* grid, char* gridname) +{ + if (0 == s->comm->rank) printf("Grid name : %s", gridname); + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + + for (int i = 0; i < s->comm->size; i++) { + if (i == s->comm->rank) { + sleep(1 * s->comm->rank); + printf("### RANK %d LVL " + "###################################################### #\n ", + s->comm->rank); + for (int j = 0; j < jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < imaxLocal + 2; i++) { + printf("%2.2f ", grid[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + } +} + +static void restrictMG(Solver* s, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double* r = s->r[level + 1]; + double* old = s->r[level]; + +#ifdef _MPI + commExchange(comm, old); +#endif + + for (int j = 1; j < (jmaxLocal / 2) + 1; j++) { + for (int i = 1; i < (imaxLocal / 2) + 1; i++) { + R(i, j) = (OLD(2 * i - 1, 2 * j - 1) + OLD(2 * i, 2 * j - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1) + OLD(2 * i - 1, 2 * j) * 2 + + OLD(2 * i, 2 * j) * 4 + OLD(2 * i + 1, 2 * j) * 2 + + OLD(2 * i - 1, 2 * j + 1) + OLD(2 * i, 2 * j + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1)) / + 16.0; + } + } +} + +static void prolongate(Solver* s, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double* old = s->r[level + 1]; + double* e = s->r[level]; + + for (int j = 2; j < jmaxLocal + 1; j += 2) { + for (int i = 2; i < imaxLocal + 1; i += 2) { + E(i, j) = OLD(i / 2, j / 2); + } + } +} + +static void correct(Solver* s, double* p, int level, Comm* comm) +{ + double* e = s->e[level]; + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + for (int j = 1; j < jmaxLocal + 1; ++j) { + for (int i = 1; i < imaxLocal + 1; ++i) { + P(i, j) += E(i, j); + } + } +} + +static void setBoundaryCondition(Solver* s, double* p, int imaxLocal, int jmaxLocal) +{ +#ifdef _MPI + if (commIsBoundary(s->comm, B)) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (commIsBoundary(s->comm, T)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (commIsBoundary(s->comm, L)) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (commIsBoundary(s->comm, R)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } +#else + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + P(imaxLocal + 1, j) = P(imaxLocal, j); + } +#endif +} + +static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + +#ifdef _MPI + commExchange(comm, p); +#endif + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + P(i, j) -= factor * + (RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + +#ifdef _MPI + commExchange(comm, p); +#endif + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + R(i, j) = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + res += (R(i, j) * R(i, j)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + +#ifdef _MPI + commReduction(&res, SUM); +#endif + + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + return res; +} + +static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, comm); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, comm); + + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + } + + // calculate residuals + res = calculateResidual(s, p, rhs, level, comm); + + // restrict + restrictMG(s, level, comm); + + Comm newcomm; + commUpdateDatatypes(s->comm, &newcomm, comm->imaxLocal / 2, comm->jmaxLocal / 2); + + // MGSolver on residual and error. + multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm); + + commFreeCommunicator(&newcomm); + + // prolongate + prolongate(s, level, comm); + + // correct p on finer level using residual + correct(s, p, level, comm); + + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->comm = &d->comm; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int levels = s->levels; + printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * sizeof(double); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size); + s->e[j] = allocate(64, size); + + for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->comm); + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Residuum: %.6f\n", res); + } +#endif +return res; +} diff --git a/EnhancedSolver/2D-mpi/src/solver-rb.c b/EnhancedSolver/2D-mpi/src/solver-rb.c new file mode 100644 index 0000000..7d67075 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/solver-rb.c @@ -0,0 +1,131 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "discretization.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +void printSolver(Solver* s, double* grid, char* gridname) +{ + printf("Grid name : %s", gridname); + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + + for (int i = 0; i < s->comm->size; i++) { + if (i == s->comm->rank) { + sleep(1 * s->comm->rank); + printf("### RANK %d LVL " + "###################################################### #\n ", + s->comm->rank); + for (int j = 0; j < jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < imaxLocal + 2; i++) { + printf("%2.2f ", grid[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + } +} + + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->comm = &d->comm; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double epssq = eps * eps; + int pass, jsw, isw; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + jsw = 1; + for (pass = 0; pass < 2; pass++) { + isw = jsw; + commExchange(s->comm, p); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + + if (commIsBoundary(s->comm, B)) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (commIsBoundary(s->comm, T)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (commIsBoundary(s->comm, L)) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (commIsBoundary(s->comm, R)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } + + commReduction(&res, SUM); + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + it++; + } + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + +return res; +} diff --git a/EnhancedSolver/2D-mpi/src/solver.h b/EnhancedSolver/2D-mpi/src/solver.h new file mode 100644 index 0000000..fd47ff1 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/solver.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "comm.h" +#include "discretization.h" +#include "grid.h" +#include "mpi.h" +#include "parameter.h" + +typedef struct { + /* geometry and grid information */ + Grid* grid; + /* parameters */ + double eps, omega; + int itermax; + int levels, presmooth, postsmooth; + double **r, **e; + /* communication */ + Comm* comm; +} Solver; + +extern void initSolver(Solver*, Discretization*, Parameter*); +extern double solve(Solver*, double*, double*); +extern void printSolver(Solver* , double*, char*); +#endif diff --git a/EnhancedSolver/2D-mpi/src/timing.c b/EnhancedSolver/2D-mpi/src/timing.c new file mode 100644 index 0000000..78b01c4 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/timing.c @@ -0,0 +1,22 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution(void) +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} diff --git a/EnhancedSolver/2D-mpi/src/timing.h b/EnhancedSolver/2D-mpi/src/timing.h new file mode 100644 index 0000000..ed05a8c --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/timing.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(void); +extern double getTimeResolution(void); + +#endif // __TIMING_H_ diff --git a/EnhancedSolver/2D-mpi/src/util.h b/EnhancedSolver/2D-mpi/src/util.h new file mode 100644 index 0000000..ae396bd --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/util.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#define P(i, j) p[(j) * (imaxLocal + 2) + (i)] +#define F(i, j) f[(j) * (imaxLocal + 2) + (i)] +#define G(i, j) g[(j) * (imaxLocal + 2) + (i)] +#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] +#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] +#define S(i, j) s[(j) * (imaxLocal + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)] + +#endif // __UTIL_H_ diff --git a/EnhancedSolver/2D-mpi/surface.plot b/EnhancedSolver/2D-mpi/surface.plot new file mode 100644 index 0000000..4f7ccd9 --- /dev/null +++ b/EnhancedSolver/2D-mpi/surface.plot @@ -0,0 +1,7 @@ +set terminal png size 1024,768 enhanced font ,12 +set output 'p.png' +set datafile separator whitespace + +set grid +set hidden3d +splot 'pressure.dat' using 1:2:3 with lines diff --git a/EnhancedSolver/2D-mpi/vector.plot b/EnhancedSolver/2D-mpi/vector.plot new file mode 100644 index 0000000..e24b2b2 --- /dev/null +++ b/EnhancedSolver/2D-mpi/vector.plot @@ -0,0 +1,6 @@ +set terminal png size 3600,1400 enhanced font ,12 +set output 'velocity.png' +set datafile separator whitespace +set size ratio -1 + +plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/EnhancedSolver/2D-mpi/vis_files/animate.plot b/EnhancedSolver/2D-mpi/vis_files/animate.plot new file mode 100644 index 0000000..d54ba32 --- /dev/null +++ b/EnhancedSolver/2D-mpi/vis_files/animate.plot @@ -0,0 +1,10 @@ +unset border; unset tics; unset key; +set term gif animate delay 30 +set output "trace.gif" +set xrange [0:1] +set yrange [0:1] + +do for [ts=0:120] { + plot "particles_".ts.".dat" with points pointtype 7 +} +unset output diff --git a/EnhancedSolver/2D-mpi/vis_files/backstep_animate.plot b/EnhancedSolver/2D-mpi/vis_files/backstep_animate.plot new file mode 100644 index 0000000..028a26f --- /dev/null +++ b/EnhancedSolver/2D-mpi/vis_files/backstep_animate.plot @@ -0,0 +1,14 @@ +unset border; unset tics; unset key; +set term gif animate delay 10 +set output "trace.gif" +set xrange [0:7] +set yrange [0:1.5] +set size ratio -1 + +set object 1 rect from 0.0,0.0 to 1.0,0.5 lw 5 + + +do for [ts=0:550] { + plot "particles_".ts.".dat" with points pointtype 7 +} +unset output diff --git a/EnhancedSolver/2D-mpi/vis_files/canal_animate.plot b/EnhancedSolver/2D-mpi/vis_files/canal_animate.plot new file mode 100644 index 0000000..bf9ce60 --- /dev/null +++ b/EnhancedSolver/2D-mpi/vis_files/canal_animate.plot @@ -0,0 +1,10 @@ +unset border; unset tics; unset key; +set term gif animate delay 30 +set output "trace.gif" +set xrange [0:30] +set yrange [0:4] + +do for [ts=0:120] { + plot "particles_".ts.".dat" with points pointtype 7 +} +unset output diff --git a/EnhancedSolver/2D-mpi/vis_files/karman_animate.plot b/EnhancedSolver/2D-mpi/vis_files/karman_animate.plot new file mode 100644 index 0000000..d78f0c3 --- /dev/null +++ b/EnhancedSolver/2D-mpi/vis_files/karman_animate.plot @@ -0,0 +1,13 @@ +unset border; unset tics; unset key; +set term gif animate delay 10 +set output "trace.gif" +set xrange [0:30] +set yrange [0:8] +set size ratio -1 +set object 1 circle front at 5.0,4.0 size 1.0 fillcolor rgb "black" lw 2 + + +do for [ts=0:500] { + plot "particles_".ts.".dat" with points pointtype 7 pointsize 0.3 +} +unset output diff --git a/EnhancedSolver/2D-seq/Makefile b/EnhancedSolver/2D-seq/Makefile index dc8a79d..33c7ad8 100644 --- a/EnhancedSolver/2D-seq/Makefile +++ b/EnhancedSolver/2D-seq/Makefile @@ -38,9 +38,9 @@ $(BUILD_DIR)/%.s: %.c $(info ===> GENERATE ASM $@) $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ -.PHONY: clean distclean tags info asm format +.PHONY: clean distclean vis vis_clean tags info asm format -clean: +clean: vis_clean $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @rm -f tags @@ -70,6 +70,19 @@ format: done @echo "Done" +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='pressure.dat'" ./surface.plot + @gnuplot -e "filename='velocity.dat'" ./vector.plot + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.png + @rm -f ./vis_files/*.dat + @rm -f ./vis_files/*.gif + $(BUILD_DIR): @mkdir $(BUILD_DIR) diff --git a/EnhancedSolver/2D-seq/backstep.par b/EnhancedSolver/2D-seq/backstep.par index 656ac31..fb25403 100644 --- a/EnhancedSolver/2D-seq/backstep.par +++ b/EnhancedSolver/2D-seq/backstep.par @@ -15,7 +15,7 @@ bcRight 3 # gx 0.0 # Body forces (e.g. gravity) gy 0.0 # -re 36500.0 # Reynolds number +re 36000.0 # Reynolds number u_init 1.0 # initial value for velocity in x-direction v_init 0.0 # initial value for velocity in y-direction @@ -45,11 +45,18 @@ rho 0.52 omg 1.8 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Particle Tracing Data: # ----------------------- numberOfParticles 200 -startTime 100 +startTime 0 injectTimePeriod 1.0 writeTimePeriod 0.5 diff --git a/EnhancedSolver/2D-seq/canal.par b/EnhancedSolver/2D-seq/canal.par index 1661db6..87a797a 100644 --- a/EnhancedSolver/2D-seq/canal.par +++ b/EnhancedSolver/2D-seq/canal.par @@ -25,14 +25,14 @@ p_init 1.0 # initial value for pressure # ------------- xlength 30.0 # domain size in x-direction -ylength 4.0 # domain size in y-direction -imax 256 # number of interior cells in x-direction -jmax 64 # number of interior cells in y-direction +ylength 4.0 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 40 # number of interior cells in y-direction # Time Data: # --------- -te 80.0 # final time +te 60.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) @@ -44,7 +44,13 @@ eps 0.0001 # stopping tolerance for pressure iteration rho 0.52 omg 1.8 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma -levels 5 # Multigrid levels + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations # Particle Tracing Data: # ----------------------- diff --git a/EnhancedSolver/2D-seq/config.mk b/EnhancedSolver/2D-seq/config.mk index ceed6bf..af8912f 100644 --- a/EnhancedSolver/2D-seq/config.mk +++ b/EnhancedSolver/2D-seq/config.mk @@ -1,5 +1,5 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC ENABLE_OPENMP ?= false # Supported: sor, rb, mg SOLVER ?= mg diff --git a/EnhancedSolver/2D-seq/dcavity.par b/EnhancedSolver/2D-seq/dcavity.par index 8ff64b6..2b7b416 100644 --- a/EnhancedSolver/2D-seq/dcavity.par +++ b/EnhancedSolver/2D-seq/dcavity.par @@ -44,7 +44,13 @@ eps 0.001 # stopping tolerance for pressure iteration rho 0.5 omg 1.8 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma -levels 5 # Multigrid levels + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations # Particle Tracing Data: # ----------------------- diff --git a/EnhancedSolver/2D-seq/karman.par b/EnhancedSolver/2D-seq/karman.par index 60664b6..fb61961 100644 --- a/EnhancedSolver/2D-seq/karman.par +++ b/EnhancedSolver/2D-seq/karman.par @@ -44,13 +44,19 @@ eps 0.001 # stopping tolerance for pressure iteration rho 0.52 omg 1.75 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma -levels 5 # Multigrid levels + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations # Particle Tracing Data: # ----------------------- numberOfParticles 200 -startTime 201 +startTime 50 injectTimePeriod 1.0 writeTimePeriod 0.5 diff --git a/EnhancedSolver/2D-seq/residual.plot b/EnhancedSolver/2D-seq/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/EnhancedSolver/2D-seq/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/EnhancedSolver/2D-seq/src/main.c b/EnhancedSolver/2D-seq/src/main.c index 079d5e3..34deea4 100644 --- a/EnhancedSolver/2D-seq/src/main.c +++ b/EnhancedSolver/2D-seq/src/main.c @@ -24,6 +24,8 @@ int main(int argc, char** argv) ParticleTracer particletracer; initParameter(&p); + FILE* fp; + fp = initResidualWriter(); if (argc != 2) { printf("Usage: %s \n", argv[0]); @@ -45,8 +47,9 @@ int main(int argc, char** argv) double te = d.te; double t = 0.0; int nt = 0; + double res = 0.0; - timeStart = getTimeStamp(); + timeStart = getTimeStamp(); while (t <= te) { if (tau > 0.0) computeTimestep(&d); @@ -57,10 +60,12 @@ int main(int argc, char** argv) computeFG(&d); computeRHS(&d); if (nt % 100 == 0) normalizePressure(&d); - solve(&s, d.p, d.rhs); + res = solve(&s, d.p, d.rhs); adaptUV(&d); trace(&particletracer, d.u, d.v, d.dt, t); + writeResidual(fp, t, res); + t += d.dt; nt++; @@ -73,6 +78,7 @@ int main(int argc, char** argv) timeStop = getTimeStamp(); + fclose(fp); stopProgress(); freeParticles(&particletracer); printf("Solution took %.2fs\n", timeStop - timeStart); diff --git a/EnhancedSolver/2D-seq/src/parameter.c b/EnhancedSolver/2D-seq/src/parameter.c index cf40d51..48c3181 100644 --- a/EnhancedSolver/2D-seq/src/parameter.c +++ b/EnhancedSolver/2D-seq/src/parameter.c @@ -14,18 +14,20 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->imax = 100; - param->jmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.7; - param->re = 100.0; - param->gamma = 0.9; - param->tau = 0.5; - param->rho = 0.99; - param->levels = 5; + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; + param->rho = 0.99; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) @@ -64,6 +66,8 @@ void readParameter(Parameter* param, const char* filename) PARSE_INT(jmax); PARSE_INT(itermax); PARSE_INT(levels); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); PARSE_REAL(eps); PARSE_REAL(omg); PARSE_REAL(re); diff --git a/EnhancedSolver/2D-seq/src/parameter.h b/EnhancedSolver/2D-seq/src/parameter.h index 9b905ca..ffc880e 100644 --- a/EnhancedSolver/2D-seq/src/parameter.h +++ b/EnhancedSolver/2D-seq/src/parameter.h @@ -10,7 +10,7 @@ typedef struct { double xlength, ylength; int imax, jmax; - int itermax, levels; + int itermax, levels, presmooth, postsmooth; double eps, omg, rho; double re, tau, gamma; double te, dt; diff --git a/EnhancedSolver/2D-seq/src/particletracing.c b/EnhancedSolver/2D-seq/src/particletracing.c index 07d575d..3208245 100644 --- a/EnhancedSolver/2D-seq/src/particletracing.c +++ b/EnhancedSolver/2D-seq/src/particletracing.c @@ -98,7 +98,7 @@ static void advanceParticles( if (!gridIsFluid(p->grid, newI, newJ)) { p->particlePool[i].flag = false; p->removedParticles++; - printf("Forbidden movement of particle into obstacle!\n"); + // printf("Forbidden movement of particle into obstacle!\n"); } } } @@ -110,7 +110,7 @@ static void compress(ParticleTracer* p) Particle tempPool[p->totalParticles]; int totalParticles = 0; - printf("Performing compression ..."); + // printf("Performing compression ..."); for (int i = 0; i < p->totalParticles; i++) { if (memPool[i].flag == 1) { @@ -121,7 +121,7 @@ static void compress(ParticleTracer* p) } } - printf(" remove %d particles\n", p->totalParticles - totalParticles); + // printf(" remove %d particles\n", p->totalParticles - totalParticles); p->totalParticles = totalParticles; p->removedParticles = 0; p->pointer = totalParticles + 1; @@ -130,7 +130,9 @@ static void compress(ParticleTracer* p) void writeParticles(ParticleTracer* p) { - static int ts = 0; + static int ts = 0; + compress(p); + VtkOptions opts = { .particletracer = p }; char filename[50]; diff --git a/EnhancedSolver/2D-seq/src/progress.c b/EnhancedSolver/2D-seq/src/progress.c index d5393c4..54a1dbc 100644 --- a/EnhancedSolver/2D-seq/src/progress.c +++ b/EnhancedSolver/2D-seq/src/progress.c @@ -49,3 +49,21 @@ void stopProgress() printf("\n"); fflush(stdout); } +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} \ No newline at end of file diff --git a/EnhancedSolver/2D-seq/src/progress.h b/EnhancedSolver/2D-seq/src/progress.h index 314e921..240c279 100644 --- a/EnhancedSolver/2D-seq/src/progress.h +++ b/EnhancedSolver/2D-seq/src/progress.h @@ -10,5 +10,6 @@ extern void initProgress(double); extern void printProgress(double); extern void stopProgress(void); - +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/EnhancedSolver/2D-seq/src/solver-mg.c b/EnhancedSolver/2D-seq/src/solver-mg.c index d1e56b4..395d536 100644 --- a/EnhancedSolver/2D-seq/src/solver-mg.c +++ b/EnhancedSolver/2D-seq/src/solver-mg.c @@ -71,7 +71,38 @@ static void setBoundaryCondition(double* p, int imax, int jmax) } } -static double smooth(Solver* s, double* p, double* rhs, int level, int imax, int jmax) +static void smooth(Solver* s, double* p, double* rhs, int level, int imax, int jmax) +{ + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + P(i, j) -= factor * (RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2)); + + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, int imax, int jmax) { double dx2 = s->grid->dx * s->grid->dx; double dy2 = s->grid->dy * s->grid->dy; @@ -94,7 +125,6 @@ static double smooth(Solver* s, double* p, double* rhs, int level, int imax, int ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - P(i, j) -= (factor * R(i, j)); res += (R(i, j) * R(i, j)); } isw = 3 - isw; @@ -110,7 +140,7 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax, { double res = 0.0; - // coarsest level TODO: Use direct solver? + // coarsest level if (level == COARSEST_LEVEL) { for (int i = 0; i < 5; i++) { smooth(s, p, rhs, level, imax, jmax); @@ -118,17 +148,18 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax, return res; } - // pre-smoothing TODO: Make smoothing steps configurable? - for (int i = 0; i < 5; i++) { + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { smooth(s, p, rhs, level, imax, jmax); if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); } + res = calculateResidual(s, p, rhs, level, imax, jmax); + // restrict restrictMG(s, level, imax, jmax); // MGSolver on residual and error. - // TODO: What if there is a rest? multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, imax / 2, jmax / 2); // prolongate @@ -139,8 +170,8 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax, if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); // post-smoothing - for (int i = 0; i < 5; i++) { - res = smooth(s, p, rhs, level, imax, jmax); + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, imax, jmax); if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); } @@ -154,6 +185,8 @@ void initSolver(Solver* s, Discretization* d, Parameter* p) s->itermax = p->itermax; s->levels = p->levels; s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; int imax = s->grid->imax; int jmax = s->grid->jmax; @@ -176,11 +209,13 @@ void initSolver(Solver* s, Discretization* d, Parameter* p) } } -void solve(Solver* s, double* p, double* rhs) +double solve(Solver* s, double* p, double* rhs) { double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax); #ifdef VERBOSE printf("Residuum: %.6f\n", res); #endif + +return res; } diff --git a/EnhancedSolver/2D-seq/src/solver-rb.c b/EnhancedSolver/2D-seq/src/solver-rb.c index 13d253b..e2716a0 100644 --- a/EnhancedSolver/2D-seq/src/solver-rb.c +++ b/EnhancedSolver/2D-seq/src/solver-rb.c @@ -18,18 +18,9 @@ void initSolver(Solver* s, Discretization* d, Parameter* p) Grid* g = s->grid; int imax = s->grid->imax; int jmax = s->grid->jmax; - - s->totalFluidCells = 0; - for (int j = 0; j < jmax + 2; j++) { - for (int i = 0; i < imax + 2; i++) { - if (gridIsFluid(g, i, j)) { - s->totalFluidCells++; - } - } - } } -void solve(Solver* s, double* p, double* rhs) +double solve(Solver* s, double* p, double* rhs) { int imax = s->grid->imax; int jmax = s->grid->jmax; @@ -55,15 +46,13 @@ void solve(Solver* s, double* p, double* rhs) for (int j = 1; j < jmax + 1; j++) { for (int i = isw; i < imax + 1; i += 2) { - if (gridIsFluid(g, i, j)) { - double r = RHS(i, j) - - ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + - (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * - idy2); - P(i, j) -= (factor * r); - res += (r * r); - } + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); } isw = 3 - isw; } @@ -80,7 +69,7 @@ void solve(Solver* s, double* p, double* rhs) P(imax + 1, j) = P(imax, j); } - res = res / (double)(s->totalFluidCells); + res = res / (double)(imax * jmax); #ifdef DEBUG printf("%d Residuum: %e\n", it, res); #endif @@ -90,4 +79,6 @@ void solve(Solver* s, double* p, double* rhs) #ifdef VERBOSE printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); #endif + + return res; } diff --git a/EnhancedSolver/2D-seq/src/solver-sor.c b/EnhancedSolver/2D-seq/src/solver-sor.c index de5ceb9..2db5452 100644 --- a/EnhancedSolver/2D-seq/src/solver-sor.c +++ b/EnhancedSolver/2D-seq/src/solver-sor.c @@ -18,18 +18,9 @@ void initSolver(Solver* s, Discretization* d, Parameter* p) Grid* g = s->grid; int imax = s->grid->imax; int jmax = s->grid->jmax; - - s->totalFluidCells = 0; - for (int j = 0; j < jmax + 2; j++) { - for (int i = 0; i < imax + 2; i++) { - if (gridIsFluid(g, i, j)) { - s->totalFluidCells++; - } - } - } } -void solve(Solver* s, double* p, double* rhs) +double solve(Solver* s, double* p, double* rhs) { int imax = s->grid->imax; int jmax = s->grid->jmax; @@ -50,14 +41,12 @@ void solve(Solver* s, double* p, double* rhs) for (int j = 1; j < jmax + 1; j++) { for (int i = 1; i < imax + 1; i++) { - if (gridIsFluid(g, i, j)) { - double r = RHS(i, j) - - ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + - (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - P(i, j) -= (factor * r); - res += (r * r); - } + P(i, j) -= (factor * r); + res += (r * r); } } @@ -71,7 +60,7 @@ void solve(Solver* s, double* p, double* rhs) P(imax + 1, j) = P(imax, j); } - res = res / (double)(s->totalFluidCells); + res = res / (double)(imax * jmax); #ifdef DEBUG printf("%d Residuum: %e\n", it, res); #endif @@ -81,4 +70,6 @@ void solve(Solver* s, double* p, double* rhs) #ifdef VERBOSE printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); #endif + + return res; } diff --git a/EnhancedSolver/2D-seq/src/solver.h b/EnhancedSolver/2D-seq/src/solver.h index 8fff905..a8dc072 100644 --- a/EnhancedSolver/2D-seq/src/solver.h +++ b/EnhancedSolver/2D-seq/src/solver.h @@ -16,12 +16,12 @@ typedef struct { /* parameters */ double eps, omega, rho; int itermax; - int levels; + int levels, presmooth, postsmooth; int totalFluidCells; double **r, **e; } Solver; extern void initSolver(Solver*, Discretization*, Parameter*); -extern void solve(Solver*, double*, double*); +extern double solve(Solver*, double*, double*); #endif diff --git a/EnhancedSolver/2D-seq/vis_files/canal_animate.plot b/EnhancedSolver/2D-seq/vis_files/canal_animate.plot new file mode 100644 index 0000000..bf9ce60 --- /dev/null +++ b/EnhancedSolver/2D-seq/vis_files/canal_animate.plot @@ -0,0 +1,10 @@ +unset border; unset tics; unset key; +set term gif animate delay 30 +set output "trace.gif" +set xrange [0:30] +set yrange [0:4] + +do for [ts=0:120] { + plot "particles_".ts.".dat" with points pointtype 7 +} +unset output diff --git a/EnhancedSolver/3D-mpi/Makefile b/EnhancedSolver/3D-mpi/Makefile new file mode 100644 index 0000000..b7fce27 --- /dev/null +++ b/EnhancedSolver/3D-mpi/Makefile @@ -0,0 +1,96 @@ +#======================================================================================= +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/vtkWriter-$(VTK_OUTPUT_FMT).o +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +ifeq ($(VTK_OUTPUT_FMT),mpi) +DEFINES += -D_VTK_WRITER_MPI +endif + +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: sanity-checks $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean vis vis_clean tags info asm format + +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.vtk + @rm -f vtk_files/*.vtk + @rm -f *.png + +clean: vis_clean + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +sanity-checks: +ifeq ($(VTK_OUTPUT_FMT),mpi) +ifeq ($(ENABLE_MPI),false) + $(error VTK_OUTPUT_FMT mpi only supported for ENABLE_MPI true!) +endif +endif + + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/EnhancedSolver/3D-mpi/README.md b/EnhancedSolver/3D-mpi/README.md new file mode 100644 index 0000000..d980b54 --- /dev/null +++ b/EnhancedSolver/3D-mpi/README.md @@ -0,0 +1,78 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about solver, affinity settings, allocation sizes and timer resolution. +For debugging you may want to set the VERBOSE option: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +` + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. + +## Usage + +You have to provide a parameter file describing the problem you want to solve: +``` +./exe-CLANG dcavity.par +``` + +Examples are given in in dcavity (a lid driven cavity test case) and canal (simulating a empty canal). + +You can plot the resulting velocity and pressure fields using gnuplot: +``` +gnuplot vector.plot +``` +and for the pressure: +``` +gnuplot surface.plot +``` diff --git a/EnhancedSolver/3D-mpi/backstep.par b/EnhancedSolver/3D-mpi/backstep.par new file mode 100644 index 0000000..8075448 --- /dev/null +++ b/EnhancedSolver/3D-mpi/backstep.par @@ -0,0 +1,89 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name backstep # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 36000.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 1.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 7.0 # domain size in x-direction +ylength 1.5 # domain size in y-direction +zlength 1.0 # domain size in z-direction +imax 210 # number of interior cells in x-direction +jmax 45 # number of interior cells in y-direction +kmax 30 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 40.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 2000 +startTime 0 +injectTimePeriod 1.0 +writeTimePeriod 0.2 + +x1 0.0 +y1 0.5 +z1 0.0 +x2 0.0 +y2 1.5 +z2 1.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 1 +xCenter 0.0 +yCenter 0.0 +zCenter 0.0 +xRectLength 2.0 +yRectLength 1.0 +zRectLength 2.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/EnhancedSolver/3D-mpi/canal.par b/EnhancedSolver/3D-mpi/canal.par new file mode 100644 index 0000000..59fbe85 --- /dev/null +++ b/EnhancedSolver/3D-mpi/canal.par @@ -0,0 +1,88 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 100.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +zlength 4.0 # domain size in z-direction +imax 200 # number of interior cells in x-direction +jmax 80 # number of interior cells in y-direction +kmax 80 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 60.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +omg 0.52 +omg 1.6 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 500 +startTime 30 +injectTimePeriod 1.0 +writeTimePeriod 0.5 + +x1 0.0 +y1 0.0 +z1 0.0 +x2 4.0 +y2 4.0 +z2 4.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 10.0 +yCenter 2.0 +zCenter 2.0 +xRectLength 8.0 +yRectLength 2.0 +zRectLength 2.0 +circleRadius 1.0 +#=============================================================================== diff --git a/EnhancedSolver/3D-mpi/config.mk b/EnhancedSolver/3D-mpi/config.mk new file mode 100644 index 0000000..0dbe440 --- /dev/null +++ b/EnhancedSolver/3D-mpi/config.mk @@ -0,0 +1,18 @@ +# Supported: GCC, CLANG, ICC +TAG ?= ICC +# Supported: true, false +ENABLE_MPI ?= true +ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg +# Supported: seq, mpi +VTK_OUTPUT_FMT ?= seq + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DDEBUG +#OPTIONS += -DBOUNDCHECK +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/EnhancedSolver/3D-mpi/dcavity.par b/EnhancedSolver/3D-mpi/dcavity.par new file mode 100644 index 0000000..204e007 --- /dev/null +++ b/EnhancedSolver/3D-mpi/dcavity.par @@ -0,0 +1,88 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcLeft 1 # flags for boundary conditions +bcRight 1 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 1000.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +zlength 1.0 # domain size in z-direction +imax 80 # number of interior cells in x-direction +jmax 80 # number of interior cells in y-direction +kmax 80 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 10.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.5 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 0 +injectTimePeriod 1.0 +writeTimePeriod 0.1 + +x1 0.0 +y1 0.1 +z1 0.0 +x2 1.0 +y2 0.1 +z2 1.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 0.5 +yCenter 0.5 +zCenter 0.5 +xRectLength 0.25 +yRectLength 0.25 +zRectLength 0.25 +circleRadius 0.25 +#=============================================================================== diff --git a/EnhancedSolver/3D-mpi/include_CLANG.mk b/EnhancedSolver/3D-mpi/include_CLANG.mk new file mode 100644 index 0000000..3641cad --- /dev/null +++ b/EnhancedSolver/3D-mpi/include_CLANG.mk @@ -0,0 +1,21 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpicc +DEFINES = -D_MPI +else +CC = cc +endif + +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +CFLAGS = -Ofast -std=c17 +LFLAGS = $(OPENMP) -lm +DEFINES += -D_GNU_SOURCE# -DDEBUG +INCLUDES = -I/opt/homebrew/include diff --git a/EnhancedSolver/3D-mpi/include_GCC.mk b/EnhancedSolver/3D-mpi/include_GCC.mk new file mode 100644 index 0000000..cf1e2aa --- /dev/null +++ b/EnhancedSolver/3D-mpi/include_GCC.mk @@ -0,0 +1,20 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpicc +DEFINES = -D_MPI +else +CC = gcc +endif + +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES += -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/EnhancedSolver/3D-mpi/include_ICC.mk b/EnhancedSolver/3D-mpi/include_ICC.mk new file mode 100644 index 0000000..1ffac3a --- /dev/null +++ b/EnhancedSolver/3D-mpi/include_ICC.mk @@ -0,0 +1,20 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpiicc +DEFINES = -D_MPI +else +CC = icc +endif + +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -qopt-zmm-usage=high -std=c99 $(OPENMP) -xHost +LFLAGS = $(OPENMP) +DEFINES += -D_GNU_SOURCE# -DDEBUG +INCLUDES = +LIBS = diff --git a/EnhancedSolver/3D-mpi/karman.par b/EnhancedSolver/3D-mpi/karman.par new file mode 100644 index 0000000..a74bf2b --- /dev/null +++ b/EnhancedSolver/3D-mpi/karman.par @@ -0,0 +1,89 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name karman # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 5050.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 8.0 # domain size in y-direction +zlength 8.0 # domain size in z-direction +imax 200 # number of interior cells in x-direction +jmax 80 # number of interior cells in y-direction +kmax 80 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 200.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 200 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.75 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 1000 +startTime 50 +injectTimePeriod 1.0 +writeTimePeriod 1.0 + +x1 0.0 +y1 3.6 +z1 3.6 +x2 0.0 +y2 4.7 +z2 4.7 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 2 +xCenter 5.0 +yCenter 4.0 +zCenter 4.0 +xRectLength 8.0 +yRectLength 2.0 +zRectLength 2.0 +circleRadius 2.0 + +#=============================================================================== diff --git a/EnhancedSolver/3D-mpi/residual.plot b/EnhancedSolver/3D-mpi/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/EnhancedSolver/3D-mpi/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/EnhancedSolver/3D-mpi/src/allocate.c b/EnhancedSolver/3D-mpi/src/allocate.c new file mode 100644 index 0000000..cf2efd6 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/allocate.c @@ -0,0 +1,38 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/EnhancedSolver/3D-mpi/src/allocate.h b/EnhancedSolver/3D-mpi/src/allocate.h new file mode 100644 index 0000000..77f4ba0 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(size_t alignment, size_t bytesize); + +#endif diff --git a/EnhancedSolver/3D-mpi/src/comm.c b/EnhancedSolver/3D-mpi/src/comm.c new file mode 100644 index 0000000..df9f795 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/comm.c @@ -0,0 +1,647 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "allocate.h" +#include "comm.h" + +#if defined(_MPI) +// subroutines local to this module +static int sizeOfRank(int rank, int size, int N) +{ + return N / size + ((N % size > rank) ? 1 : 0); +} + +static void setupCommunication(Comm* c, Direction direction, int layer) +{ + int imaxLocal = c->imaxLocal; + int jmaxLocal = c->jmaxLocal; + int kmaxLocal = c->kmaxLocal; + + size_t dblsize = sizeof(double); + int sizes[NDIMS]; + int subSizes[NDIMS]; + int starts[NDIMS]; + int offset = 0; + + sizes[IDIM] = imaxLocal + 2; + sizes[JDIM] = jmaxLocal + 2; + sizes[KDIM] = kmaxLocal + 2; + + if (layer == HALO) { + offset = 1; + } + + switch (direction) { + case LEFT: + subSizes[IDIM] = 1; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = 1 - offset; + starts[JDIM] = 1; + starts[KDIM] = 1; + break; + case RIGHT: + subSizes[IDIM] = 1; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = imaxLocal + offset; + starts[JDIM] = 1; + starts[KDIM] = 1; + break; + case BOTTOM: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = 1; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = 1; + starts[JDIM] = 1 - offset; + starts[KDIM] = 1; + break; + case TOP: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = 1; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = 1; + starts[JDIM] = jmaxLocal + offset; + starts[KDIM] = 1; + break; + case FRONT: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = 1; + starts[IDIM] = 1; + starts[JDIM] = 1; + starts[KDIM] = 1 - offset; + break; + case BACK: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = 1; + starts[IDIM] = 1; + starts[JDIM] = 1; + starts[KDIM] = kmaxLocal + offset; + break; + case NDIRS: + printf("ERROR!\n"); + break; + } + + if (layer == HALO) { + MPI_Type_create_subarray(NDIMS, + sizes, + subSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &c->rbufferTypes[direction]); + MPI_Type_commit(&c->rbufferTypes[direction]); + } else if (layer == BULK) { + MPI_Type_create_subarray(NDIMS, + sizes, + subSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &c->sbufferTypes[direction]); + MPI_Type_commit(&c->sbufferTypes[direction]); + } +} + +static void assembleResult(Comm* c, + double* src, + double* dst, + int imaxLocal[], + int jmaxLocal[], + int kmaxLocal[], + int offset[], + int kmax, + int jmax, + int imax) +{ + int numRequests = 1; + + if (c->rank == 0) { + numRequests = c->size + 1; + } + + MPI_Request requests[numRequests]; + + /* all ranks send their interpolated bulk array */ + MPI_Isend(src, + c->imaxLocal * c->jmaxLocal * c->kmaxLocal, + MPI_DOUBLE, + 0, + 0, + c->comm, + &requests[0]); + + /* rank 0 assembles the subdomains */ + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + MPI_Datatype domainType; + int oldSizes[NDIMS] = { kmax, jmax, imax }; + int newSizes[NDIMS] = { kmaxLocal[i], jmaxLocal[i], imaxLocal[i] }; + int starts[NDIMS] = { offset[i * NDIMS + KDIM], + offset[i * NDIMS + JDIM], + offset[i * NDIMS + IDIM] }; + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} + +// subroutines local to this module +static int sum(int* sizes, int init, int offset, int coord) +{ + int sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} +#endif // defined _MPI + +// exported subroutines +void commReduction(double* v, int op) +{ +#if defined(_MPI) + if (op == MAX) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + } else if (op == SUM) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + } +#endif +} + +int commIsBoundary(Comm* c, Direction direction) +{ +#if defined(_MPI) + switch (direction) { + case LEFT: + return c->coords[ICORD] == 0; + break; + case RIGHT: + return c->coords[ICORD] == (c->dims[ICORD] - 1); + break; + case BOTTOM: + return c->coords[JCORD] == 0; + break; + case TOP: + return c->coords[JCORD] == (c->dims[JCORD] - 1); + break; + case FRONT: + return c->coords[KCORD] == 0; + break; + case BACK: + return c->coords[KCORD] == (c->dims[KCORD] - 1); + break; + case NDIRS: + printf("ERROR!\n"); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ +#if defined(_MPI) + int counts[6] = { 1, 1, 1, 1, 1, 1 }; + MPI_Aint displs[6] = { 0, 0, 0, 0, 0, 0 }; + + MPI_Neighbor_alltoallw(grid, + counts, + displs, + c->sbufferTypes, + grid, + counts, + displs, + c->rbufferTypes, + c->comm); +#endif +} + +void commShift(Comm* c, double* f, double* g, double* h) +{ +#if defined(_MPI) + MPI_Request requests[6] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + MPI_Irecv(g, + 1, + c->rbufferTypes[BOTTOM], + c->neighbours[BOTTOM], + 0, + c->comm, + &requests[0]); + + /* send ghost cells to top neighbor */ + MPI_Isend(g, 1, c->sbufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]); + + /* shift F */ + /* receive ghost cells from left neighbor */ + MPI_Irecv(f, 1, c->rbufferTypes[LEFT], c->neighbours[LEFT], 1, c->comm, &requests[2]); + + /* send ghost cells to right neighbor */ + MPI_Isend(f, + 1, + c->sbufferTypes[RIGHT], + c->neighbours[RIGHT], + 1, + c->comm, + &requests[3]); + + /* shift H */ + /* receive ghost cells from front neighbor */ + MPI_Irecv(h, + 1, + c->rbufferTypes[FRONT], + c->neighbours[FRONT], + 2, + c->comm, + &requests[4]); + + /* send ghost cells to back neighbor */ + MPI_Isend(h, 1, c->sbufferTypes[BACK], c->neighbours[BACK], 2, c->comm, &requests[5]); + + MPI_Waitall(6, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax) +{ +#if defined(_MPI) + int sum = 0; + + for (int i = 0; i < c->coords[ICORD]; i++) { + sum += sizeOfRank(i, c->dims[ICORD], imax); + } + offsets[IDIM] = sum; + sum = 0; + + for (int i = 0; i < c->coords[JCORD]; i++) { + sum += sizeOfRank(i, c->dims[JCORD], jmax); + } + offsets[JDIM] = sum; + sum = 0; + + for (int i = 0; i < c->coords[KCORD]; i++) { + sum += sizeOfRank(i, c->dims[KCORD], kmax); + } + offsets[KDIM] = sum; +#endif +} + +#define G(v, i, j, k) \ + v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* wg, + double* pg, + double* u, + double* v, + double* w, + double* p, + int kmax, + int jmax, + int imax) +{ + int imaxLocal = c->imaxLocal; + int jmaxLocal = c->jmaxLocal; + int kmaxLocal = c->kmaxLocal; +#if defined(_MPI) + int offset[c->size * NDIMS]; + int imaxLocalAll[c->size]; + int jmaxLocalAll[c->size]; + int kmaxLocalAll[c->size]; + + MPI_Gather(&imaxLocal, 1, MPI_INT, imaxLocalAll, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&jmaxLocal, 1, MPI_INT, jmaxLocalAll, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&kmaxLocal, 1, MPI_INT, kmaxLocalAll, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + int coords[NCORDS]; + MPI_Cart_coords(c->comm, i, NDIMS, coords); + offset[i * NDIMS + IDIM] = sum(imaxLocalAll, + i, + c->dims[IDIM] * c->dims[JDIM], + coords[ICORD]); + offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, i, c->dims[IDIM], coords[JCORD]); + offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, i, 1, coords[KCORD]); + + printf("Rank: %d, Coords(k,j,i): %d %d %d, Size(k,j,i): %d %d %d, " + "Offset(k,j,i): %d %d %d\n", + i, + coords[KCORD], + coords[JCORD], + coords[ICORD], + kmaxLocalAll[i], + jmaxLocalAll[i], + imaxLocalAll[i], + offset[i * NDIMS + KDIM], + offset[i * NDIMS + JDIM], + offset[i * NDIMS + IDIM]); + } + } + + size_t bytesize = imaxLocal * jmaxLocal * kmaxLocal * sizeof(double); + double* tmp = allocate(64, bytesize); + int idx = 0; + + /* collect P */ + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = G(p, i, j, k); + } + } + } + + assembleResult(c, + tmp, + pg, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + /* collect U */ + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(u, i, j, k) + G(u, i - 1, j, k)) / 2.0; + } + } + } + + assembleResult(c, + tmp, + ug, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + /* collect V */ + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(v, i, j, k) + G(v, i, j - 1, k)) / 2.0; + } + } + } + + assembleResult(c, + tmp, + vg, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + /* collect W */ + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(w, i, j, k) + G(w, i, j, k - 1)) / 2.0; + } + } + } + + assembleResult(c, + tmp, + wg, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + free(tmp); +#else + int idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + pg[idx++] = G(p, i, j, k); + } + } + } + + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + ug[idx++] = (G(u, i, j, k) + G(u, i - 1, j, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + vg[idx++] = (G(v, i, j, k) + G(v, i, j - 1, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + wg[idx++] = (G(w, i, j, k) + G(w, i, j, k - 1)) / 2.0; + } + } + } +#endif +} + +void commPrintConfig(Comm* c) +{ +#if defined(_MPI) + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + if (commIsMaster(c)) { + printf("Communication setup:\n"); + } + + for (int i = 0; i < c->size; i++) { + if (i == c->rank) { + printf("\tRank %d of %d\n", c->rank, c->size); + printf("\tNeighbours (front, back, bottom, top, left, right): %d, %d, %d, " + "%d, %d, %d\n", + c->neighbours[FRONT], + c->neighbours[BACK], + c->neighbours[BOTTOM], + c->neighbours[TOP], + c->neighbours[LEFT], + c->neighbours[RIGHT]); + printf("\tCoordinates (k,j,i) %d %d %d\n", + c->coords[KCORD], + c->coords[JCORD], + c->coords[ICORD]); + printf("\tLocal domain size (k,j,i) %dx%dx%d\n", + c->kmaxLocal, + c->jmaxLocal, + c->imaxLocal); + fflush(stdout); + } + } + MPI_Barrier(MPI_COMM_WORLD); +#endif +} + +void commInit(Comm* c, int argc, char** argv) +{ +#if defined(_MPI) + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank)); + MPI_Comm_size(MPI_COMM_WORLD, &(c->size)); +#else + c->rank = 0; + c->size = 1; +#endif +} + +void commPartition(Comm* c, int kmax, int jmax, int imax) +{ +#if defined(_MPI) + int dims[NDIMS] = { 0, 0, 0 }; + int periods[NDIMS] = { 0, 0, 0 }; + MPI_Dims_create(c->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NCORDS, dims, periods, 0, &c->comm); + MPI_Cart_shift(c->comm, ICORD, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]); + MPI_Cart_shift(c->comm, JCORD, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]); + MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]); + MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords); + + c->imaxLocal = sizeOfRank(c->coords[KDIM], dims[ICORD], imax); + c->jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JCORD], jmax); + c->kmaxLocal = sizeOfRank(c->coords[IDIM], dims[KCORD], kmax); + + // setup buffer types for communication + setupCommunication(c, LEFT, BULK); + setupCommunication(c, LEFT, HALO); + setupCommunication(c, RIGHT, BULK); + setupCommunication(c, RIGHT, HALO); + setupCommunication(c, BOTTOM, BULK); + setupCommunication(c, BOTTOM, HALO); + setupCommunication(c, TOP, BULK); + setupCommunication(c, TOP, HALO); + setupCommunication(c, FRONT, BULK); + setupCommunication(c, FRONT, HALO); + setupCommunication(c, BACK, BULK); + setupCommunication(c, BACK, HALO); +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; + c->kmaxLocal = kmax; +#endif +} + +void commFinalize(Comm* c) +{ +#if defined(_MPI) + for (int i = 0; i < NDIRS; i++) { + MPI_Type_free(&c->sbufferTypes[i]); + MPI_Type_free(&c->rbufferTypes[i]); + } + + MPI_Finalize(); +#endif +} + +void commUpdateDatatypes( + Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal) +{ +#if defined _MPI + + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + newcomm->kmaxLocal = kmaxLocal / 2; + + setupCommunication(newcomm, LEFT, BULK); + setupCommunication(newcomm, LEFT, HALO); + setupCommunication(newcomm, RIGHT, BULK); + setupCommunication(newcomm, RIGHT, HALO); + setupCommunication(newcomm, BOTTOM, BULK); + setupCommunication(newcomm, BOTTOM, HALO); + setupCommunication(newcomm, TOP, BULK); + setupCommunication(newcomm, TOP, HALO); + setupCommunication(newcomm, FRONT, BULK); + setupCommunication(newcomm, FRONT, HALO); + setupCommunication(newcomm, BACK, BULK); + setupCommunication(newcomm, BACK, HALO); +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; + newcomm->kmaxLocal = kmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/EnhancedSolver/3D-mpi/src/comm.h b/EnhancedSolver/3D-mpi/src/comm.h new file mode 100644 index 0000000..58ed4d7 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/comm.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __COMM_H_ +#define __COMM_H_ +#if defined(_MPI) +#include +#endif +/* + * Spatial directions: + * ICORD (0) from 0 (LEFT) to imax (RIGHT) + * JCORD (1) from 0 (BOTTOM) to jmax (TOP) + * KCORD (2) from 0 (FRONT) to kmax (BACK) + * All derived Subarray types are in C ordering + * with indices KDIM (0), JDIM(1) and IDIM(2) + * */ +typedef enum direction { LEFT = 0, RIGHT, BOTTOM, TOP, FRONT, BACK, NDIRS } Direction; +typedef enum coordinates { ICORD = 0, JCORD, KCORD, NCORDS } Coordinates; +typedef enum dimension { KDIM = 0, JDIM, IDIM, NDIMS } Dimension; +enum layer { HALO = 0, BULK }; +enum op { MAX = 0, SUM }; + + +typedef struct { + int rank; + int size; +#if defined(_MPI) + MPI_Comm comm; + MPI_Datatype sbufferTypes[NDIRS]; + MPI_Datatype rbufferTypes[NDIRS]; +#endif + int neighbours[NDIRS]; + int coords[NDIMS], dims[NDIMS]; + int imaxLocal, jmaxLocal, kmaxLocal; +} Comm; + +extern void commInit(Comm* c, int argc, char** argv); +extern void commPartition(Comm* c, int kmax, int jmax, int imax); +extern void commFinalize(Comm* comm); +extern void commPrintConfig(Comm*); +extern void commExchange(Comm*, double*); +extern void commShift(Comm* c, double* f, double* g, double* h); +extern void commReduction(double* v, int op); +extern int commIsBoundary(Comm* c, Direction direction); +extern void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax); +extern void commFreeCommunicator(Comm* comm); +extern void commUpdateDatatypes( + Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal); +extern void commCollectResult(Comm* c, + double* ug, + double* vg, + double* wg, + double* pg, + double* u, + double* v, + double* w, + double* p, + int kmax, + int jmax, + int imax); + +static inline int commIsMaster(Comm* c) { return c->rank == 0; } +#endif // __COMM_H_ diff --git a/EnhancedSolver/3D-mpi/src/discretization.c b/EnhancedSolver/3D-mpi/src/discretization.c new file mode 100644 index 0000000..c77408a --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/discretization.c @@ -0,0 +1,1356 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "allocate.h" +#include "discretization.h" +#include "parameter.h" +#include "util.h" + +static double distance( + double i, double j, double k, double iCenter, double jCenter, double kCenter) +{ + return sqrt(pow(iCenter - i, 2) + pow(jCenter - j, 2) + pow(kCenter - k, 2) * 1.0); +} + +static double sumOffset(double* sizes, int init, int offset, int coord) +{ + double sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +static void printConfig(Discretization* d) +{ + if (commIsMaster(&d->comm)) { + printf("Parameters for #%s#\n", d->problem); + printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n", + d->bcLeft, + d->bcRight, + d->bcBottom, + d->bcTop, + d->bcFront, + d->bcBack); + printf("\tReynolds number: %.2f\n", d->re); + printf("\tGx Gy: %.2f %.2f %.2f\n", d->gx, d->gy, d->gz); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + d->grid.xlength, + d->grid.ylength, + d->grid.zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", + d->grid.imax, + d->grid.jmax, + d->grid.kmax); + printf("\tCell size (dx, dy, dz): %f, %f, %f\n", + d->grid.dx, + d->grid.dy, + d->grid.dz); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te); + printf("\tdt bound: %.6f\n", d->dtBound); + printf("\tTau factor: %.2f\n", d->tau); + printf("Iterative parameters:\n"); + printf("\tMax iterations: %d\n", d->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", d->eps); + printf("\tgamma factor: %f\n", d->gamma); + printf("\tomega (SOR relaxation): %f\n", d->omega); + } + commPrintConfig(&d->comm); +} + +void printGrid(Discretization* d, double* s) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + for (int k = 0; k < kmaxLocal + 2; k++) { + printf("K : %02d:\n", k); + for (int j = 0; j < jmaxLocal + 2; j++) { + printf("J : %02d: ", j); + for (int i = 0; i problem = params->name; + d->bcLeft = params->bcLeft; + d->bcRight = params->bcRight; + d->bcBottom = params->bcBottom; + d->bcTop = params->bcTop; + d->bcFront = params->bcFront; + d->bcBack = params->bcBack; + + d->grid.imax = params->imax; + d->grid.jmax = params->jmax; + d->grid.kmax = params->kmax; + d->grid.xlength = params->xlength; + d->grid.ylength = params->ylength; + d->grid.zlength = params->zlength; + d->grid.dx = params->xlength / params->imax; + d->grid.dy = params->ylength / params->jmax; + d->grid.dz = params->zlength / params->kmax; + + d->eps = params->eps; + d->omega = params->omg; + d->itermax = params->itermax; + d->re = params->re; + d->gx = params->gx; + d->gy = params->gy; + d->gz = params->gz; + d->dt = params->dt; + d->te = params->te; + d->tau = params->tau; + d->gamma = params->gamma; + + /* allocate arrays */ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + size_t size = (imaxLocal + 2) * (jmaxLocal + 2) * (kmaxLocal + 2); + + d->u = allocate(64, size * sizeof(double)); + d->v = allocate(64, size * sizeof(double)); + d->w = allocate(64, size * sizeof(double)); + d->p = allocate(64, size * sizeof(double)); + d->rhs = allocate(64, size * sizeof(double)); + d->f = allocate(64, size * sizeof(double)); + d->g = allocate(64, size * sizeof(double)); + d->h = allocate(64, size * sizeof(double)); + d->s = allocate(64, size * sizeof(double)); + + for (int i = 0; i < size; i++) { + d->u[i] = params->u_init; + d->v[i] = params->v_init; + d->w[i] = params->w_init; + d->p[i] = params->p_init; + d->rhs[i] = 0.0; + d->f[i] = 0.0; + d->g[i] = 0.0; + d->h[i] = 0.0; + d->s[i] = FLUID; + } + + double dx = d->grid.dx; + double dy = d->grid.dy; + double dz = d->grid.dz; + + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz); + d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; + + double xCenter = 0, yCenter = 0, zCenter = 0, radius = 0; + double x1 = 0, x2 = 0, y1 = 0, y2 = 0, z1 = 0, z2 = 0; + int iOffset = 0, jOffset = 0, kOffset = 0; + + d->xLocal = d->comm.imaxLocal * d->grid.dx; + d->yLocal = d->comm.jmaxLocal * d->grid.dy; + d->zLocal = d->comm.kmaxLocal * d->grid.dz; + +#ifdef _MPI + double xLocal[d->comm.size]; + double yLocal[d->comm.size]; + double zLocal[d->comm.size]; + + MPI_Allgather(&d->xLocal, 1, MPI_DOUBLE, xLocal, 1, MPI_DOUBLE, d->comm.comm); + MPI_Allgather(&d->yLocal, 1, MPI_DOUBLE, yLocal, 1, MPI_DOUBLE, d->comm.comm); + MPI_Allgather(&d->zLocal, 1, MPI_DOUBLE, zLocal, 1, MPI_DOUBLE, d->comm.comm); + + d->xOffset = sumOffset(xLocal, + d->comm.rank, + (d->comm.dims[1] * d->comm.dims[2]), + d->comm.coords[0]); + d->yOffset = sumOffset(yLocal, d->comm.rank, d->comm.dims[2], d->comm.coords[1]); + d->zOffset = sumOffset(zLocal, d->comm.rank, 1, d->comm.coords[2]); + d->xOffsetEnd = d->xOffset + d->xLocal; + d->yOffsetEnd = d->yOffset + d->yLocal; + d->zOffsetEnd = d->zOffset + d->zLocal; + +#else + d->xOffset = 0; + d->yOffset = 0; + d->zOffset = 0; + d->xOffsetEnd = d->xOffset + d->xLocal; + d->yOffsetEnd = d->yOffset + d->yLocal; + d->zOffsetEnd = d->zOffset + d->zLocal; +#endif + + iOffset = d->xOffset / dx; + jOffset = d->yOffset / dy; + kOffset = d->zOffset / dz; + + double* s = d->s; + + switch (params->shape) { + case NOSHAPE: + break; + case RECT: + x1 = params->xCenter - params->xRectLength / 2; + x2 = params->xCenter + params->xRectLength / 2; + y1 = params->yCenter - params->yRectLength / 2; + y2 = params->yCenter + params->yRectLength / 2; + z1 = params->zCenter - params->zRectLength / 2; + z2 = params->zCenter + params->zRectLength / 2; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if ((x1 <= ((i + iOffset) * dx)) && (((i + iOffset) * dx) <= x2) && + (y1 <= ((j + jOffset) * dy)) && (((j + jOffset) * dy) <= y2) && + ((z1 <= ((k + kOffset) * dz)) && (((k + kOffset) * dz) <= z2))) { + S(i, j, k) = OBSTACLE; + } + } + } + } + + break; + case CIRCLE: + + xCenter = params->xCenter; + yCenter = params->yCenter; + zCenter = params->zCenter; + radius = params->circleRadius; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if (distance(((i + iOffset) * dx), + ((j + jOffset) * dy), + ((k + kOffset) * dz), + xCenter, + yCenter, + zCenter) <= radius) { + S(i, j, k) = OBSTACLE; + } + } + } + } + break; + default: + break; + } + +#ifdef _MPI + commExchange(&d->comm, s); +#endif + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + + /* Assigning enum values to Corners */ + if (S(i - 1, j + 1, k - 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i + 1, j - 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPLEFTCORNER; // + } + if (S(i + 1, j + 1, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j - 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPRIGHTCORNER; // + } + if (S(i - 1, j - 1, k - 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i + 1, j + 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMLEFTCORNER; // + } + if (S(i + 1, j - 1, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j + 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMRIGHTCORNER; // + } + if (S(i - 1, j + 1, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i + 1, j - 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPLEFTCORNER; // + } + if (S(i + 1, j + 1, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i - 1, j - 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPRIGHTCORNER; + } + if (S(i - 1, j - 1, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i + 1, j + 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMLEFTCORNER; + } + if (S(i + 1, j - 1, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i - 1, j + 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMRIGHTCORNER; + } + /* Assigning enum values to Lines */ + if (S(i - 1, j, k - 1) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j, k) == FLUID && S(i + 1, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTLEFTLINE; + } + if (S(i + 1, j, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j, k - 1) == FLUID && S(i - 1, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTRIGHTLINE; + } + if (S(i, j + 1, k - 1) == FLUID && S(i, j + 1, k) == FLUID && + S(i, j, k - 1) == FLUID && S(i, j - 1, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPLINE; + } + if (S(i, j - 1, k - 1) == FLUID && S(i, j, k - 1) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j + 1, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMLINE; + } + if (S(i - 1, j + 1, k) == FLUID && S(i, j + 1, k) == FLUID && + S(i - 1, j, k) == FLUID && S(i + 1, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDTOPLEFTLINE; + } + if (S(i + 1, j + 1, k) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i - 1, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDTOPRIGHTLINE; + } + if (S(i - 1, j - 1, k) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i + 1, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDBOTTOMLEFTLINE; + } + if (S(i + 1, j - 1, k) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i - 1, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDBOTTOMRIGHTLINE; + } + if (S(i - 1, j, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i + 1, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKLEFTLINE; + } + if (S(i + 1, j, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i - 1, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKRIGHTLINE; + } + if (S(i, j + 1, k + 1) == FLUID && S(i, j + 1, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i, j - 1, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPLINE; + } + if (S(i, j - 1, k + 1) == FLUID && S(i, j - 1, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i, j + 1, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMLINE; + } + /* Assigning enum values to Faces */ + if (S(i, j, k - 1) == FLUID && S(i, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTFACE; // + } + if (S(i, j, k + 1) == FLUID && S(i, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKFACE; // + } + if (S(i, j - 1, k) == FLUID && S(i, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BOTTOMFACE; // + } + if (S(i, j + 1, k) == FLUID && S(i, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = TOPFACE; // + } + if (S(i - 1, j, k) == FLUID && S(i + 1, j, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = LEFTFACE; // + } + if (S(i + 1, j, k) == FLUID && S(i - 1, j, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = RIGHTFACE; // + } + } + } + } + +#ifdef VERBOSE + printConfig(d); +#endif /* VERBOSE */ +} + +void setBoundaryConditions(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double* u = d->u; + double* v = d->v; + double* w = d->w; + + if (commIsBoundary(&d->comm, TOP)) { + switch (d->bcTop) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = -U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = 0.0; + W(i, jmaxLocal + 1, k) = -W(i, jmaxLocal, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = 0.0; + W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = V(i, jmaxLocal - 1, k); + W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, BOTTOM)) { + switch (d->bcBottom) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = -U(i, 1, k); + V(i, 0, k) = 0.0; + W(i, 0, k) = -W(i, 1, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = 0.0; + W(i, 0, k) = W(i, 1, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = V(i, 1, k); + W(i, 0, k) = W(i, 1, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, LEFT)) { + switch (d->bcLeft) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = -V(1, j, k); + W(0, j, k) = -W(1, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = U(1, j, k); + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, RIGHT)) { + switch (d->bcRight) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = 0.0; + V(imaxLocal + 1, j, k) = -V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = -W(imaxLocal, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = 0.0; + V(imaxLocal + 1, j, k) = V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = W(imaxLocal, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = U(imaxLocal - 1, j, k); + V(imaxLocal + 1, j, k) = V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = W(imaxLocal, j, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, FRONT)) { + switch (d->bcFront) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = -U(i, j, 1); + V(i, j, 0) = -V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = W(i, j, 1); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, BACK)) { + switch (d->bcBack) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = -U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = -V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = W(i, j, kmaxLocal - 1); + } + } + break; + case PERIODIC: + break; + } + } +} + +void computeRHS(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double idx = 1.0 / d->grid.dx; + double idy = 1.0 / d->grid.dy; + double idz = 1.0 / d->grid.dz; + double idt = 1.0 / d->dt; + + double* rhs = d->rhs; + double* f = d->f; + double* g = d->g; + double* h = d->h; + + commShift(&d->comm, f, g, h); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx + + (G(i, j, k) - G(i, j - 1, k)) * idy + + (H(i, j, k) - H(i, j, k - 1)) * idz) * + idt; + } + } + } +} + +void setSpecialBoundaryCondition(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double* u = d->u; + + if (strcmp(d->problem, "dcavity") == 0) { + if (commIsBoundary(&d->comm, TOP)) { + for (int k = 1; k < kmaxLocal; k++) { + for (int i = 1; i < imaxLocal; i++) { + U(i, jmaxLocal + 1, k) = 2.0 - U(i, jmaxLocal, k); + } + } + } + } else if (strcmp(d->problem, "canal") == 0) { + if (commIsBoundary(&d->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 2.0; + } + } + } + } else if (strcmp(d->problem, "backstep") == 0) { + if (commIsBoundary(&d->comm, LEFT)) { + double* s = d->s; + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + if (S(1, j, k) == FLUID) U(0, j, k) = 1.0; + else { + U(0, j, k) = 0.0; + U(1, j, k) = 0.0; + } + } + } + } + } else if (strcmp(d->problem, "karman") == 0) { + if (commIsBoundary(&d->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(1, j, k) = 1.0; + } + } + } + } +} + +static double maxElement(Discretization* d, double* m) +{ + int size = (d->comm.imaxLocal + 2) * (d->comm.jmaxLocal + 2) * + (d->comm.kmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + commReduction(&maxval, MAX); + return maxval; +} + +void normalizePressure(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double* p = d->p; + double avgP = 0.0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + avgP += P(i, j, k); + } + } + } + commReduction(&avgP, SUM); + avgP /= (d->grid.imax * d->grid.jmax * d->grid.kmax); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, k) = P(i, j, k) - avgP; + } + } + } +} + +void computeTimestep(Discretization* d) +{ + double dt = d->dtBound; + double dx = d->grid.dx; + double dy = d->grid.dy; + double dz = d->grid.dz; + + double umax = maxElement(d, d->u); + double vmax = maxElement(d, d->v); + double wmax = maxElement(d, d->w); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + if (wmax > 0) { + dt = (dt > dz / wmax) ? dz / wmax : dt; + } + + d->dt = dt * d->tau; +} + +void setObjectBoundaryCondition(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* s = d->s; + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + switch ((int)S(i, j, k)) { + case TOPFACE: + U(i, j, k) = -U(i, j + 1, k); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + ; + break; + case BOTTOMFACE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case LEFTFACE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i - 1, j, k); + break; + case RIGHTFACE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i + 1, j, k); + break; + case FRONTFACE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = 0.0; + break; + case BACKFACE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = 0.0; + break; + case FRONTLEFTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = -W(i - 1, j, k); + break; + case FRONTRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = -W(i + 1, j, k); + break; + case FRONTTOPLINE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTBOTTOMLINE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case MIDTOPLEFTLINE: + U(i, j, k) = -U(i, j + 1, k); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = 0.0; + break; + case MIDTOPRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = 0.0; + U(i - 1, j, k) = -U(i - 1, j + 1, k); + V(i, j - 1, k) = -V(i + 1, j - 1, k); + W(i, j, k) = 0.0; + break; + case MIDBOTTOMLEFTLINE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = 0.0; + break; + case MIDBOTTOMRIGHTLINE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = 0.0; + break; + case BACKLEFTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = -W(i - 1, j, k); + break; + case BACKRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = -W(i + 1, j, k); + break; + case BACKTOPLINE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKBOTTOMLINE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case FRONTTOPLEFTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTTOPRIGHTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTBOTTOMLEFTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case FRONTBOTTOMRIGHTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case BACKTOPLEFTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKTOPRIGHTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKBOTTOMLEFTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case BACKBOTTOMRIGHTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + } + } + } + } +} + +void computeFG(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* f = d->f; + double* g = d->g; + double* h = d->h; + double* s = d->s; + + double gx = d->gx; + double gy = d->gy; + double gz = d->gz; + double dt = d->dt; + + double gamma = d->gamma; + double inverseRe = 1.0 / d->re; + double inverseDx = 1.0 / d->grid.dx; + double inverseDy = 1.0 / d->grid.dy; + double inverseDz = 1.0 / d->grid.dz; + double du2dx, dv2dy, dw2dz; + double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz; + double du2dx2, du2dy2, du2dz2; + double dv2dx2, dv2dy2, dv2dz2; + double dw2dx2, dw2dy2, dw2dz2; + + commExchange(&d->comm, u); + commExchange(&d->comm, v); + commExchange(&d->comm, w); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if (S(i, j, k) == FLUID) { + du2dx = inverseDx * 0.25 * + ((U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) + U(i + 1, j, k)) - + (U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) + U(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) - U(i + 1, j, k)) + + fabs(U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) - U(i - 1, j, k))); + + duvdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) + U(i, j + 1, k)) - + (V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) + U(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) - U(i, j + 1, k)) + + fabs(V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) - U(i, j - 1, k))); + + duwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) + U(i, j, k + 1)) - + (W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) + U(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) - U(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) - U(i, j, k - 1))); + + du2dx2 = inverseDx * inverseDx * + (U(i + 1, j, k) - 2.0 * U(i, j, k) + U(i - 1, j, k)); + du2dy2 = inverseDy * inverseDy * + (U(i, j + 1, k) - 2.0 * U(i, j, k) + U(i, j - 1, k)); + du2dz2 = inverseDz * inverseDz * + (U(i, j, k + 1) - 2.0 * U(i, j, k) + U(i, j, k - 1)); + F(i, j, k) = U(i, j, k) + + dt * (inverseRe * (du2dx2 + du2dy2 + du2dz2) - du2dx - + duvdy - duwdz + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) + V(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) + V(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) - V(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) - V(i - 1, j, k))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) + V(i, j + 1, k)) - + (V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) + V(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) - V(i, j + 1, k)) + + fabs(V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) - V(i, j - 1, k))); + + dvwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) + V(i, j, k + 1)) - + (W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) + V(i, j, k + 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) - V(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) - V(i, j, k + 1))); + + dv2dx2 = inverseDx * inverseDx * + (V(i + 1, j, k) - 2.0 * V(i, j, k) + V(i - 1, j, k)); + dv2dy2 = inverseDy * inverseDy * + (V(i, j + 1, k) - 2.0 * V(i, j, k) + V(i, j - 1, k)); + dv2dz2 = inverseDz * inverseDz * + (V(i, j, k + 1) - 2.0 * V(i, j, k) + V(i, j, k - 1)); + G(i, j, k) = V(i, j, k) + + dt * (inverseRe * (dv2dx2 + dv2dy2 + dv2dz2) - duvdx - + dv2dy - dvwdz + gy); + + duwdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) + W(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) + W(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) - W(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) - W(i - 1, j, k))); + + dvwdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) + W(i, j + 1, k)) - + (V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) + W(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) - W(i, j + 1, k)) + + fabs(V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) - W(i, j - 1, k))); + + dw2dz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) + W(i, j, k + 1)) - + (W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) + W(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) - W(i, j, k + 1)) + + fabs(W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) - W(i, j, k - 1))); + + dw2dx2 = inverseDx * inverseDx * + (W(i + 1, j, k) - 2.0 * W(i, j, k) + W(i - 1, j, k)); + dw2dy2 = inverseDy * inverseDy * + (W(i, j + 1, k) - 2.0 * W(i, j, k) + W(i, j - 1, k)); + dw2dz2 = inverseDz * inverseDz * + (W(i, j, k + 1) - 2.0 * W(i, j, k) + W(i, j, k - 1)); + H(i, j, k) = W(i, j, k) + + dt * (inverseRe * (dw2dx2 + dw2dy2 + dw2dz2) - duwdx - + dvwdy - dw2dz + gz); + } else { + switch ((int)S(i, j, k)) { + case TOPFACE: + G(i, j, k) = V(i, j, k); + break; + case BOTTOMFACE: + G(i, j, k) = V(i, j, k); + break; + case LEFTFACE: + F(i, j, k) = U(i, j, k); + break; + case RIGHTFACE: + F(i, j, k) = U(i, j, k); + break; + case FRONTFACE: + H(i, j, k) = W(i, j, k); + break; + case BACKFACE: + H(i, j, k) = W(i, j, k); + break; + case FRONTLEFTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTRIGHTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case MIDTOPLEFTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case MIDTOPRIGHTLINE: + F(i, j, k) = U(i, j, k); + G(i, j, k) = V(i, j, k); + break; + case MIDBOTTOMLEFTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case MIDBOTTOMRIGHTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case BACKLEFTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKRIGHTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + } + } + } + } + } + + /* ----------------------------- boundary of F --------------------------- + */ + if (commIsBoundary(&d->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(0, j, k) = U(0, j, k); + } + } + } + + if (commIsBoundary(&d->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(imaxLocal, j, k) = U(imaxLocal, j, k); + } + } + } + + /* ----------------------------- boundary of G --------------------------- + */ + if (commIsBoundary(&d->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, 0, k) = V(i, 0, k); + } + } + } + + if (commIsBoundary(&d->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, jmaxLocal, k) = V(i, jmaxLocal, k); + } + } + } + + /* ----------------------------- boundary of H --------------------------- + */ + if (commIsBoundary(&d->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + H(i, j, 0) = W(i, j, 0); + } + } + } + + if (commIsBoundary(&d->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + H(i, j, kmaxLocal) = W(i, j, kmaxLocal); + } + } + } +} + +void adaptUV(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double* p = d->p; + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* f = d->f; + double* g = d->g; + double* h = d->h; + + double factorX = d->dt / d->grid.dx; + double factorY = d->dt / d->grid.dy; + double factorZ = d->dt / d->grid.dz; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, k) = F(i, j, k) - (P(i + 1, j, k) - P(i, j, k)) * factorX; + V(i, j, k) = G(i, j, k) - (P(i, j + 1, k) - P(i, j, k)) * factorY; + W(i, j, k) = H(i, j, k) - (P(i, j, k + 1) - P(i, j, k)) * factorZ; + } + } + } +} \ No newline at end of file diff --git a/EnhancedSolver/3D-mpi/src/discretization.h b/EnhancedSolver/3D-mpi/src/discretization.h new file mode 100644 index 0000000..c09780c --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/discretization.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ + +#include "comm.h" +#include "grid.h" +#include "parameter.h" +#include "util.h" + +enum OBJECTBOUNDARY { + FLUID = 0, + /* Front Corners */ + FRONTTOPLEFTCORNER, + FRONTTOPRIGHTCORNER, + FRONTBOTTOMLEFTCORNER, + FRONTBOTTOMRIGHTCORNER, + /* Back Corners */ + BACKTOPLEFTCORNER, + BACKTOPRIGHTCORNER, + BACKBOTTOMLEFTCORNER, + BACKBOTTOMRIGHTCORNER, + /* Faces */ + FRONTFACE, + BACKFACE, + LEFTFACE, + RIGHTFACE, + TOPFACE, + BOTTOMFACE, + /* Front Lines remaining after Corners and Faces */ + FRONTLEFTLINE, + FRONTRIGHTLINE, + FRONTTOPLINE, + FRONTBOTTOMLINE, + /* Bottom Lines remaining after Corners and Faces */ + BACKLEFTLINE, + BACKRIGHTLINE, + BACKTOPLINE, + BACKBOTTOMLINE, + /* Mid Lines remaining after Corners and Faces */ + MIDTOPLEFTLINE, + MIDTOPRIGHTLINE, + MIDBOTTOMLEFTLINE, + MIDBOTTOMRIGHTLINE, + /* Local where its an object but not a boundary */ + OBSTACLE, + /*Ghost cells boundary */ + OUTSIDEBOUNDARY +}; + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +enum SHAPE { NOSHAPE = 0, RECT, CIRCLE }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g, *h; + double *u, *v, *w; + double* s; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy, gz; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + double xLocal, yLocal, zLocal, xOffset, yOffset, zOffset, xOffsetEnd, yOffsetEnd, + zOffsetEnd; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + Comm comm; +} Discretization; + +extern void initDiscretization(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setObjectBoundaryCondition(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +extern void printGrid(Discretization*, double*); +#endif diff --git a/EnhancedSolver/3D-mpi/src/grid.h b/EnhancedSolver/3D-mpi/src/grid.h new file mode 100644 index 0000000..0689ebc --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/grid.h @@ -0,0 +1,16 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __GRID_H_ +#define __GRID_H_ + +typedef struct { + double dx, dy, dz; + int imax, jmax, kmax; + double xlength, ylength, zlength; +} Grid; + +#endif // __GRID_H_ diff --git a/EnhancedSolver/3D-mpi/src/likwid-marker.h b/EnhancedSolver/3D-mpi/src/likwid-marker.h new file mode 100644 index 0000000..c3770c0 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/EnhancedSolver/3D-mpi/src/main.c b/EnhancedSolver/3D-mpi/src/main.c new file mode 100644 index 0000000..cfd2bf6 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/main.c @@ -0,0 +1,151 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "allocate.h" +#include "discretization.h" +#include "parameter.h" +#include "particletracing.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" +#include "vtkWriter.h" + +int main(int argc, char** argv) +{ + double timeStart, timeStop; + Parameter p; + Solver s; + Discretization d; + ParticleTracer particletracer; + + commInit(&d.comm, argc, argv); + initParameter(&p); + FILE* fp; + if (commIsMaster(&d.comm)) fp = initResidualWriter(); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(&p, argv[1]); + commPartition(&d.comm, p.kmax, p.jmax, p.imax); + + if (commIsMaster(&d.comm)) { + printParameter(&p); + } + + initDiscretization(&d, &p); + initSolver(&s, &d, &p); + initParticleTracer(&particletracer, &p, &d); + + +#ifndef VERBOSE + initProgress(d.te); +#endif + + double tau = d.tau; + double te = d.te; + double t = 0.0; + int nt = 0; + double res = 0.0; + + timeStart = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + setObjectBoundaryCondition(&d); + computeFG(&d); + computeRHS(&d); + + if (nt % 100 == 0) normalizePressure(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + trace(&particletracer, d.u, d.v, d.w, d.s, t); + + if (commIsMaster(&d.comm)) writeResidual(fp, t, res); + + t += d.dt; + nt++; + +#ifdef VERBOSE + if (commIsMaster(s.comm)) { + printf("TIME %f , TIMESTEP %f\n", t, d.dt); + } +#else + printProgress(t); +#endif + } + timeStop = getTimeStamp(); + +#ifndef VERBOSE + stopProgress(); +#endif + + if (commIsMaster(s.comm)) { + printf("Solution took %.2fs\n", timeStop - timeStart); + } + + timeStart = getTimeStamp(); +#ifdef _VTK_WRITER_MPI + VtkOptions opts = { .grid = s.grid, .comm = s.comm }; + vtkOpen(&opts, s.problem); + vtkScalar(&opts, "pressure", d.p); + vtkVector(&opts, "velocity", (VtkVector) { d.u, d.v, d.w }); + vtkClose(&opts); +#else + + if (commIsMaster(&d.comm)) fclose(fp); + freeParticles(&particletracer); + + double *pg, *ug, *vg, *wg; + + if (commIsMaster(s.comm)) { + size_t bytesize = s.grid->imax * s.grid->jmax * s.grid->kmax * sizeof(double); + + pg = allocate(64, bytesize); + ug = allocate(64, bytesize); + vg = allocate(64, bytesize); + wg = allocate(64, bytesize); + } + + commCollectResult(s.comm, + ug, + vg, + wg, + pg, + d.u, + d.v, + d.w, + d.p, + s.grid->kmax, + s.grid->jmax, + s.grid->imax); + + if (commIsMaster(s.comm)) { + VtkOptions opts = { .grid = s.grid }; + vtkOpen(&opts, s.problem); + vtkScalar(&opts, "pressure", pg); + vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); + vtkClose(&opts); + } + +#endif + + timeStop = getTimeStamp(); + + if (commIsMaster(s.comm)) { + printf("Result output took %.2fs\n", timeStop - timeStart); + } + + commFinalize(s.comm); + return EXIT_SUCCESS; +} diff --git a/EnhancedSolver/3D-mpi/src/parameter.c b/EnhancedSolver/3D-mpi/src/parameter.c new file mode 100644 index 0000000..aa8e485 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/parameter.c @@ -0,0 +1,154 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->zlength = 1.0; + param->imax = 100; + param->jmax = 100; + param->kmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_REAL(zlength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(kmax); + PARSE_INT(itermax); + PARSE_INT(levels); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_REAL(gz); + PARSE_STRING(name); + PARSE_INT(bcLeft); + PARSE_INT(bcRight); + PARSE_INT(bcBottom); + PARSE_INT(bcTop); + PARSE_INT(bcFront); + PARSE_INT(bcBack); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(w_init); + PARSE_REAL(p_init); + + /* Added new particle tracing parameters */ + PARSE_INT(numberOfParticles); + PARSE_REAL(startTime); + PARSE_REAL(injectTimePeriod); + PARSE_REAL(writeTimePeriod); + PARSE_REAL(x1); + PARSE_REAL(y1); + PARSE_REAL(z1); + PARSE_REAL(x2); + PARSE_REAL(y2); + PARSE_REAL(z2); + + /* Added obstacle geometry parameters */ + PARSE_INT(shape); + PARSE_REAL(xCenter); + PARSE_REAL(yCenter); + PARSE_REAL(zCenter); + PARSE_REAL(xRectLength); + PARSE_REAL(yRectLength); + PARSE_REAL(zRectLength); + PARSE_REAL(circleRadius); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d Front:%d " + "Back:%d\n", + param->bcLeft, + param->bcRight, + param->bcBottom, + param->bcTop, + param->bcFront, + param->bcBack); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f W:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->w_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + param->xlength, + param->ylength, + param->zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", param->imax, param->jmax, param->kmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); +} diff --git a/EnhancedSolver/3D-mpi/src/parameter.h b/EnhancedSolver/3D-mpi/src/parameter.h new file mode 100644 index 0000000..553e82a --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/parameter.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + int imax, jmax, kmax; + double xlength, ylength, zlength; + int itermax; + double eps, omg; + double re, tau, gamma; + double te, dt; + double gx, gy, gz; + char* name; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + double u_init, v_init, w_init, p_init; + int levels, presmooth, postsmooth; + + int numberOfParticles; + double startTime, injectTimePeriod, writeTimePeriod; + + double x1, y1, z1, x2, y2, z2; + + int shape; + double xCenter, yCenter, zCenter, xRectLength, yRectLength, zRectLength, circleRadius; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/EnhancedSolver/3D-mpi/src/particletracing.c b/EnhancedSolver/3D-mpi/src/particletracing.c new file mode 100644 index 0000000..86386ef --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/particletracing.c @@ -0,0 +1,749 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "particletracing.h" +#include "solver.h" + +static int ts = 0; +unsigned int seed = 32767; + +#define XOFFSET 0 +#define YOFFSET 1 +#define ZOFFSET 2 +#define XOFFSETEND 3 +#define YOFFSETEND 4 +#define ZOFFSETEND 5 + +static double sum(int* sizes, int size) +{ + double sum = 0; + + for (int i = 0; i < size; ++i) { + sum += sizes[i]; + } + + return sum; +} + +static int maxElement(int* sizes, int size) +{ + + int maxValue = sizes[0]; + + for (size_t i = 1; i < size; ++i) { + if (sizes[i] > maxValue) { + maxValue = sizes[i]; + } + } + return maxValue; +} + +static double sumOffset(double* sizes, int init, int offset, int coord) +{ + double sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +void printParticles(ParticleTracer* particletracer) +{ + for (int i = 0; i < particletracer->totalParticles; ++i) { + printf("Rank : %d Particle position X : %.2f, Y : %.2f, flag : %d, total pt : " + "%d, pointer : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, " + "yOffsetEnd : %.2f\n", + particletracer->rank, + particletracer->particlePool[i].x, + particletracer->particlePool[i].y, + particletracer->particlePool[i].flag, + particletracer->totalParticles, + particletracer->pointer, + particletracer->xOffset, + particletracer->yOffset, + particletracer->xOffsetEnd, + particletracer->yOffsetEnd); + } +} +void injectParticles(ParticleTracer* particletracer, double* restrict s) +{ + compress(particletracer); + particleRandomizer(particletracer); + + double x, y, z; + int imaxLocal = particletracer->imaxLocal; + int jmaxLocal = particletracer->jmaxLocal; + int kmaxLocal = particletracer->kmaxLocal; + for (int i = 0; i < particletracer->numberOfParticles; ++i) { + x = particletracer->linSpaceLine[i].x; + y = particletracer->linSpaceLine[i].y; + z = particletracer->linSpaceLine[i].z; + if (x >= particletracer->xOffset && y >= particletracer->yOffset && + z >= particletracer->zOffset && x <= particletracer->xOffsetEnd && + y <= particletracer->yOffsetEnd && y <= particletracer->zOffsetEnd) { + + particletracer->particlePool[particletracer->pointer].x = x; + particletracer->particlePool[particletracer->pointer].y = y; + particletracer->particlePool[particletracer->pointer].z = z; + + int i = particletracer->particlePool[particletracer->pointer].x / + particletracer->dx; + int j = particletracer->particlePool[particletracer->pointer].y / + particletracer->dy; + int k = particletracer->particlePool[particletracer->pointer].z / + particletracer->dz; + + int iOffset = particletracer->xOffset / particletracer->dx, + jOffset = particletracer->yOffset / particletracer->dy, + kOffset = particletracer->zOffset / particletracer->dz; + + if (S(i - iOffset, j - jOffset, k - kOffset) == FLUID) { + particletracer->particlePool[particletracer->pointer].flag = true; + ++(particletracer->pointer); + ++(particletracer->totalParticles); + } + } + } +} + +void advanceParticles(ParticleTracer* particletracer, + double* restrict u, + double* restrict v, + double* restrict w, + double* restrict s, + double time) +{ + int imax = particletracer->imax; + int jmax = particletracer->jmax; + int kmax = particletracer->kmax; + int imaxLocal = particletracer->imaxLocal; + int jmaxLocal = particletracer->jmaxLocal; + int kmaxLocal = particletracer->kmaxLocal; + + double dx = particletracer->dx; + double dy = particletracer->dy; + double dz = particletracer->dz; + + double xlength = particletracer->xlength; + double ylength = particletracer->ylength; + double zlength = particletracer->zlength; + + Particle buff[particletracer->size][(particletracer->estimatedNumParticles / particletracer->size)]; + memset(buff, 0, sizeof(buff)); + Particle recvbuff[particletracer->size][(particletracer->estimatedNumParticles / particletracer->size)]; + memset(buff, 0, sizeof(recvbuff)); + + int particleBufIndex[particletracer->size], + recvparticleBufIndex[particletracer->size]; + + memset(particleBufIndex, 0, sizeof(particleBufIndex)); + memset(recvparticleBufIndex, 0, sizeof(recvparticleBufIndex)); + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (particletracer->particlePool[i].flag == true) { + double xTemp = particletracer->particlePool[i].x; + double yTemp = particletracer->particlePool[i].y; + double zTemp = particletracer->particlePool[i].z; + + double x = xTemp - particletracer->xOffset; + double y = yTemp - particletracer->yOffset; + double z = zTemp - particletracer->zOffset; + + int iCoord = (int)(x / dx) + 1; + int jCoord = (int)((y + 0.5 * dy) / dy) + 1; + int kCoord = (int)((z + 0.5 * dz) / dz) + 1; + + double x1 = (double)(iCoord - 1) * dx; + double y1 = ((double)(jCoord - 1) - 0.5) * dy; + double z1 = ((double)(kCoord - 1) - 0.5) * dz; + + double x2 = (double)iCoord * dx; + double y2 = ((double)jCoord - 0.5) * dy; + double z2 = ((double)kCoord - 0.5) * dz; + + double u_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * U(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * U(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * U(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * U(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * U(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * U(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * U(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * U(iCoord, jCoord, kCoord)); + + double new_x = (x + particletracer->xOffset) + particletracer->dt * u_n; + particletracer->particlePool[i].x = new_x; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)(y / dy) + 1; + kCoord = (int)((z + 0.5 * dz) / dz) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = (double)(jCoord - 1) * dy; + z1 = ((double)(kCoord - 1) - 0.5) * dz; + + x2 = ((double)iCoord - 0.5) * dx; + y2 = (double)jCoord * dy; + z2 = ((double)kCoord - 0.5) * dz; + + double v_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * V(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * V(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * V(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * V(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * V(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * V(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * V(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * V(iCoord, jCoord, kCoord)); + + double new_y = (y + particletracer->yOffset) + particletracer->dt * v_n; + particletracer->particlePool[i].y = new_y; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)((y + 0.5 * dy) / dy) + 1; + kCoord = (int)(z / dz) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = ((double)(jCoord - 1) - 0.5) * dy; + z1 = (double)(kCoord - 1) * dz; + + x2 = ((double)iCoord - 0.5) * dx; + y2 = ((double)jCoord - 0.5) * dy; + z2 = (double)kCoord * dz; + + double w_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * W(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * W(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * W(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * W(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * W(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * W(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * W(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * W(iCoord, jCoord, kCoord)); + + double new_z = (z + particletracer->zOffset) + particletracer->dt * w_n; + particletracer->particlePool[i].z = new_z; + + if (((new_x < particletracer->xOffset) || + (new_x >= particletracer->xOffsetEnd) || + (new_y < particletracer->yOffset) || + (new_y >= particletracer->yOffsetEnd) || + (new_z < particletracer->zOffset) || + (new_z >= particletracer->zOffsetEnd))) { + // New logic to transfer particles to neighbouring ranks or discard the + // particle. + +#ifdef _MPI + for (int i = 0; i < particletracer->size; ++i) { + if ((new_x >= + particletracer->offset[i + particletracer->size * XOFFSET]) && + (new_x <= particletracer + ->offset[i + particletracer->size * XOFFSETEND]) && + (new_y >= + particletracer->offset[i + particletracer->size * YOFFSET]) && + (new_y <= particletracer + ->offset[i + particletracer->size * YOFFSETEND]) && + (new_z >= + particletracer->offset[i + particletracer->size * ZOFFSET]) && + (new_z <= particletracer + ->offset[i + particletracer->size * ZOFFSETEND]) && + i != particletracer->rank) { + buff[i][particleBufIndex[i]].x = new_x; + buff[i][particleBufIndex[i]].y = new_y; + buff[i][particleBufIndex[i]].z = new_z; + buff[i][particleBufIndex[i]].flag = true; + ++particleBufIndex[i]; + } + } +#endif + + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + + int i_new = new_x / dx, j_new = new_y / dy, k_new = new_z / dz; + + int iOffset = particletracer->xOffset / dx, + jOffset = particletracer->yOffset / dy, + kOffset = particletracer->zOffset / dz; + + if (S(i_new - iOffset, j_new - jOffset, k_new - kOffset) != FLUID) { + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + } + } + +#ifdef _MPI + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Send(&particleBufIndex[i], 1, MPI_INT, i, 0, particletracer->comm); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Recv(&recvparticleBufIndex[i], + 1, + MPI_INT, + i, + 0, + particletracer->comm, + MPI_STATUS_IGNORE); + + // if (0 !=recvparticleBufIndex[i]) { + // printf("Rank %d will receive %d particles from rank %d\n", + // particletracer->rank, + // recvparticleBufIndex[i], + // i); + // } + } + } + + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Send(buff[i], + particleBufIndex[i], + particletracer->mpi_particle, + i, + 0, + particletracer->comm); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Recv(recvbuff[i], + recvparticleBufIndex[i], + particletracer->mpi_particle, + i, + 0, + particletracer->comm, + MPI_STATUS_IGNORE); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + for (int j = 0; j < recvparticleBufIndex[i]; ++j) { + particletracer->particlePool[particletracer->pointer].x = recvbuff[i][j] + .x; + particletracer->particlePool[particletracer->pointer].y = recvbuff[i][j] + .y; + particletracer->particlePool[particletracer->pointer].z = recvbuff[i][j] + .z; + particletracer->particlePool[particletracer->pointer].flag = true; + ++(particletracer->pointer); + ++(particletracer->totalParticles); + } + } + } +#endif +} + +void freeParticles(ParticleTracer* particletracer) +{ + free(particletracer->particlePool); + free(particletracer->linSpaceLine); + free(particletracer->offset); +} + +void writeParticles(ParticleTracer* particletracer) +{ + compress(particletracer); + +#ifdef _MPI + int collectedBuffIndex[particletracer->size]; + + MPI_Gather(&particletracer->totalParticles, + 1, + MPI_INT, + collectedBuffIndex, + 1, + MPI_INT, + 0, + particletracer->comm); + + if (particletracer->rank != 0) { + Particle buff[particletracer->totalParticles]; + for (int i = 0; i < particletracer->totalParticles; ++i) { + buff[i].x = particletracer->particlePool[i].x; + buff[i].y = particletracer->particlePool[i].y; + buff[i].z = particletracer->particlePool[i].z; + buff[i].flag = particletracer->particlePool[i].flag; + // printf("Rank : %d sending to rank 0 X : %.2f, Y : %.2f with totalpt : + // %d\n", particletracer->rank, buff[i].x, buff[i].y, + // particletracer->totalParticles); + } + MPI_Send(buff, + particletracer->totalParticles, + particletracer->mpi_particle, + 0, + 1, + particletracer->comm); + } +#endif + + if (particletracer->rank == 0) { + char filename[50]; + FILE* fp; + + snprintf(filename, 50, "vtk_files/particles_%d.vtk", ts); + fp = fopen(filename, "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + fprintf(fp, "# vtk DataFile Version 3.0\n"); + fprintf(fp, "PAMPI cfd solver particle tracing file\n"); + fprintf(fp, "ASCII\n"); + + fprintf(fp, "DATASET UNSTRUCTURED_GRID\n"); + fprintf(fp, "FIELD FieldData 2\n"); + fprintf(fp, "TIME 1 1 double\n"); + fprintf(fp, "%d\n", ts); + fprintf(fp, "CYCLE 1 1 int\n"); + fprintf(fp, "1\n"); + +#ifdef _MPI + int overallTotalParticles = sum(collectedBuffIndex, particletracer->size); + + fprintf(fp, "POINTS %d float\n", overallTotalParticles); + + // printf("Total particles : %d\n", overallTotalParticles); + + for (int i = 1; i < particletracer->size; ++i) { + Particle recvBuff[collectedBuffIndex[i]]; + MPI_Recv(&recvBuff, + collectedBuffIndex[i], + particletracer->mpi_particle, + i, + 1, + particletracer->comm, + MPI_STATUS_IGNORE); + + for (int j = 0; j < collectedBuffIndex[i]; ++j) { + double x = recvBuff[j].x; + double y = recvBuff[j].y; + double z = recvBuff[j].z; + fprintf(fp, "%f %f %f\n", x, y, z); + } + } +#else + int overallTotalParticles = particletracer->totalParticles; + fprintf(fp, "POINTS %d float\n", overallTotalParticles); + + // printf("Total particles : %d\n", overallTotalParticles); +#endif + for (int i = 0; i < particletracer->totalParticles; ++i) { + double x = particletracer->particlePool[i].x; + double y = particletracer->particlePool[i].y; + double z = particletracer->particlePool[i].z; + fprintf(fp, "%f %f %f\n", x, y, z); + } + + fprintf(fp, "CELLS %d %d\n", overallTotalParticles, 2 * overallTotalParticles); + + for (int i = 0; i < overallTotalParticles; ++i) { + fprintf(fp, "1 %d\n", i); + } + + fprintf(fp, "CELL_TYPES %d\n", overallTotalParticles); + + for (int i = 0; i < overallTotalParticles; ++i) { + fprintf(fp, "1\n"); + } + + fclose(fp); + } + + ++ts; +} + +void initParticleTracer( + ParticleTracer* particletracer, Parameter* params, Discretization* d) +{ + /* initializing local properties from params */ + + particletracer->numberOfParticles = params->numberOfParticles; + particletracer->startTime = params->startTime; + particletracer->injectTimePeriod = params->injectTimePeriod; + particletracer->writeTimePeriod = params->writeTimePeriod; + + particletracer->rank = d->comm.rank; + particletracer->size = d->comm.size; + + particletracer->dt = params->dt; + particletracer->dx = params->xlength / params->imax; + particletracer->dy = params->ylength / params->jmax; + particletracer->dz = params->zlength / params->kmax; + + particletracer->xlength = params->xlength; + particletracer->ylength = params->ylength; + particletracer->zlength = params->zlength; + + particletracer->x1 = params->x1; + particletracer->y1 = params->y1; + particletracer->z1 = params->z1; + particletracer->x2 = params->x2; + particletracer->y2 = params->y2; + particletracer->z2 = params->z2; + + particletracer->lastInjectTime = params->startTime; + particletracer->lastUpdateTime = params->startTime; + particletracer->lastWriteTime = params->startTime; + + particletracer->pointer = 0; + particletracer->totalParticles = 0; + particletracer->removedParticles = 0; + + particletracer->imax = params->imax; + particletracer->jmax = params->jmax; + particletracer->kmax = params->kmax; + + particletracer->imaxLocal = d->comm.imaxLocal; + particletracer->jmaxLocal = d->comm.jmaxLocal; + particletracer->kmaxLocal = d->comm.kmaxLocal; + + particletracer->estimatedNumParticles = (particletracer->imaxLocal * + particletracer->jmaxLocal * + particletracer->kmaxLocal); + + particletracer->particlePool = malloc( + sizeof(Particle) * particletracer->estimatedNumParticles); + + memset(particletracer->particlePool, + 0, + sizeof(Particle) * particletracer->estimatedNumParticles); + + particletracer->linSpaceLine = malloc( + sizeof(Particle) * particletracer->numberOfParticles); + + memset(particletracer->linSpaceLine, + 0, + sizeof(Particle) * particletracer->numberOfParticles); + + particletracer->offset = (double*)malloc(sizeof(double) * 6 * particletracer->size); + double offset[6][particletracer->size]; + +#ifdef _MPI + /* duplicating communication from solver */ + MPI_Comm_dup(d->comm.comm, &particletracer->comm); + + memcpy(particletracer->dims, d->comm.dims, sizeof(d->comm.dims)); + + memcpy(particletracer->coords, d->comm.coords, sizeof(d->comm.coords)); + +#endif + + particletracer->xOffset = d->xOffset; + particletracer->yOffset = d->yOffset; + particletracer->zOffset = d->zOffset; + particletracer->xOffsetEnd = d->xOffsetEnd; + particletracer->yOffsetEnd = d->yOffsetEnd; + particletracer->zOffsetEnd = d->zOffsetEnd; + +#ifdef _MPI + MPI_Allgather(&particletracer->xOffset, + 1, + MPI_DOUBLE, + offset[0], + 1, + MPI_DOUBLE, + particletracer->comm); + MPI_Allgather(&particletracer->yOffset, + 1, + MPI_DOUBLE, + offset[1], + 1, + MPI_DOUBLE, + particletracer->comm); + MPI_Allgather(&particletracer->zOffset, + 1, + MPI_DOUBLE, + offset[2], + 1, + MPI_DOUBLE, + particletracer->comm); + MPI_Allgather(&particletracer->xOffsetEnd, + 1, + MPI_DOUBLE, + offset[3], + 1, + MPI_DOUBLE, + particletracer->comm); + MPI_Allgather(&particletracer->yOffsetEnd, + 1, + MPI_DOUBLE, + offset[4], + 1, + MPI_DOUBLE, + particletracer->comm); + MPI_Allgather(&particletracer->zOffsetEnd, + 1, + MPI_DOUBLE, + offset[5], + 1, + MPI_DOUBLE, + particletracer->comm); + +#endif + memcpy(particletracer->offset, offset, sizeof(offset)); + + particleRandomizer(particletracer); + +#ifdef _MPI + // Create the mpi_particle datatype + MPI_Datatype mpi_particle; + int lengths[4] = { 1, 1, 1, 1 }; + + MPI_Aint displacements[4]; + Particle dummy_particle; + MPI_Aint base_address; + MPI_Get_address(&dummy_particle, &base_address); + MPI_Get_address(&dummy_particle.x, &displacements[0]); + MPI_Get_address(&dummy_particle.y, &displacements[1]); + MPI_Get_address(&dummy_particle.z, &displacements[2]); + MPI_Get_address(&dummy_particle.flag, &displacements[3]); + displacements[0] = MPI_Aint_diff(displacements[0], base_address); + displacements[1] = MPI_Aint_diff(displacements[1], base_address); + displacements[2] = MPI_Aint_diff(displacements[2], base_address); + displacements[3] = MPI_Aint_diff(displacements[3], base_address); + MPI_Datatype types[4] = { MPI_DOUBLE, MPI_DOUBLE, MPI_DOUBLE, MPI_C_BOOL }; + MPI_Type_create_struct(4, + lengths, + displacements, + types, + &particletracer->mpi_particle); + MPI_Type_commit(&particletracer->mpi_particle); +#endif +} + +void printParticleTracerParameters(ParticleTracer* particletracer) +{ + printf("Particle Tracing data:\n"); + printf("Rank : %d\n", particletracer->rank); + printf("\tNumber of particles : %d being injected for every period of %.2f\n", + particletracer->numberOfParticles, + particletracer->injectTimePeriod); + printf("\tstartTime : %.2f\n", particletracer->startTime); + printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : " + "%.2f, z1 : %.2f, x2 : %.2f, y2 : %.2f, z2 : %.2f\n", + particletracer->x1, + particletracer->y1, + particletracer->z1, + particletracer->x2, + particletracer->y2, + particletracer->z2); + printf("\tPointer : %d, TotalParticles : %d\n", + particletracer->pointer, + particletracer->totalParticles); + printf("\tdt : %.2f, dx : %.2f, dy : %.2f, dz : %.2f\n", + particletracer->dt, + particletracer->dx, + particletracer->dy, + particletracer->dz); + printf("\tcoord[0] : %d, coord[1] : %d, coord[2] : %d\n", + particletracer->coords[IDIM], + particletracer->coords[JDIM], + particletracer->coords[KDIM]); + printf("\txOffset : %.2f, yOffset : %.2f, zOffset : %.2f\n", + particletracer->xOffset, + particletracer->yOffset, + particletracer->zOffset); + printf("\txOffsetEnd : %.2f, yOffsetEnd : %.2f, zOffsetEnd : %.2f\n", + particletracer->xOffsetEnd, + particletracer->yOffsetEnd, + particletracer->zOffsetEnd); + printf("\txLocal : %.2f, yLocal : %.2f, zLocal : %.2f\n", + particletracer->xLocal, + particletracer->yLocal, + particletracer->zLocal); +} + +void trace(ParticleTracer* particletracer, + double* restrict u, + double* restrict v, + double* restrict w, + double* restrict s, + double time) +{ + if (time >= particletracer->startTime) { + if ((time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod) { + injectParticles(particletracer, s); + particletracer->lastInjectTime = time; + } + if ((time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod) { + writeParticles(particletracer); + particletracer->lastWriteTime = time; + } + advanceParticles(particletracer, u, v, w, s, time); + + if (particletracer->removedParticles > (particletracer->totalParticles * 0.2)) { + compress(particletracer); + } + particletracer->lastUpdateTime = time; + } +} + +void compress(ParticleTracer* particletracer) +{ + Particle* memPool = particletracer->particlePool; + Particle tempPool[particletracer->totalParticles]; + + memset(tempPool, 0, sizeof(Particle) * particletracer->totalParticles); + + int totalParticles = 0; + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (memPool[i].flag == true) { + tempPool[totalParticles].x = memPool[i].x; + tempPool[totalParticles].y = memPool[i].y; + tempPool[totalParticles].z = memPool[i].z; + tempPool[totalParticles].flag = true; + ++totalParticles; + } + } + + particletracer->totalParticles = totalParticles; + particletracer->removedParticles = 0; + particletracer->pointer = totalParticles; + + memcpy(particletracer->particlePool, tempPool, totalParticles * sizeof(Particle)); +} + +void particleRandomizer(ParticleTracer* particletracer) +{ + memset(particletracer->linSpaceLine, + 0, + sizeof(Particle) * particletracer->numberOfParticles); + + for (int i = 0; i < particletracer->numberOfParticles; ++i) { + + particletracer->linSpaceLine[i].x = (((double)rand() / RAND_MAX) * + (particletracer->x2 - + particletracer->x1)) + + particletracer->x1; + particletracer->linSpaceLine[i].y = (((double)rand() / RAND_MAX) * + (particletracer->y2 - + particletracer->y1)) + + particletracer->y1; + particletracer->linSpaceLine[i].z = (((double)rand() / RAND_MAX) * + (particletracer->z2 - + particletracer->z1)) + + particletracer->z1; + particletracer->linSpaceLine[i].flag = true; + } +} \ No newline at end of file diff --git a/EnhancedSolver/3D-mpi/src/particletracing.h b/EnhancedSolver/3D-mpi/src/particletracing.h new file mode 100644 index 0000000..61da8a5 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/particletracing.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARTICLETRACING_H_ +#define __PARTICLETRACING_H_ +#include "allocate.h" +#include "parameter.h" +#include "discretization.h" +#include +#include + +#define NDIMS 3 + +typedef enum COORD { X = 0, Y, NCOORD } COORD; +typedef struct { + double x, y, z; + bool flag; +} Particle; + +typedef struct { + int numberOfParticles, totalParticles; + double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime, + lastWriteTime; + + int estimatedNumParticles, removedParticles; + + double dx, dy, dz, dt; + Particle* linSpaceLine; + Particle* particlePool; + + int pointer; + + double imax, jmax, kmax, xlength, ylength, zlength, imaxLocal, jmaxLocal, kmaxLocal; + + double x1, y1, z1, x2, y2, z2; + +#ifdef _MPI + MPI_Comm comm; + MPI_Datatype mpi_particle; +#endif + + int rank, size; + int coords[NDIMS], dims[NDIMS]; + + double xLocal, yLocal, zLocal, xOffset, yOffset, zOffset, xOffsetEnd, yOffsetEnd, + zOffsetEnd; + + double* offset; + +} ParticleTracer; + +void initParticleTracer(ParticleTracer*, Parameter*, Discretization*); +void injectParticles(ParticleTracer*, double*); +void advanceParticles(ParticleTracer*, double*, double*, double*, double*, double); +void freeParticles(ParticleTracer*); +void writeParticles(ParticleTracer*); +void printParticleTracerParameters(ParticleTracer*); +void printParticles(ParticleTracer*); +void trace(ParticleTracer*, double*, double*, double*, double*, double); +void compress(ParticleTracer*); +extern void particleRandomizer(ParticleTracer*); +#endif \ No newline at end of file diff --git a/EnhancedSolver/3D-mpi/src/progress.c b/EnhancedSolver/3D-mpi/src/progress.c new file mode 100644 index 0000000..8249ed1 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/progress.c @@ -0,0 +1,65 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "progress.h" +#include +#include +#include +#include + +static double _end; +static int _current; + +void initProgress(double end) +{ + _end = end; + _current = 0; + + printf("[ ]"); + fflush(stdout); +} + +void printProgress(double current) +{ + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); +} + +void stopProgress() +{ + printf("\n"); + fflush(stdout); +} + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; +} + +void writeResidual(FILE* fp, double ts, double res) { fprintf(fp, "%f, %f\n", ts, res); } \ No newline at end of file diff --git a/EnhancedSolver/3D-mpi/src/progress.h b/EnhancedSolver/3D-mpi/src/progress.h new file mode 100644 index 0000000..d612ebe --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/progress.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ + +#include + +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(void); +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); +#endif diff --git a/EnhancedSolver/3D-mpi/src/solver-mg.c b/EnhancedSolver/3D-mpi/src/solver-mg.c new file mode 100644 index 0000000..7324735 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/solver-mg.c @@ -0,0 +1,389 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +static void restrictMG(Solver* s, int level, Comm* comm) +{ + double* r = s->r[level + 1]; + double* old = s->r[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + commExchange(comm, old); + + for (int k = 1; k < comm->kmaxLocal + 1; k++) { + for (int j = 1; j < comm->jmaxLocal + 1; j++) { + for (int i = 1; i < comm->imaxLocal + 1; ++i) { + R(i, j, k) = (OLD(2 * i - 1, 2 * j - 1, 2 * k) + + OLD(2 * i, 2 * j - 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k) + + OLD(2 * i - 1, 2 * j, 2 * k) * 2 + + OLD(2 * i, 2 * j, 2 * k) * 8 + + OLD(2 * i + 1, 2 * j, 2 * k) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k) + + OLD(2 * i, 2 * j + 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i, 2 * j - 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i - 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i, 2 * j, 2 * k - 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k - 1) + + OLD(2 * i, 2 * j + 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k - 1) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i, 2 * j - 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i - 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i, 2 * j, 2 * k + 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k + 1) + + OLD(2 * i, 2 * j + 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k + 1)) / + 64.0; + } + } + } +} + +static void prolongate(Solver* s, int level, Comm* comm) +{ + double* old = s->r[level + 1]; + double* e = s->r[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + for (int k = 2; k < kmaxLocal + 1; k += 2) { + for (int j = 2; j < jmaxLocal + 1; j += 2) { + for (int i = 2; i < imaxLocal + 1; i += 2) { + E(i, j, k) = OLD(i / 2, j / 2, k / 2); + } + } + } +} + +static void correct(Solver* s, double* p, int level, Comm* comm) +{ + double* e = s->e[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + for (int k = 1; k < kmaxLocal + 1; ++k) { + for (int j = 1; j < jmaxLocal + 1; ++j) { + for (int i = 1; i < imaxLocal + 1; ++i) { + P(i, j, k) += E(i, j, k); + } + } + } +} + +static void setBoundaryCondition( + Solver* s, double* p, int imaxLocal, int jmaxLocal, int kmaxLocal) +{ +#ifdef _MPI + if (commIsBoundary(s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + } + } + } + + if (commIsBoundary(s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + } + + if (commIsBoundary(s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + } + } + } + + if (commIsBoundary(s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + } + + if (commIsBoundary(s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + } + } + } + + if (commIsBoundary(s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } + } +#else + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } +#endif +} + +static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + commExchange(comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + P(i, j, k) -= + factor * + (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + commExchange(comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + R(i, + j, + k) = (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * + idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + + res += (R(i, j, k) * R(i, j, k)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + commReduction(&res, SUM); + + res = res / (double)(imaxLocal * jmaxLocal * kmaxLocal); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + return res; +} + +static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, comm); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + } + + res = calculateResidual(s, p, rhs, level, comm); + + // restrict + restrictMG(s, level, comm); + + Comm newcomm; + commUpdateDatatypes(s->comm, &newcomm, imaxLocal, jmaxLocal, kmaxLocal); + + // MGSolver on residual and error. + multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm); + + commFreeCommunicator(&newcomm); + + // prolongate + prolongate(s, level, comm); + + // correct p on finer level using residual + correct(s, p, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + s->comm = &d->comm; + s->problem = p->name; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + int levels = s->levels; + if (commIsMaster(s->comm)) printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * (kmax + 2); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size * sizeof(double)); + s->e[j] = allocate(64, size * sizeof(double)); + + for (size_t i = 0; i < size; i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->comm); +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Residuum: %.6f\n", res); + } +#endif + + return res; +} diff --git a/EnhancedSolver/3D-mpi/src/solver-rb.c b/EnhancedSolver/3D-mpi/src/solver-rb.c new file mode 100644 index 0000000..d64fecd --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/solver-rb.c @@ -0,0 +1,175 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + s->comm = &d->comm; + s->problem = p->name; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double epssq = eps * eps; + int it = 0; + double res = 1.0; + int pass, ksw, jsw, isw; + + while ((res >= epssq) && (it < itermax)) { + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + commExchange(s->comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + double r = + RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2); + + P(i, j, k) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +#ifdef _MPI + if (commIsBoundary(s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + } + } + } + + if (commIsBoundary(s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + } + + if (commIsBoundary(s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + } + } + } + + if (commIsBoundary(s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + } + + if (commIsBoundary(s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + } + } + } + + if (commIsBoundary(s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } + } +#else + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } +#endif + commReduction(&res, SUM); + res = res / (double)(imax * jmax * kmax); +#ifdef DEBUG + if (commIsMaster(&s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + commExchange(s->comm, p); + it++; + } + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + +return res; +} \ No newline at end of file diff --git a/EnhancedSolver/3D-mpi/src/solver.h b/EnhancedSolver/3D-mpi/src/solver.h new file mode 100644 index 0000000..d9a0c39 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/solver.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "comm.h" +#include "grid.h" +#include "parameter.h" +#include "discretization.h" + +typedef struct { + /* geometry and grid information */ + Grid* grid; + /* arrays */ + double *p, *rhs; + double *f, *g, *h; + double *u, *v, *w; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy, gz; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + /* communication */ + double **r, **e; + int levels, presmooth, postsmooth; + Comm* comm; +} Solver; + +extern double solve(Solver* , double* , double* ); +extern void initSolver(Solver*, Discretization*, Parameter*); +#endif diff --git a/EnhancedSolver/3D-mpi/src/test.c b/EnhancedSolver/3D-mpi/src/test.c new file mode 100644 index 0000000..bf2ad16 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/test.c @@ -0,0 +1,129 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "test.h" + +#define X(v, i, j, k) \ + v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +void testInit(Solver* s) +{ + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; + int myrank = s->comm->rank; + double* p = s->p; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + for (int k = 0; k < kmaxLocal + 2; k++) { + for (int j = 0; j < jmaxLocal + 2; j++) { + for (int i = 0; i < imaxLocal + 2; i++) { + X(p, i, j, k) = 10.0; + X(f, i, j, k) = myrank + 1.0; + X(g, i, j, k) = myrank + 1.0; + X(h, i, j, k) = myrank + 1.0; + } + } + } + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + X(p, i, j, k) = myrank + 1.0; + X(f, i, j, k) = myrank + 1.0; + X(g, i, j, k) = myrank + 1.0; + X(h, i, j, k) = myrank + 1.0; + } + } + } +} + +static char* direction2String(Direction dir) +{ + switch (dir) { + case LEFT: + return "left"; + break; + case RIGHT: + return "right"; + break; + case BOTTOM: + return "bottom"; + break; + case TOP: + return "top"; + break; + case FRONT: + return "front"; + break; + case BACK: + return "back"; + break; + case NDIRS: + return "ERROR"; + default: + return "ERROR"; + } +} + +static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir) +{ + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; + char filename[50]; + snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm->rank); + FILE* fh = fopen(filename, "w"); + + for (int y = 0; y < ymax; y++) { + for (int x = 0; x < xmax; x++) { + switch (dir) { + case LEFT: + fprintf(fh, "%12.8f ", X(a, 0, x, y)); + break; + case RIGHT: + fprintf(fh, "%12.8f ", X(a, imaxLocal + 1, x, y)); + break; + case BOTTOM: + fprintf(fh, "%12.8f ", X(a, x, 0, y)); + break; + case TOP: + fprintf(fh, "%12.8f ", X(a, x, jmaxLocal + 1, y)); + break; + case FRONT: + fprintf(fh, "%12.8f ", X(a, x, y, 0)); + break; + case BACK: + fprintf(fh, "%12.8f ", X(a, x, y, kmaxLocal + 1)); + break; + case NDIRS: + printf("ERROR\n"); + break; + } + } + fprintf(fh, "\n"); + } + fclose(fh); +} + +void testPrintHalo(Solver* s, double* a) +{ + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; + + printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, BOTTOM); + printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, TOP); + printPlane(s, a, kmaxLocal + 2, jmaxLocal + 2, LEFT); + printPlane(s, a, kmaxLocal + 2, jmaxLocal + 2, RIGHT); + printPlane(s, a, jmaxLocal + 2, imaxLocal + 2, FRONT); + printPlane(s, a, jmaxLocal + 2, imaxLocal + 2, BACK); +} diff --git a/EnhancedSolver/3D-mpi/src/test.h b/EnhancedSolver/3D-mpi/src/test.h new file mode 100644 index 0000000..779b509 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/test.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __TEST_H_ +#define __TEST_H_ +#include "solver.h" + +extern void testInit(Solver* s); +extern void testPrintHalo(Solver* s, double* a); +#endif // __TEST_H_ diff --git a/EnhancedSolver/3D-mpi/src/timing.c b/EnhancedSolver/3D-mpi/src/timing.c new file mode 100644 index 0000000..78b01c4 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/timing.c @@ -0,0 +1,22 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution(void) +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} diff --git a/EnhancedSolver/3D-mpi/src/timing.h b/EnhancedSolver/3D-mpi/src/timing.h new file mode 100644 index 0000000..ed05a8c --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/timing.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(void); +extern double getTimeResolution(void); + +#endif // __TIMING_H_ diff --git a/EnhancedSolver/3D-mpi/src/util.h b/EnhancedSolver/3D-mpi/src/util.h new file mode 100644 index 0000000..735209f --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/util.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#define P(i, j, k) p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define F(i, j, k) f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define G(i, j, k) g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define H(i, j, k) h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define U(i, j, k) u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define V(i, j, k) v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define W(i, j, k) w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define S(i, j, k) s[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define E(i, j, k) e[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define R(i, j, k) r[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define OLD(i, j, k) old[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define RHS(i, j, k) rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +#endif // __UTIL_H_ diff --git a/EnhancedSolver/3D-mpi/src/vtkWriter-mpi.c b/EnhancedSolver/3D-mpi/src/vtkWriter-mpi.c new file mode 100644 index 0000000..d9670d5 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/vtkWriter-mpi.c @@ -0,0 +1,227 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "vtkWriter.h" + +// reset fileview for output of string headers +static void resetFileview(VtkOptions* o) +{ + MPI_Offset disp; + MPI_File_sync(o->fh); + MPI_Barrier(o->comm.comm); + MPI_File_get_size(o->fh, &disp); + MPI_File_set_view(o->fh, disp, MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL); +} + +static void writeVersion(VtkOptions* o) +{ + char header[50] = "# vtk DataFile Version 3.0\n"; + // always overwrite exiting files + MPI_File_set_view(o->fh, 0, MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL); + + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + } +} + +static void writeHeader(VtkOptions* o) +{ + char header[400]; + char* cursor = header; + + cursor += sprintf(cursor, "PAMPI cfd solver output\n"); + cursor += sprintf(cursor, "BINARY\n"); + + cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n"); + cursor += sprintf(cursor, + "DIMENSIONS %d %d %d\n", + o->grid->imax, + o->grid->jmax, + o->grid->kmax); + cursor += sprintf(cursor, + "ORIGIN %f %f %f\n", + o->grid->dx * 0.5, + o->grid->dy * 0.5, + o->grid->dz * 0.5); + cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz); + cursor += sprintf(cursor, + "POINT_DATA %d\n", + o->grid->imax * o->grid->jmax * o->grid->kmax); + + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + } +} + +void vtkOpen(VtkOptions* o, char* problem) +{ + char filename[50]; + snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size); + MPI_File_open(o->comm.comm, + filename, + MPI_MODE_WRONLY | MPI_MODE_CREATE, + MPI_INFO_NULL, + &o->fh); + + if (commIsMaster(&o->comm)) { + printf("Writing VTK output for %s\n", problem); + } + + writeVersion(o); + writeHeader(o); +} + +void vtkScalar(VtkOptions* o, char* name, double* s) +{ + resetFileview(o); + if (commIsMaster(&o->comm)) printf("Register scalar %s\n", name); + + char header[100]; + char* cursor = header; + + cursor += sprintf(cursor, "SCALARS %s double\n", name); + + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + } + + int offsets[NDIMS]; + commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax); + + // set global view in file + MPI_Offset disp; + MPI_Datatype fileViewType; + MPI_File_sync(o->fh); + MPI_Barrier(o->comm.comm); + MPI_File_get_size(o->fh, &disp); + + MPI_Type_create_subarray(NDIMS, + (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax }, + (int[NDIMS]) { o->comm.kmaxLocal, o->comm.jmaxLocal, o->comm.imaxLocal }, + offsets, + MPI_ORDER_C, + MPI_DOUBLE, + &fileViewType); + MPI_Type_commit(&fileViewType); + MPI_File_set_view(o->fh, disp, MPI_DOUBLE, fileViewType, "external32", MPI_INFO_NULL); + +#ifdef VERBOSE + printf("Rank: %d, Disp: %lld, Size(k,j,i): %d %d %d, Offset(k,j,i): %d %d %d\n", + o->comm.rank, + disp, + o->comm.kmaxLocal, + o->comm.jmaxLocal, + o->comm.imaxLocal, + offsets[KDIM], + offsets[JDIM], + offsets[IDIM]); +#endif + + // create local bulk type + MPI_Datatype bulkType; + + MPI_Type_create_subarray(NDIMS, + (int[NDIMS]) { o->comm.kmaxLocal + 2, + o->comm.jmaxLocal + 2, + o->comm.imaxLocal + 2 }, // oldsizes + (int[NDIMS]) { o->comm.kmaxLocal, + o->comm.jmaxLocal, + o->comm.imaxLocal }, // newsizes + (int[NDIMS]) { 1, 1, 1 }, // offsets + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + + MPI_File_write(o->fh, s, 1, bulkType, MPI_STATUS_IGNORE); + MPI_Type_free(&bulkType); + MPI_Type_free(&fileViewType); + + // Binary segment must be terminated with newline character + resetFileview(o); + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, "\n", 1, MPI_CHAR, MPI_STATUS_IGNORE); + } +} + +#define G(v, i, j, k) \ + v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +void vtkVector(VtkOptions* o, char* name, VtkVector vec) +{ + int imaxLocal = o->comm.imaxLocal; + int jmaxLocal = o->comm.jmaxLocal; + int kmaxLocal = o->comm.kmaxLocal; + + if (commIsMaster(&o->comm)) printf("Register vector %s\n", name); + const size_t MAX_HEADER = 100; + + char* header = (char*)malloc(MAX_HEADER); + sprintf(header, "VECTORS %s double\n", name); + + resetFileview(o); + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + } + + int offsets[NDIMS]; + commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax); + + // set global view in file + MPI_Offset disp; + MPI_Datatype fileViewType, vectorType; + MPI_File_sync(o->fh); + MPI_Barrier(o->comm.comm); + MPI_File_get_size(o->fh, &disp); + + MPI_Type_contiguous(NDIMS, MPI_DOUBLE, &vectorType); + MPI_Type_commit(&vectorType); + + MPI_Type_create_subarray(NDIMS, + (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax }, + (int[NDIMS]) { kmaxLocal, jmaxLocal, imaxLocal }, + offsets, + MPI_ORDER_C, + vectorType, + &fileViewType); + MPI_Type_commit(&fileViewType); + MPI_File_set_view(o->fh, disp, MPI_DOUBLE, fileViewType, "external32", MPI_INFO_NULL); + + size_t cnt = imaxLocal * jmaxLocal * kmaxLocal; + double* tmp = allocate(64, cnt * NDIMS * sizeof(double)); + int idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(vec.u, i, j, k) + G(vec.u, i - 1, j, k)) / 2.0; + tmp[idx++] = (G(vec.v, i, j, k) + G(vec.v, i, j - 1, k)) / 2.0; + tmp[idx++] = (G(vec.w, i, j, k) + G(vec.w, i, j, k - 1)) / 2.0; + } + } + } + + if (commIsMaster(&o->comm)) printf("Write %d vectors\n", (int)cnt); + MPI_File_write(o->fh, tmp, cnt, vectorType, MPI_STATUS_IGNORE); + MPI_Type_free(&fileViewType); + MPI_Type_free(&vectorType); + + // Binary segment must be terminated with newline character + resetFileview(o); + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, "\n", 1, MPI_CHAR, MPI_STATUS_IGNORE); + } +} + +void vtkClose(VtkOptions* o) { MPI_File_close(&o->fh); } diff --git a/EnhancedSolver/3D-mpi/src/vtkWriter-seq.c b/EnhancedSolver/3D-mpi/src/vtkWriter-seq.c new file mode 100644 index 0000000..346e367 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/vtkWriter-seq.c @@ -0,0 +1,147 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "vtkWriter.h" +#define Z(v, i, j, k) v[(k)*imax * jmax + (j)*imax + (i)] + +static double floatSwap(double f) +{ + union { + double f; + char b[8]; + } dat1, dat2; + + dat1.f = f; + dat2.b[0] = dat1.b[7]; + dat2.b[1] = dat1.b[6]; + dat2.b[2] = dat1.b[5]; + dat2.b[3] = dat1.b[4]; + dat2.b[4] = dat1.b[3]; + dat2.b[5] = dat1.b[2]; + dat2.b[6] = dat1.b[1]; + dat2.b[7] = dat1.b[0]; + return dat2.f; +} + +static void writeHeader(VtkOptions* o) +{ + fprintf(o->fh, "# vtk DataFile Version 3.0\n"); + fprintf(o->fh, "PAMPI cfd solver output\n"); + if (o->fmt == ASCII) { + fprintf(o->fh, "ASCII\n"); + } else if (o->fmt == BINARY) { + fprintf(o->fh, "BINARY\n"); + } + + fprintf(o->fh, "DATASET STRUCTURED_POINTS\n"); + fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid->imax, o->grid->jmax, o->grid->kmax); + fprintf(o->fh, + "ORIGIN %f %f %f\n", + o->grid->dx * 0.5, + o->grid->dy * 0.5, + o->grid->dz * 0.5); + fprintf(o->fh, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz); + fprintf(o->fh, "POINT_DATA %d\n", o->grid->imax * o->grid->jmax * o->grid->kmax); +} + +void vtkOpen(VtkOptions* o, char* problem) +{ + char filename[50]; + + snprintf(filename, 50, "%s.vtk", problem); + o->fh = fopen(filename, "w"); + writeHeader(o); + + printf("Writing VTK output for %s\n", problem); +} + +static void writeScalar(VtkOptions* o, double* s) +{ + int imax = o->grid->imax; + int jmax = o->grid->jmax; + int kmax = o->grid->kmax; + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, "%f\n", Z(s, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((double[1]) { floatSwap(Z(s, i, j, k)) }, + sizeof(double), + 1, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +static bool isInitialized(FILE* ptr) +{ + if (ptr == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + return false; + } + return true; +} + +void vtkScalar(VtkOptions* o, char* name, double* s) +{ + printf("Register scalar %s\n", name); + if (!isInitialized(o->fh)) return; + fprintf(o->fh, "SCALARS %s double 1\n", name); + fprintf(o->fh, "LOOKUP_TABLE default\n"); + writeScalar(o, s); +} + +static void writeVector(VtkOptions* o, VtkVector vec) +{ + int imax = o->grid->imax; + int jmax = o->grid->jmax; + int kmax = o->grid->kmax; + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, + "%f %f %f\n", + Z(vec.u, i, j, k), + Z(vec.v, i, j, k), + Z(vec.w, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((double[3]) { floatSwap(Z(vec.u, i, j, k)), + floatSwap(Z(vec.v, i, j, k)), + floatSwap(Z(vec.w, i, j, k)) }, + sizeof(double), + 3, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +void vtkVector(VtkOptions* o, char* name, VtkVector vec) +{ + printf("Register vector %s\n", name); + if (!isInitialized(o->fh)) return; + fprintf(o->fh, "VECTORS %s double\n", name); + writeVector(o, vec); +} + +void vtkClose(VtkOptions* o) +{ + fclose(o->fh); + o->fh = NULL; +} diff --git a/EnhancedSolver/3D-mpi/src/vtkWriter.h b/EnhancedSolver/3D-mpi/src/vtkWriter.h new file mode 100644 index 0000000..e93835c --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/vtkWriter.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __VTKWRITER_H_ +#define __VTKWRITER_H_ +#include + +#include "comm.h" +#include "grid.h" + +typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat; + +typedef struct VtkOptions { + Grid* grid; +#ifdef _VTK_WRITER_MPI + MPI_File fh; +#else + FILE* fh; + VtkFormat fmt; +#endif // _VTK_WRITER_MPI + Comm comm; +} VtkOptions; + +typedef struct VtkVector { + double *u, *v, *w; +} VtkVector; + +extern void vtkOpen(VtkOptions* opts, char* filename); +extern void vtkVector(VtkOptions* opts, char* name, VtkVector vec); +extern void vtkScalar(VtkOptions* opts, char* name, double* p); +extern void vtkClose(VtkOptions* opts); +#endif // __VTKWRITER_H_ diff --git a/EnhancedSolver/3D-seq/Makefile b/EnhancedSolver/3D-seq/Makefile new file mode 100644 index 0000000..6d1c49d --- /dev/null +++ b/EnhancedSolver/3D-seq/Makefile @@ -0,0 +1,88 @@ +#======================================================================================= +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm format + +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f vtk_files/*.vtk + @rm -f *.png + @rm -f *.vtk + + +clean: vis_clean + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + @rm -f *.dat + @rm -f *.png + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/EnhancedSolver/3D-seq/README.md b/EnhancedSolver/3D-seq/README.md new file mode 100644 index 0000000..d980b54 --- /dev/null +++ b/EnhancedSolver/3D-seq/README.md @@ -0,0 +1,78 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about solver, affinity settings, allocation sizes and timer resolution. +For debugging you may want to set the VERBOSE option: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +` + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. + +## Usage + +You have to provide a parameter file describing the problem you want to solve: +``` +./exe-CLANG dcavity.par +``` + +Examples are given in in dcavity (a lid driven cavity test case) and canal (simulating a empty canal). + +You can plot the resulting velocity and pressure fields using gnuplot: +``` +gnuplot vector.plot +``` +and for the pressure: +``` +gnuplot surface.plot +``` diff --git a/EnhancedSolver/3D-seq/backstep.par b/EnhancedSolver/3D-seq/backstep.par new file mode 100644 index 0000000..8075448 --- /dev/null +++ b/EnhancedSolver/3D-seq/backstep.par @@ -0,0 +1,89 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name backstep # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 36000.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 1.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 7.0 # domain size in x-direction +ylength 1.5 # domain size in y-direction +zlength 1.0 # domain size in z-direction +imax 210 # number of interior cells in x-direction +jmax 45 # number of interior cells in y-direction +kmax 30 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 40.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 2000 +startTime 0 +injectTimePeriod 1.0 +writeTimePeriod 0.2 + +x1 0.0 +y1 0.5 +z1 0.0 +x2 0.0 +y2 1.5 +z2 1.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 1 +xCenter 0.0 +yCenter 0.0 +zCenter 0.0 +xRectLength 2.0 +yRectLength 1.0 +zRectLength 2.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/EnhancedSolver/3D-seq/canal.par b/EnhancedSolver/3D-seq/canal.par new file mode 100644 index 0000000..3ca4a9e --- /dev/null +++ b/EnhancedSolver/3D-seq/canal.par @@ -0,0 +1,88 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 100.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +zlength 4.0 # domain size in z-direction +imax 200 # number of interior cells in x-direction +jmax 40 # number of interior cells in y-direction +kmax 40 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 60.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +omg 0.52 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 10 +injectTimePeriod 2.0 +writeTimePeriod 1.0 + +x1 0.0 +y1 0.0 +z1 0.0 +x2 4.0 +y2 4.0 +z2 4.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 10.0 +yCenter 2.0 +zCenter 2.0 +xRectLength 8.0 +yRectLength 2.0 +zRectLength 2.0 +circleRadius 1.0 +#=============================================================================== diff --git a/EnhancedSolver/3D-seq/config.mk b/EnhancedSolver/3D-seq/config.mk new file mode 100644 index 0000000..2cf33f9 --- /dev/null +++ b/EnhancedSolver/3D-seq/config.mk @@ -0,0 +1,12 @@ +# Supported: GCC, CLANG, ICC +TAG ?= ICC +ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg +# Run in debug settings +DEBUG ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DDEBUG diff --git a/EnhancedSolver/3D-seq/dcavity.par b/EnhancedSolver/3D-seq/dcavity.par new file mode 100644 index 0000000..204e007 --- /dev/null +++ b/EnhancedSolver/3D-seq/dcavity.par @@ -0,0 +1,88 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcLeft 1 # flags for boundary conditions +bcRight 1 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 1000.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +zlength 1.0 # domain size in z-direction +imax 80 # number of interior cells in x-direction +jmax 80 # number of interior cells in y-direction +kmax 80 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 10.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.5 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 0 +injectTimePeriod 1.0 +writeTimePeriod 0.1 + +x1 0.0 +y1 0.1 +z1 0.0 +x2 1.0 +y2 0.1 +z2 1.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 0.5 +yCenter 0.5 +zCenter 0.5 +xRectLength 0.25 +yRectLength 0.25 +zRectLength 0.25 +circleRadius 0.25 +#=============================================================================== diff --git a/EnhancedSolver/3D-seq/include_CLANG.mk b/EnhancedSolver/3D-seq/include_CLANG.mk new file mode 100644 index 0000000..a124053 --- /dev/null +++ b/EnhancedSolver/3D-seq/include_CLANG.mk @@ -0,0 +1,19 @@ +CC = clang +GCC = cc +LINKER = $(CC) + +ifeq ($(strip $(ENABLE_OPENMP)),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif +ifeq ($(strip $(DEBUG)),true) +CFLAGS = -O0 -g -std=c17 +else +CFLAGS = -O3 -std=c17 $(OPENMP) +endif + +VERSION = --version +LFLAGS = $(OPENMP) -lm +DEFINES = -D_GNU_SOURCE +INCLUDES = diff --git a/EnhancedSolver/3D-seq/include_GCC.mk b/EnhancedSolver/3D-seq/include_GCC.mk new file mode 100644 index 0000000..427e798 --- /dev/null +++ b/EnhancedSolver/3D-seq/include_GCC.mk @@ -0,0 +1,14 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/EnhancedSolver/3D-seq/include_ICC.mk b/EnhancedSolver/3D-seq/include_ICC.mk new file mode 100644 index 0000000..94b8e20 --- /dev/null +++ b/EnhancedSolver/3D-seq/include_ICC.mk @@ -0,0 +1,14 @@ +CC = icc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/EnhancedSolver/3D-seq/karman.par b/EnhancedSolver/3D-seq/karman.par new file mode 100644 index 0000000..a3eda7f --- /dev/null +++ b/EnhancedSolver/3D-seq/karman.par @@ -0,0 +1,89 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name karman # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 5050.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 8.0 # domain size in y-direction +zlength 8.0 # domain size in z-direction +imax 200 # number of interior cells in x-direction +jmax 80 # number of interior cells in y-direction +kmax 80 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 250.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 200 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.75 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 500 +startTime 50 +injectTimePeriod 1.0 +writeTimePeriod 1.0 + +x1 0.0 +y1 3.6 +z1 3.6 +x2 0.0 +y2 4.7 +z2 4.7 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 2 +xCenter 5.0 +yCenter 4.0 +zCenter 4.0 +xRectLength 8.0 +yRectLength 2.0 +zRectLength 2.0 +circleRadius 2.0 + +#=============================================================================== diff --git a/EnhancedSolver/3D-seq/residual.plot b/EnhancedSolver/3D-seq/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/EnhancedSolver/3D-seq/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/EnhancedSolver/3D-seq/src/allocate.c b/EnhancedSolver/3D-seq/src/allocate.c new file mode 100644 index 0000000..cf2efd6 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/allocate.c @@ -0,0 +1,38 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/EnhancedSolver/3D-seq/src/allocate.h b/EnhancedSolver/3D-seq/src/allocate.h new file mode 100644 index 0000000..77f4ba0 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(size_t alignment, size_t bytesize); + +#endif diff --git a/EnhancedSolver/3D-seq/src/discretization.c b/EnhancedSolver/3D-seq/src/discretization.c new file mode 100644 index 0000000..3661c70 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/discretization.c @@ -0,0 +1,1128 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "allocate.h" +#include "discretization.h" +#include "parameter.h" +#include "util.h" + +static double distance( + double i, double j, double k, double iCenter, double jCenter, double kCenter) +{ + return sqrt(pow(iCenter - i, 2) + pow(jCenter - j, 2) + pow(kCenter - k, 2) * 1.0); +} + +static void printConfig(Discretization* d) +{ + printf("Parameters for #%s#\n", d->problem); + printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n", + d->bcLeft, + d->bcRight, + d->bcBottom, + d->bcTop, + d->bcFront, + d->bcBack); + printf("\tReynolds number: %.2f\n", d->re); + printf("\tGx Gy: %.2f %.2f %.2f\n", d->gx, d->gy, d->gz); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + d->grid.xlength, + d->grid.ylength, + d->grid.zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", d->grid.imax, d->grid.jmax, d->grid.kmax); + printf("\tCell size (dx, dy, dz): %f, %f, %f\n", d->grid.dx, d->grid.dy, d->grid.dz); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te); + printf("\tdt bound: %.6f\n", d->dtBound); + printf("\tTau factor: %.2f\n", d->tau); + printf("Iterative parameters:\n"); + printf("\tepsilon (stopping tolerance) : %f\n", d->eps); + printf("\tgamma factor: %f\n", d->gamma); + printf("\tomega (SOR relaxation): %f\n", d->omega); +} + +void initDiscretization(Discretization* d, Parameter* p) +{ + d->problem = p->name; + d->bcLeft = p->bcLeft; + d->bcRight = p->bcRight; + d->bcBottom = p->bcBottom; + d->bcTop = p->bcTop; + d->bcFront = p->bcFront; + d->bcBack = p->bcBack; + + d->grid.imax = p->imax; + d->grid.jmax = p->jmax; + d->grid.kmax = p->kmax; + d->grid.xlength = p->xlength; + d->grid.ylength = p->ylength; + d->grid.zlength = p->zlength; + d->grid.dx = p->xlength / p->imax; + d->grid.dy = p->ylength / p->jmax; + d->grid.dz = p->zlength / p->kmax; + + d->eps = p->eps; + d->omega = p->omg; + d->re = p->re; + d->gx = p->gx; + d->gy = p->gy; + d->gz = p->gz; + d->dt = p->dt; + d->te = p->te; + d->tau = p->tau; + d->gamma = p->gamma; + + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + size_t bytesize = (imax + 2) * (jmax + 2) * (kmax + 2) * sizeof(double); + d->u = allocate(64, bytesize); + d->v = allocate(64, bytesize); + d->w = allocate(64, bytesize); + d->p = allocate(64, bytesize); + d->rhs = allocate(64, bytesize); + d->f = allocate(64, bytesize); + d->g = allocate(64, bytesize); + d->h = allocate(64, bytesize); + d->s = allocate(64, bytesize); + + for (int i = 0; i < (imax + 2) * (jmax + 2) * (kmax + 2); i++) { + d->u[i] = p->u_init; + d->v[i] = p->v_init; + d->w[i] = p->w_init; + d->p[i] = p->p_init; + d->rhs[i] = 0.0; + d->f[i] = 0.0; + d->g[i] = 0.0; + d->h[i] = 0.0; + d->s[i] = FLUID; + } + + double dx = d->grid.dx; + double dy = d->grid.dy; + double dz = d->grid.dz; + + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz); + d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; + + double xCenter = 0, yCenter = 0, zCenter = 0, radius = 0; + double x1 = 0, x2 = 0, y1 = 0, y2 = 0, z1 = 0, z2 = 0; + + double* s = d->s; + + switch (p->shape) { + case NOSHAPE: + break; + case RECT: + x1 = p->xCenter - p->xRectLength / 2; + x2 = p->xCenter + p->xRectLength / 2; + y1 = p->yCenter - p->yRectLength / 2; + y2 = p->yCenter + p->yRectLength / 2; + z1 = p->zCenter - p->zRectLength / 2; + z2 = p->zCenter + p->zRectLength / 2; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + if ((x1 <= (i * dx)) && ((i * dx) <= x2) && (y1 <= (j * dy)) && + ((j * dy) <= y2) && ((z1 <= (k * dz)) && ((k * dz) <= z2))) { + S(i, j, k) = OBSTACLE; + } + } + } + } + + break; + case CIRCLE: + xCenter = p->xCenter; + yCenter = p->yCenter; + zCenter = p->zCenter; + radius = p->circleRadius; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + if (distance((i * dx), + (j * dy), + (k * dz), + xCenter, + yCenter, + zCenter) <= radius) { + S(i, j, k) = OBSTACLE; + } + } + } + } + break; + default: + break; + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + + /* Assigning enum values to Corners */ + if (S(i - 1, j + 1, k - 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i + 1, j - 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPLEFTCORNER; // + } + if (S(i + 1, j + 1, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j - 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPRIGHTCORNER; // + } + if (S(i - 1, j - 1, k - 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i + 1, j + 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMLEFTCORNER; // + } + if (S(i + 1, j - 1, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j + 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMRIGHTCORNER; // + } + if (S(i - 1, j + 1, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i + 1, j - 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPLEFTCORNER; // + } + if (S(i + 1, j + 1, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i - 1, j - 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPRIGHTCORNER; + } + if (S(i - 1, j - 1, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i + 1, j + 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMLEFTCORNER; + } + if (S(i + 1, j - 1, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i - 1, j + 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMRIGHTCORNER; + } + /* Assigning enum values to Lines */ + if (S(i - 1, j, k - 1) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j, k) == FLUID && S(i + 1, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTLEFTLINE; + } + if (S(i + 1, j, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j, k - 1) == FLUID && S(i - 1, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTRIGHTLINE; + } + if (S(i, j + 1, k - 1) == FLUID && S(i, j + 1, k) == FLUID && + S(i, j, k - 1) == FLUID && S(i, j - 1, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPLINE; + } + if (S(i, j - 1, k - 1) == FLUID && S(i, j, k - 1) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j + 1, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMLINE; + } + if (S(i - 1, j + 1, k) == FLUID && S(i, j + 1, k) == FLUID && + S(i - 1, j, k) == FLUID && S(i + 1, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDTOPLEFTLINE; + } + if (S(i + 1, j + 1, k) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i - 1, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDTOPRIGHTLINE; + } + if (S(i - 1, j - 1, k) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i + 1, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDBOTTOMLEFTLINE; + } + if (S(i + 1, j - 1, k) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i - 1, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDBOTTOMRIGHTLINE; + } + if (S(i - 1, j, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i + 1, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKLEFTLINE; + } + if (S(i + 1, j, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i - 1, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKRIGHTLINE; + } + if (S(i, j + 1, k + 1) == FLUID && S(i, j + 1, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i, j - 1, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPLINE; + } + if (S(i, j - 1, k + 1) == FLUID && S(i, j - 1, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i, j + 1, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMLINE; + } + /* Assigning enum values to Faces */ + if (S(i, j, k - 1) == FLUID && S(i, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTFACE; // + } + if (S(i, j, k + 1) == FLUID && S(i, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKFACE; // + } + if (S(i, j - 1, k) == FLUID && S(i, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BOTTOMFACE; // + } + if (S(i, j + 1, k) == FLUID && S(i, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = TOPFACE; // + } + if (S(i - 1, j, k) == FLUID && S(i + 1, j, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = LEFTFACE; // + } + if (S(i + 1, j, k) == FLUID && S(i - 1, j, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = RIGHTFACE; // + } + } + } + } + +#ifdef VERBOSE + printConfig(d); +#endif /* VERBOSE */ +} + +void computeRHS(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + double idx = 1.0 / d->grid.dx; + double idy = 1.0 / d->grid.dy; + double idz = 1.0 / d->grid.dz; + double idt = 1.0 / d->dt; + + double* rhs = d->rhs; + double* f = d->f; + double* g = d->g; + double* h = d->h; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx + + (G(i, j, k) - G(i, j - 1, k)) * idy + + (H(i, j, k) - H(i, j, k - 1)) * idz) * + idt; + } + } + } +} + +static double maxElement(Discretization* d, double* m) +{ + int size = (d->grid.imax + 2) * (d->grid.jmax + 2) * (d->grid.kmax + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + return maxval; +} + +void normalizePressure(Discretization* d) +{ + int size = (d->grid.imax + 2) * (d->grid.jmax + 2) * (d->grid.kmax + 2); + double* p = d->p; + double avgP = 0.0; + + for (int i = 0; i < size; i++) { + avgP += p[i]; + } + avgP /= size; + + for (int i = 0; i < size; i++) { + p[i] = p[i] - avgP; + } +} + +void computeTimestep(Discretization* d) +{ + double dt = d->dtBound; + double dx = d->grid.dx; + double dy = d->grid.dy; + double dz = d->grid.dz; + + double umax = maxElement(d, d->u); + double vmax = maxElement(d, d->v); + double wmax = maxElement(d, d->w); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + if (wmax > 0) { + dt = (dt > dz / wmax) ? dz / wmax : dt; + } + + d->dt = dt * d->tau; +} + +void setBoundaryConditions(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + + double* u = d->u; + double* v = d->v; + double* w = d->w; + + switch (d->bcTop) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, jmax, k) = 0.0; + U(i, jmax + 1, k) = -U(i, jmax, k); + W(i, jmax + 1, k) = -W(i, jmax, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, jmax, k) = 0.0; + U(i, jmax + 1, k) = U(i, jmax, k); + W(i, jmax + 1, k) = W(i, jmax, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + U(i, jmax + 1, k) = U(i, jmax, k); + V(i, jmax, k) = V(i, jmax - 1, k); + W(i, jmax + 1, k) = W(i, jmax, k); + } + } + break; + case PERIODIC: + break; + } + + switch (d->bcBottom) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, 0, k) = 0.0; + U(i, 0, k) = -U(i, 1, k); + W(i, 0, k) = -W(i, 1, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, 0, k) = 0.0; + U(i, 0, k) = U(i, 1, k); + W(i, 0, k) = W(i, 1, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = V(i, 1, k); + W(i, 0, k) = W(i, 1, k); + } + } + break; + case PERIODIC: + break; + } + + switch (d->bcLeft) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = -V(1, j, k); + W(0, j, k) = -W(1, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(0, j, k) = U(1, j, k); + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case PERIODIC: + break; + } + + switch (d->bcRight) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(imax, j, k) = 0.0; + V(imax + 1, j, k) = -V(imax, j, k); + W(imax + 1, j, k) = -W(imax, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(imax, j, k) = 0.0; + V(imax + 1, j, k) = V(imax, j, k); + W(imax + 1, j, k) = W(imax, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(imax, j, k) = U(imax - 1, j, k); + V(imax + 1, j, k) = V(imax, j, k); + W(imax + 1, j, k) = W(imax, j, k); + } + } + break; + case PERIODIC: + break; + } + + switch (d->bcFront) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, 0) = -U(i, j, 1); + V(i, j, 0) = -V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = W(i, j, 1); + } + } + break; + case PERIODIC: + break; + } + + switch (d->bcBack) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, kmax + 1) = -U(i, j, kmax); + V(i, j, kmax + 1) = -V(i, j, kmax); + W(i, j, kmax + 1) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, kmax + 1) = U(i, j, kmax); + V(i, j, kmax + 1) = V(i, j, kmax); + W(i, j, kmax + 1) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, kmax + 1) = U(i, j, kmax); + V(i, j, kmax + 1) = V(i, j, kmax); + W(i, j, kmax) = W(i, j, kmax - 1); + } + } + break; + case PERIODIC: + break; + } +} + +void setSpecialBoundaryCondition(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + + double mDy = d->grid.dy; + double* u = d->u; + + if (strcmp(d->problem, "dcavity") == 0) { + for (int k = 1; k < kmax; k++) { + for (int i = 1; i < imax; i++) { + U(i, jmax + 1, k) = 2.0 - U(i, jmax, k); + } + } + } else if (strcmp(d->problem, "canal") == 0) { + double ylength = d->grid.ylength; + double y; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + y = mDy * (j - 0.5); + U(0, j, k) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } + } else if (strcmp(d->problem, "backstep") == 0) { + + double* s = d->s; + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + if (S(1, j, k) == FLUID) U(0, j, k) = 1.0; + else { + U(0, j, k) = 0.0; + U(1, j, k) = 0.0; + } + } + } + } else if (strcmp(d->problem, "karman") == 0) { + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(1, j, k) = 1.0; + } + } + } +} + +void computeFG(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* f = d->f; + double* g = d->g; + double* h = d->h; + double* s = d->s; + + double gx = d->gx; + double gy = d->gy; + double gz = d->gz; + double dt = d->dt; + + double gamma = d->gamma; + double inverseRe = 1.0 / d->re; + double inverseDx = 1.0 / d->grid.dx; + double inverseDy = 1.0 / d->grid.dy; + double inverseDz = 1.0 / d->grid.dz; + double du2dx, dv2dy, dw2dz; + double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz; + double du2dx2, du2dy2, du2dz2; + double dv2dx2, dv2dy2, dv2dz2; + double dw2dx2, dw2dy2, dw2dz2; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + if ((int)S(i, j, k) == FLUID) { + du2dx = inverseDx * 0.25 * + ((U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) + U(i + 1, j, k)) - + (U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) + U(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) - U(i + 1, j, k)) + + fabs(U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) - U(i - 1, j, k))); + + duvdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) + U(i, j + 1, k)) - + (V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) + U(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) - U(i, j + 1, k)) + + fabs(V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) - U(i, j - 1, k))); + + duwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) + U(i, j, k + 1)) - + (W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) + U(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) - U(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) - U(i, j, k - 1))); + + du2dx2 = inverseDx * inverseDx * + (U(i + 1, j, k) - 2.0 * U(i, j, k) + U(i - 1, j, k)); + du2dy2 = inverseDy * inverseDy * + (U(i, j + 1, k) - 2.0 * U(i, j, k) + U(i, j - 1, k)); + du2dz2 = inverseDz * inverseDz * + (U(i, j, k + 1) - 2.0 * U(i, j, k) + U(i, j, k - 1)); + F(i, j, k) = U(i, j, k) + + dt * (inverseRe * (du2dx2 + du2dy2 + du2dz2) - du2dx - + duvdy - duwdz + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) + V(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) + V(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) - V(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) - V(i - 1, j, k))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) + V(i, j + 1, k)) - + (V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) + V(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) - V(i, j + 1, k)) + + fabs(V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) - V(i, j - 1, k))); + + dvwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) + V(i, j, k + 1)) - + (W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) + V(i, j, k + 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) - V(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) - V(i, j, k + 1))); + + dv2dx2 = inverseDx * inverseDx * + (V(i + 1, j, k) - 2.0 * V(i, j, k) + V(i - 1, j, k)); + dv2dy2 = inverseDy * inverseDy * + (V(i, j + 1, k) - 2.0 * V(i, j, k) + V(i, j - 1, k)); + dv2dz2 = inverseDz * inverseDz * + (V(i, j, k + 1) - 2.0 * V(i, j, k) + V(i, j, k - 1)); + G(i, j, k) = V(i, j, k) + + dt * (inverseRe * (dv2dx2 + dv2dy2 + dv2dz2) - duvdx - + dv2dy - dvwdz + gy); + + duwdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) + W(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) + W(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) - W(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) - W(i - 1, j, k))); + + dvwdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) + W(i, j + 1, k)) - + (V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) + W(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) - W(i, j + 1, k)) + + fabs(V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) - W(i, j - 1, k))); + + dw2dz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) + W(i, j, k + 1)) - + (W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) + W(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) - W(i, j, k + 1)) + + fabs(W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) - W(i, j, k - 1))); + + dw2dx2 = inverseDx * inverseDx * + (W(i + 1, j, k) - 2.0 * W(i, j, k) + W(i - 1, j, k)); + dw2dy2 = inverseDy * inverseDy * + (W(i, j + 1, k) - 2.0 * W(i, j, k) + W(i, j - 1, k)); + dw2dz2 = inverseDz * inverseDz * + (W(i, j, k + 1) - 2.0 * W(i, j, k) + W(i, j, k - 1)); + H(i, j, k) = W(i, j, k) + + dt * (inverseRe * (dw2dx2 + dw2dy2 + dw2dz2) - duwdx - + dvwdy - dw2dz + gz); + } else { + switch ((int)S(i, j, k)) { + case TOPFACE: + G(i, j, k) = 0.0; + break; + case BOTTOMFACE: + G(i, j, k) = 0.0; + break; + case LEFTFACE: + F(i, j, k) = 0.0; + break; + case RIGHTFACE: + F(i, j, k) = 0.0; + break; + case FRONTFACE: + H(i, j, k) = 0.0; + break; + case BACKFACE: + H(i, j, k) = 0.0; + break; + case FRONTLEFTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTRIGHTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case MIDTOPLEFTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case MIDTOPRIGHTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case MIDBOTTOMLEFTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case MIDBOTTOMRIGHTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case BACKLEFTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKRIGHTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case OBSTACLE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + } + } + } + } + } + /* ----------------------------- boundary of F --------------------------- + */ + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + F(0, j, k) = U(0, j, k); + F(imax, j, k) = U(imax, j, k); + } + } + + /* ----------------------------- boundary of G --------------------------- + */ + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + G(i, 0, k) = V(i, 0, k); + G(i, jmax, k) = V(i, jmax, k); + } + } + + /* ----------------------------- boundary of G --------------------------- + */ + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + H(i, j, 0) = W(i, j, 0); + H(i, j, kmax) = W(i, j, kmax); + } + } +} + +void adaptUV(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + + double* p = d->p; + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* f = d->f; + double* g = d->g; + double* h = d->h; + + double factorX = d->dt / d->grid.dx; + double factorY = d->dt / d->grid.dy; + double factorZ = d->dt / d->grid.dz; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, k) = F(i, j, k) - (P(i + 1, j, k) - P(i, j, k)) * factorX; + V(i, j, k) = G(i, j, k) - (P(i, j + 1, k) - P(i, j, k)) * factorY; + W(i, j, k) = H(i, j, k) - (P(i, j, k + 1) - P(i, j, k)) * factorZ; + } + } + } +} + +void setObjectBoundaryCondition(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* s = d->s; + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + switch ((int)S(i, j, k)) { + case TOPFACE: + U(i, j, k) = -U(i, j + 1, k); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + + break; + case BOTTOMFACE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case LEFTFACE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i - 1, j, k); + break; + case RIGHTFACE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i + 1, j, k); + break; + case FRONTFACE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = 0.0; + break; + case BACKFACE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = 0.0; + break; + case FRONTLEFTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = -W(i - 1, j, k); + break; + case FRONTRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = -W(i + 1, j, k); + break; + case FRONTTOPLINE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTBOTTOMLINE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case MIDTOPLEFTLINE: + U(i, j, k) = -U(i, j + 1, k); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = 0.0; + break; + case MIDTOPRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = 0.0; + U(i - 1, j, k) = -U(i - 1, j + 1, k); + V(i, j - 1, k) = -V(i + 1, j - 1, k); + W(i, j, k) = 0.0; + break; + case MIDBOTTOMLEFTLINE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = 0.0; + break; + case MIDBOTTOMRIGHTLINE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = 0.0; + break; + case BACKLEFTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = -W(i - 1, j, k); + break; + case BACKRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = -W(i + 1, j, k); + break; + case BACKTOPLINE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKBOTTOMLINE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case FRONTTOPLEFTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTTOPRIGHTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTBOTTOMLEFTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case FRONTBOTTOMRIGHTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case BACKTOPLEFTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKTOPRIGHTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKBOTTOMLEFTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case BACKBOTTOMRIGHTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case OBSTACLE: + U(i, j, k) = 0.0; + V(i, j, k) = 0.0; + W(i, j, k) = 0.0; + } + } + } + } +} \ No newline at end of file diff --git a/EnhancedSolver/3D-seq/src/discretization.h b/EnhancedSolver/3D-seq/src/discretization.h new file mode 100644 index 0000000..c86ada5 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/discretization.h @@ -0,0 +1,86 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ + +#include "grid.h" +#include "parameter.h" + +enum OBJECTBOUNDARY { + FLUID = 0, + /* Front Corners */ + FRONTTOPLEFTCORNER, + FRONTTOPRIGHTCORNER, + FRONTBOTTOMLEFTCORNER, + FRONTBOTTOMRIGHTCORNER, + /* Back Corners */ + BACKTOPLEFTCORNER, + BACKTOPRIGHTCORNER, + BACKBOTTOMLEFTCORNER, + BACKBOTTOMRIGHTCORNER, + /* Faces */ + FRONTFACE, + BACKFACE, + LEFTFACE, + RIGHTFACE, + TOPFACE, + BOTTOMFACE, + /* Front Lines remaining after Corners and Faces */ + FRONTLEFTLINE, + FRONTRIGHTLINE, + FRONTTOPLINE, + FRONTBOTTOMLINE, + /* Bottom Lines remaining after Corners and Faces */ + BACKLEFTLINE, + BACKRIGHTLINE, + BACKTOPLINE, + BACKBOTTOMLINE, + /* Mid Lines remaining after Corners and Faces */ + MIDTOPLEFTLINE, + MIDTOPRIGHTLINE, + MIDBOTTOMLEFTLINE, + MIDBOTTOMRIGHTLINE, + /* Local where its an object but not a boundary */ + OBSTACLE, + /*Ghost cells boundary */ + OUTSIDEBOUNDARY +}; + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + + +enum SHAPE { NOSHAPE = 0, RECT, CIRCLE }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g, *h; + double *u, *v, *w; + double* s; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy, gz; + /* time stepping */ + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; +} Discretization; + +extern void initDiscretization(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setObjectBoundaryCondition(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +#endif diff --git a/EnhancedSolver/3D-seq/src/grid.h b/EnhancedSolver/3D-seq/src/grid.h new file mode 100644 index 0000000..0689ebc --- /dev/null +++ b/EnhancedSolver/3D-seq/src/grid.h @@ -0,0 +1,16 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __GRID_H_ +#define __GRID_H_ + +typedef struct { + double dx, dy, dz; + int imax, jmax, kmax; + double xlength, ylength, zlength; +} Grid; + +#endif // __GRID_H_ diff --git a/EnhancedSolver/3D-seq/src/likwid-marker.h b/EnhancedSolver/3D-seq/src/likwid-marker.h new file mode 100644 index 0000000..c3770c0 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/EnhancedSolver/3D-seq/src/main.c b/EnhancedSolver/3D-seq/src/main.c new file mode 100644 index 0000000..08be93c --- /dev/null +++ b/EnhancedSolver/3D-seq/src/main.c @@ -0,0 +1,157 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "allocate.h" +#include "discretization.h" +#include "parameter.h" +#include "particletracing.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" +#include "vtkWriter.h" + +#define G(v, i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +static void createBulkArrays( + Discretization* s, double* pg, double* ug, double* vg, double* wg) +{ + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + int idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + pg[idx++] = G(s->p, i, j, k); + } + } + } + + idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + ug[idx++] = (G(s->u, i, j, k) + G(s->u, i - 1, j, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + vg[idx++] = (G(s->v, i, j, k) + G(s->v, i, j - 1, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + wg[idx++] = (G(s->w, i, j, k) + G(s->w, i, j, k - 1)) / 2.0; + } + } + } +} + +int main(int argc, char** argv) +{ + double timeStart, timeStop; + Parameter p; + Discretization d; + Solver s; + ParticleTracer particletracer; + initParameter(&p); + + FILE* fp; + fp = initResidualWriter(); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(&p, argv[1]); + printParameter(&p); + initDiscretization(&d, &p); + initSolver(&s, &d, &p); + initParticleTracer(&particletracer, &p); + printParticleTracerParameters(&particletracer); + +#ifndef VERBOSE + initProgress(d.te); +#endif + + double tau = d.tau; + double te = d.te; + double t = 0.0; + int nt = 0; + double res = 0.0; + + timeStart = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + setObjectBoundaryCondition(&d); + + computeFG(&d); + computeRHS(&d); + if (nt % 100 == 0) normalizePressure(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + + trace(&particletracer, d.u, d.v, d.w, d.s, t); + + writeResidual(fp, t, res); + + t += d.dt; + nt++; + +#ifdef VERBOSE + printf("TIME %f , TIMESTEP %f\n", t, d.dt); +#else + printProgress(t); +#endif + } + timeStop = getTimeStamp(); + stopProgress(); + printf("Solution took %.2fs\n", timeStop - timeStart); + + timeStart = getTimeStamp(); + double *pg, *ug, *vg, *wg; + + size_t bytesize = (size_t)(d.grid.imax * d.grid.jmax * d.grid.kmax) * sizeof(double); + + pg = allocate(64, bytesize); + ug = allocate(64, bytesize); + vg = allocate(64, bytesize); + wg = allocate(64, bytesize); + + fclose(fp); + freeParticles(&particletracer); + + createBulkArrays(&d, pg, ug, vg, wg); + VtkOptions opts = { .grid = d.grid }; + vtkOpen(&opts, d.problem); + vtkScalar(&opts, "pressure", pg); + vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); + vtkClose(&opts); + + timeStop = getTimeStamp(); + printf("Result output took %.2fs\n", timeStop - timeStart); + + return EXIT_SUCCESS; +} diff --git a/EnhancedSolver/3D-seq/src/parameter.c b/EnhancedSolver/3D-seq/src/parameter.c new file mode 100644 index 0000000..79d5a7c --- /dev/null +++ b/EnhancedSolver/3D-seq/src/parameter.c @@ -0,0 +1,153 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->zlength = 1.0; + param->imax = 100; + param->jmax = 100; + param->kmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_REAL(zlength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(kmax); + PARSE_INT(itermax); + PARSE_INT(levels); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_REAL(gz); + PARSE_STRING(name); + PARSE_INT(bcLeft); + PARSE_INT(bcRight); + PARSE_INT(bcBottom); + PARSE_INT(bcTop); + PARSE_INT(bcFront); + PARSE_INT(bcBack); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(w_init); + PARSE_REAL(p_init); + + /* Added new particle tracing parameters */ + PARSE_INT(numberOfParticles); + PARSE_REAL(startTime); + PARSE_REAL(injectTimePeriod); + PARSE_REAL(writeTimePeriod); + PARSE_REAL(x1); + PARSE_REAL(y1); + PARSE_REAL(z1); + PARSE_REAL(x2); + PARSE_REAL(y2); + PARSE_REAL(z2); + + /* Added obstacle geometry parameters */ + PARSE_INT(shape); + PARSE_REAL(xCenter); + PARSE_REAL(yCenter); + PARSE_REAL(zCenter); + PARSE_REAL(xRectLength); + PARSE_REAL(yRectLength); + PARSE_REAL(zRectLength); + PARSE_REAL(circleRadius); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d Front:%d " + "Back:%d\n", + param->bcLeft, + param->bcRight, + param->bcBottom, + param->bcTop, + param->bcFront, + param->bcBack); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f W:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->w_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + param->xlength, + param->ylength, + param->zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", param->imax, param->jmax, param->kmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); + printf("\tMultiGrid levels : %d\n", param->levels); +} diff --git a/EnhancedSolver/3D-seq/src/parameter.h b/EnhancedSolver/3D-seq/src/parameter.h new file mode 100644 index 0000000..6a54d3a --- /dev/null +++ b/EnhancedSolver/3D-seq/src/parameter.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + int imax, jmax, kmax; + double xlength, ylength, zlength; + int itermax, levels; + double eps, omg, rho; + double re, tau, gamma; + double te, dt; + double gx, gy, gz; + char* name; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + double u_init, v_init, w_init, p_init; + int presmooth, postsmooth; + + int numberOfParticles; + double startTime, injectTimePeriod, writeTimePeriod; + + double x1, y1, z1, x2, y2, z2; + + int shape; + double xCenter, yCenter, zCenter, xRectLength, yRectLength, zRectLength, circleRadius; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/EnhancedSolver/3D-seq/src/particletracing.c b/EnhancedSolver/3D-seq/src/particletracing.c new file mode 100644 index 0000000..72a347f --- /dev/null +++ b/EnhancedSolver/3D-seq/src/particletracing.c @@ -0,0 +1,313 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "discretization.h" +#include "vtkWriter.h" + +#define U(i, j, k) u[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define V(i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define W(i, j, k) w[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define S(i, j, k) s[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +static int ts = 0; +unsigned int seed = 32767; +void printParticles(ParticleTracer* particletracer) +{ + for (int i = 0; i < particletracer->totalParticles; ++i) { + printf("Particle position X : %.2f, Y : %.2f, flag : %d\n", + particletracer->particlePool[i].x, + particletracer->particlePool[i].y, + particletracer->particlePool[i].flag); + } +} +void injectParticles(ParticleTracer* particletracer, double* s) +{ + + int imax = particletracer->imax; + int jmax = particletracer->jmax; + int kmax = particletracer->kmax; + + for (int i = 0; i < particletracer->numberOfParticles; ++i) { + + particletracer->particlePool[particletracer->pointer].x = + (((double)rand() / RAND_MAX) * (particletracer->x2 - particletracer->x1)) + + particletracer->x1; + particletracer->particlePool[particletracer->pointer].y = + (((double)rand() / RAND_MAX) * (particletracer->y2 - particletracer->y1)) + + particletracer->y1; + particletracer->particlePool[particletracer->pointer].z = + (((double)rand() / RAND_MAX) * (particletracer->z2 - particletracer->z1)) + + particletracer->z1; + + int i = particletracer->particlePool[particletracer->pointer].x / + particletracer->dx; + int j = particletracer->particlePool[particletracer->pointer].y / + particletracer->dy; + int k = particletracer->particlePool[particletracer->pointer].z / + particletracer->dz; + + if (S(i, j, k) == FLUID) { + particletracer->particlePool[particletracer->pointer].flag = true; + ++(particletracer->pointer); + ++(particletracer->totalParticles); + } else + particletracer->particlePool[particletracer->pointer].flag = false; + } +} + +void advanceParticles(ParticleTracer* particletracer, + double* restrict u, + double* restrict v, + double* restrict w, + double* restrict s, + double time) +{ + int imax = particletracer->imax; + int jmax = particletracer->jmax; + int kmax = particletracer->kmax; + + double dx = particletracer->dx; + double dy = particletracer->dy; + double dz = particletracer->dz; + + double xlength = particletracer->xlength; + double ylength = particletracer->ylength; + double zlength = particletracer->zlength; + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (particletracer->particlePool[i].flag == true) { + double x = particletracer->particlePool[i].x; + double y = particletracer->particlePool[i].y; + double z = particletracer->particlePool[i].z; + + int iCoord = (int)(x / dx) + 1; + int jCoord = (int)((y + 0.5 * dy) / dy) + 1; + int kCoord = (int)((z + 0.5 * dz) / dz) + 1; + + double x1 = (double)(iCoord - 1) * dx; + double y1 = ((double)(jCoord - 1) - 0.5) * dy; + double z1 = ((double)(kCoord - 1) - 0.5) * dz; + + double x2 = (double)iCoord * dx; + double y2 = ((double)jCoord - 0.5) * dy; + double z2 = ((double)kCoord - 0.5) * dz; + + double u_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * U(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * U(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * U(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * U(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * U(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * U(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * U(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * U(iCoord, jCoord, kCoord)); + + double new_x = x + particletracer->dt * u_n; + particletracer->particlePool[i].x = new_x; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)(y / dy) + 1; + kCoord = (int)((z + 0.5 * dz) / dz) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = (double)(jCoord - 1) * dy; + z1 = ((double)(kCoord - 1) - 0.5) * dz; + + x2 = ((double)iCoord - 0.5) * dx; + y2 = (double)jCoord * dy; + z2 = ((double)kCoord - 0.5) * dz; + + double v_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * V(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * V(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * V(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * V(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * V(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * V(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * V(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * V(iCoord, jCoord, kCoord)); + + double new_y = y + particletracer->dt * v_n; + particletracer->particlePool[i].y = new_y; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)((y + 0.5 * dy) / dy) + 1; + kCoord = (int)(z / dz) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = ((double)(jCoord - 1) - 0.5) * dy; + z1 = (double)(kCoord - 1) * dz; + + x2 = ((double)iCoord - 0.5) * dx; + y2 = ((double)jCoord - 0.5) * dy; + z2 = (double)kCoord * dz; + + double w_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * W(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * W(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * W(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * W(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * W(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * W(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * W(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * W(iCoord, jCoord, kCoord)); + + double new_z = z + particletracer->dt * w_n; + particletracer->particlePool[i].z = new_z; + + if (((new_x < 0.0) || (new_x > xlength) || (new_y < 0.0) || + (new_y > ylength) || (new_z < 0.0) || (new_z > zlength))) { + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + int i_new = new_x / dx, j_new = new_y / dy, k_new = new_z / dz; + + if (S(i_new, j_new, k_new) != FLUID) { + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + } + } +} + +void freeParticles(ParticleTracer* particletracer) { free(particletracer->particlePool); } + +void writeParticles(ParticleTracer* particletracer) +{ + VtkOptions opts = { .particletracer = particletracer }; + compress(particletracer); + + char filename[50]; + snprintf(filename, 50, "vtk_files/particles%d.vtk", ts); + vtkOpenPT(&opts, filename, ts); + vtkParticle(&opts, "particle"); + vtkClose(&opts); + ++ts; +} + +void initParticleTracer(ParticleTracer* particletracer, Parameter* params) +{ + particletracer->numberOfParticles = params->numberOfParticles; + particletracer->startTime = params->startTime; + particletracer->injectTimePeriod = params->injectTimePeriod; + particletracer->writeTimePeriod = params->writeTimePeriod; + + particletracer->dt = params->dt; + particletracer->dx = params->xlength / params->imax; + particletracer->dy = params->ylength / params->jmax; + particletracer->dz = params->zlength / params->kmax; + + particletracer->xlength = params->xlength; + particletracer->ylength = params->ylength; + particletracer->zlength = params->zlength; + + particletracer->x1 = params->x1; + particletracer->y1 = params->y1; + particletracer->z1 = params->z1; + particletracer->x2 = params->x2; + particletracer->y2 = params->y2; + particletracer->z2 = params->z2; + + particletracer->lastInjectTime = params->startTime; + particletracer->lastUpdateTime = params->startTime; + particletracer->lastWriteTime = params->startTime; + + particletracer->pointer = 0; + particletracer->totalParticles = 0; + particletracer->removedParticles = 0; + + particletracer->imax = params->imax; + particletracer->jmax = params->jmax; + particletracer->kmax = params->kmax; + + particletracer->estimatedNumParticles = ((params->te - params->startTime) + 2) * + params->numberOfParticles; + + particletracer->particlePool = malloc( + sizeof(Particle) * particletracer->estimatedNumParticles); +} + +void printParticleTracerParameters(ParticleTracer* particletracer) +{ + printf("Particle Tracing data:\n"); + printf("\tNumber of particles : %d being injected for every period of %.2f\n", + particletracer->numberOfParticles, + particletracer->injectTimePeriod); + printf("\tstartTime : %.2f\n", particletracer->startTime); + printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : " + "%.2f, z1 : %.2f, x2 : %.2f, y2 : %.2f, z2 : %.2f\n", + particletracer->x1, + particletracer->y1, + particletracer->z1, + particletracer->x2, + particletracer->y2, + particletracer->z2); + printf("\tPointer : %d, TotalParticles : %d\n", + particletracer->pointer, + particletracer->totalParticles); + printf("\tdt : %.2f, dx : %.2f, dy : %.2f, dz : %.2f\n", + particletracer->dt, + particletracer->dx, + particletracer->dy, + particletracer->dz); +} + +void trace(ParticleTracer* particletracer, + double* u, + double* v, + double* w, + double* s, + double time) +{ + if (time >= particletracer->startTime) { + // printParticles(particletracer); + if ((time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod) { + injectParticles(particletracer, s); + particletracer->lastInjectTime = time; + } + if ((time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod) { + writeParticles(particletracer); + particletracer->lastWriteTime = time; + } + advanceParticles(particletracer, u, v, w, s, time); + if (particletracer->removedParticles > (particletracer->totalParticles * 0.2)) { + compress(particletracer); + } + particletracer->lastUpdateTime = time; + } +} + +void compress(ParticleTracer* particletracer) +{ + Particle* memPool = particletracer->particlePool; + Particle tempPool[particletracer->totalParticles]; + int totalParticles = 0; + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (memPool[i].flag == true) { + tempPool[totalParticles].x = memPool[i].x; + tempPool[totalParticles].y = memPool[i].y; + tempPool[totalParticles].z = memPool[i].z; + tempPool[totalParticles].flag = memPool[i].flag; + ++totalParticles; + } + } + + particletracer->totalParticles = totalParticles; + particletracer->removedParticles = 0; + particletracer->pointer = totalParticles + 1; + + memcpy(particletracer->particlePool, tempPool, totalParticles * sizeof(Particle)); +} diff --git a/EnhancedSolver/3D-seq/src/particletracing.h b/EnhancedSolver/3D-seq/src/particletracing.h new file mode 100644 index 0000000..5f9eadf --- /dev/null +++ b/EnhancedSolver/3D-seq/src/particletracing.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARTICLETRACING_H_ +#define __PARTICLETRACING_H_ +#include "allocate.h" +#include "parameter.h" + +#include + +typedef enum COORD { X = 0, Y, NCOORD } COORD; +typedef struct { + double x, y, z; + bool flag; +} Particle; + +typedef struct { + int numberOfParticles, totalParticles; + double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime, + lastWriteTime; + + int estimatedNumParticles, activeParticles, removedParticles; + + double dx, dy, dz, dt; + Particle* particlePool; + + int pointer; + + double imax, jmax, kmax, xlength, ylength, zlength; + + double x1, y1, x2, y2, z1, z2; +} ParticleTracer; + +extern void initParticleTracer(ParticleTracer*, Parameter*); +extern void injectParticles(ParticleTracer*, double*); +extern void advanceParticles(ParticleTracer*, double*, double*, double*, double*, double); +extern void freeParticles(ParticleTracer*); +extern void writeParticles(ParticleTracer*); +extern void printParticleTracerParameters(ParticleTracer*); +extern void printParticles(ParticleTracer*); +extern void compress(ParticleTracer*); +extern void trace(ParticleTracer*, double*, double*, double*, double*, double); +#endif \ No newline at end of file diff --git a/EnhancedSolver/3D-seq/src/progress.c b/EnhancedSolver/3D-seq/src/progress.c new file mode 100644 index 0000000..3523426 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/progress.c @@ -0,0 +1,70 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; + +void initProgress(double end) +{ + _end = end; + _current = 0; + + printf("[ ]"); + fflush(stdout); +} + +void printProgress(double current) +{ + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); +} + +void stopProgress() +{ + printf("\n"); + fflush(stdout); +} + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} \ No newline at end of file diff --git a/EnhancedSolver/3D-seq/src/progress.h b/EnhancedSolver/3D-seq/src/progress.h new file mode 100644 index 0000000..240c279 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/progress.h @@ -0,0 +1,15 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ + +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(void); +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); +#endif diff --git a/EnhancedSolver/3D-seq/src/solver-mg.c b/EnhancedSolver/3D-seq/src/solver-mg.c new file mode 100644 index 0000000..743292c --- /dev/null +++ b/EnhancedSolver/3D-seq/src/solver-mg.c @@ -0,0 +1,304 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +#define S(i, j, k) s[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define E(i, j, k) e[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define R(i, j, k) r[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define OLD(i, j, k) old[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +static void restrictMG(Solver* s, int level, int imax, int jmax, int kmax) +{ + double* r = s->r[level + 1]; + double* old = s->r[level]; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; ++i) { + R(i, j, k) = (OLD(2 * i - 1, 2 * j - 1, 2 * k) + + OLD(2 * i, 2 * j - 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k) + + OLD(2 * i - 1, 2 * j, 2 * k) * 2 + + OLD(2 * i, 2 * j, 2 * k) * 8 + + OLD(2 * i + 1, 2 * j, 2 * k) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k) + + OLD(2 * i, 2 * j + 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i, 2 * j - 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i - 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i, 2 * j, 2 * k - 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k - 1) + + OLD(2 * i, 2 * j + 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k - 1) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i, 2 * j - 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i - 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i, 2 * j, 2 * k + 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k + 1) + + OLD(2 * i, 2 * j + 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k + 1)) / + 64.0; + } + } + } +} + +static void prolongate(Solver* s, int level, int imax, int jmax, int kmax) +{ + double* old = s->r[level + 1]; + double* e = s->r[level]; + + for (int k = 2; k < kmax + 1; k += 2) { + for (int j = 2; j < jmax + 1; j += 2) { + for (int i = 2; i < imax + 1; i += 2) { + E(i, j, k) = OLD(i / 2, j / 2, k / 2); + } + } + } +} + +static void correct(Solver* s, double* p, int level, int imax, int jmax, int kmax) +{ + double* e = s->e[level]; + + for (int k = 1; k < kmax + 1; ++k) { + for (int j = 1; j < jmax + 1; ++j) { + for (int i = 1; i < imax + 1; ++i) { + P(i, j, k) += E(i, j, k); + } + } + } +} + +static void setBoundaryCondition(double* p, int imax, int jmax, int kmax) +{ + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } +} + +static double smooth( + Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax) +{ + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + P(i, j, k) -= + factor * + (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +} + +static double calculateResidual( + Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax) +{ + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + R(i, + j, + k) = (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * + idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + + res += (R(i, j, k) * R(i, j, k)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + res = res / (double)(imax * jmax * kmax); + + return res; +} + +static double multiGrid( + Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax) +{ + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, imax, jmax, kmax); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, imax, jmax, kmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); + } + + res = calculateResidual(s, p, rhs, level, imax, jmax, kmax); + + // restrict + restrictMG(s, level, imax, jmax, kmax); + + // MGSolver on residual and error. + multiGrid(s, + s->e[level + 1], + s->r[level + 1], + level + 1, + imax / 2, + jmax / 2, + kmax / 2); + + // prolongate + prolongate(s, level, imax, jmax, kmax); + + // correct p on finer level using residual + correct(s, p, level, imax, jmax, kmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, imax, jmax, kmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + int levels = s->levels; + printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * (kmax + 2); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size * sizeof(double)); + s->e[j] = allocate(64, size * sizeof(double)); + + for (size_t i = 0; i < size; i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax, s->grid->kmax); + +#ifdef VERBOSE + printf("Residuum: %.6f\n", res); +#endif + +return res; +} diff --git a/EnhancedSolver/3D-seq/src/solver-rb.c b/EnhancedSolver/3D-seq/src/solver-rb.c new file mode 100644 index 0000000..6332cce --- /dev/null +++ b/EnhancedSolver/3D-seq/src/solver-rb.c @@ -0,0 +1,101 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->itermax = p->itermax; + s->eps = p->eps; + s->omega = p->omg; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double epssq = eps * eps; + int it = 0; + double res = 1.0; + int pass, ksw, jsw, isw; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + double r = + RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2); + + P(i, j, k) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } + + res = res / (double)(imax * jmax * kmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif + +return res; +} diff --git a/EnhancedSolver/3D-seq/src/solver.h b/EnhancedSolver/3D-seq/src/solver.h new file mode 100644 index 0000000..e5f1b38 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/solver.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "discretization.h" +#include "grid.h" +#include "parameter.h" + +typedef struct { + /* geometry and grid information */ + Grid* grid; + /* parameters */ + double eps, omega, rho; + int itermax; + int levels; + double **r, **e; + int presmooth, postsmooth; +} Solver; + +extern void initSolver(Solver*, Discretization*, Parameter*); +extern double solve(Solver*, double*, double*); + +#endif diff --git a/EnhancedSolver/3D-seq/src/timing.c b/EnhancedSolver/3D-seq/src/timing.c new file mode 100644 index 0000000..78b01c4 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/timing.c @@ -0,0 +1,22 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution(void) +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} diff --git a/EnhancedSolver/3D-seq/src/timing.h b/EnhancedSolver/3D-seq/src/timing.h new file mode 100644 index 0000000..ed05a8c --- /dev/null +++ b/EnhancedSolver/3D-seq/src/timing.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(void); +extern double getTimeResolution(void); + +#endif // __TIMING_H_ diff --git a/EnhancedSolver/3D-seq/src/util.h b/EnhancedSolver/3D-seq/src/util.h new file mode 100644 index 0000000..f2a6eab --- /dev/null +++ b/EnhancedSolver/3D-seq/src/util.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#define P(i, j, k) p[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define F(i, j, k) f[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define G(i, j, k) g[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define H(i, j, k) h[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define U(i, j, k) u[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define V(i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define W(i, j, k) w[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define S(i, j, k) s[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define RHS(i, j, k) rhs[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +#endif // __UTIL_H_ diff --git a/EnhancedSolver/3D-seq/src/vtkWriter.c b/EnhancedSolver/3D-seq/src/vtkWriter.c new file mode 100644 index 0000000..6c6140e --- /dev/null +++ b/EnhancedSolver/3D-seq/src/vtkWriter.c @@ -0,0 +1,198 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "vtkWriter.h" +#define G(v, i, j, k) v[(k)*imax * jmax + (j)*imax + (i)] + +static float floatSwap(float f) +{ + union { + float f; + char b[4]; + } dat1, dat2; + + dat1.f = f; + dat2.b[0] = dat1.b[3]; + dat2.b[1] = dat1.b[2]; + dat2.b[2] = dat1.b[1]; + dat2.b[3] = dat1.b[0]; + return dat2.f; +} + +static void writeHeader(VtkOptions* o) +{ + fprintf(o->fh, "# vtk DataFile Version 3.0\n"); + fprintf(o->fh, "PAMPI cfd solver output\n"); + if (o->fmt == ASCII) { + fprintf(o->fh, "ASCII\n"); + } else if (o->fmt == BINARY) { + fprintf(o->fh, "BINARY\n"); + } + + fprintf(o->fh, "DATASET STRUCTURED_POINTS\n"); + fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid.imax, o->grid.jmax, o->grid.kmax); + fprintf(o->fh, + "ORIGIN %f %f %f\n", + o->grid.dx * 0.5, + o->grid.dy * 0.5, + o->grid.dz * 0.5); + fprintf(o->fh, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz); + fprintf(o->fh, "POINT_DATA %d\n", o->grid.imax * o->grid.jmax * o->grid.kmax); +} + +void vtkOpen(VtkOptions* o, char* problem) +{ + char filename[50]; + snprintf(filename, 50, "%s.vtk", problem); + o->fh = fopen(filename, "w"); + writeHeader(o); + + printf("Writing VTK output for %s\n", problem); +} + +void vtkScalar(VtkOptions* o, char* name, double* s) +{ + int imax = o->grid.imax; + int jmax = o->grid.jmax; + int kmax = o->grid.kmax; + + printf("Register scalar %s\n", name); + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + fprintf(o->fh, "SCALARS %s float\n", name); + fprintf(o->fh, "LOOKUP_TABLE default\n"); + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, "%f\n", G(s, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((float[1]) { floatSwap(G(s, i, j, k)) }, + sizeof(float), + 1, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +void vtkVector(VtkOptions* o, char* name, VtkVector vec) +{ + int imax = o->grid.imax; + int jmax = o->grid.jmax; + int kmax = o->grid.kmax; + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + + fprintf(o->fh, "VECTORS %s float\n", name); + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, + "%f %f %f\n", + G(vec.u, i, j, k), + G(vec.v, i, j, k), + G(vec.w, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((float[3]) { floatSwap(G(vec.u, i, j, k)), + floatSwap(G(vec.v, i, j, k)), + floatSwap(G(vec.w, i, j, k)) }, + sizeof(float), + 3, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +void vtkClose(VtkOptions* o) +{ + fclose(o->fh); + o->fh = NULL; +} + +static void writeHeaderPT(VtkOptions* o, int ts) +{ + fprintf(o->fh, "# vtk DataFile Version 3.0\n"); + fprintf(o->fh, "PAMPI cfd solver particle tracing file\n"); + if (o->fmt == ASCII) { + fprintf(o->fh, "ASCII\n"); + } else if (o->fmt == BINARY) { + fprintf(o->fh, "BINARY\n"); + } + + fprintf(o->fh, "DATASET UNSTRUCTURED_GRID\n"); + fprintf(o->fh, "FIELD FieldData 2\n"); + fprintf(o->fh, "TIME 1 1 double\n"); + fprintf(o->fh, "%d\n", ts); + fprintf(o->fh, "CYCLE 1 1 int\n"); + fprintf(o->fh, "1\n"); +} + +void vtkOpenPT(VtkOptions* o, char* problem, int ts) +{ + o->fh = fopen(problem, "w"); + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + + writeHeaderPT(o, ts); + + // printf("Writing VTK output for %s\n", problem); +} + +void vtkParticle(VtkOptions* o, char* name) +{ + Particle* particlePool = o->particletracer->particlePool; + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + + fprintf(o->fh, "POINTS %d float\n", o->particletracer->totalParticles); + + for (int i = 0; i < o->particletracer->totalParticles; ++i) { + double x = particlePool[i].x; + double y = particlePool[i].y; + double z = particlePool[i].z; + fprintf(o->fh, "%.2f %.2f %.2f\n", x, y, z); + } + + fprintf(o->fh, + "CELLS %d %d\n", + o->particletracer->totalParticles, + 2 * o->particletracer->totalParticles); + + for (int i = 0; i < o->particletracer->totalParticles; ++i) { + fprintf(o->fh, "1 %d\n", i); + } + + fprintf(o->fh, "CELL_TYPES %d\n", o->particletracer->totalParticles); + + for (int i = 0; i < o->particletracer->totalParticles; ++i) { + fprintf(o->fh, "1\n"); + } +} diff --git a/EnhancedSolver/3D-seq/src/vtkWriter.h b/EnhancedSolver/3D-seq/src/vtkWriter.h new file mode 100644 index 0000000..2bc9503 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/vtkWriter.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __VTKWRITER_H_ +#define __VTKWRITER_H_ +#include + +#include "grid.h" +#include "particletracing.h" + +typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat; + +typedef struct VtkOptions { + VtkFormat fmt; + Grid grid; + FILE* fh; + ParticleTracer* particletracer; +} VtkOptions; + +typedef struct VtkVector { + double *u, *v, *w; +} VtkVector; + +extern void vtkOpen(VtkOptions* opts, char* filename); +extern void vtkVector(VtkOptions* opts, char* name, VtkVector vec); +extern void vtkScalar(VtkOptions* opts, char* name, double* p); +extern void vtkClose(VtkOptions* opts); + +extern void vtkOpenPT(VtkOptions* opts, char* filename, int ts); +extern void vtkParticle(VtkOptions* opts, char* name); +#endif // __VTKWRITER_H_ From be3db085aff13a20a4efe6ac87f82db488ed9a91 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 10 Sep 2024 10:54:30 +0200 Subject: [PATCH 2/3] Update BasicSolver/2D-mpi/src/comm-v1.c --- BasicSolver/2D-mpi/src/comm-v1.c | 1 + 1 file changed, 1 insertion(+) diff --git a/BasicSolver/2D-mpi/src/comm-v1.c b/BasicSolver/2D-mpi/src/comm-v1.c index 16a0422..e2de966 100644 --- a/BasicSolver/2D-mpi/src/comm-v1.c +++ b/BasicSolver/2D-mpi/src/comm-v1.c @@ -4,6 +4,7 @@ * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ +#include #include #include "comm.h" From 1a1420890b2b65a84ad5eeb7e01008e479bc2d16 Mon Sep 17 00:00:00 2001 From: Aditya ujeniya Date: Mon, 16 Sep 2024 13:51:10 +0200 Subject: [PATCH 3/3] Requested changes --- BasicSolver/2D-mpi/config.mk | 4 +- .../2D-mpi/{include_ICC.mk => include_ICX.mk} | 6 +- BasicSolver/2D-mpi/src/comm-v1.c | 105 ++++++++---------- BasicSolver/2D-mpi/src/main.c | 20 +++- BasicSolver/2D-mpi/src/progress.c | 19 ---- BasicSolver/2D-mpi/src/progress.h | 2 - BasicSolver/2D-mpi/src/solver-mg.c | 5 +- BasicSolver/2D-mpi/src/solver.c | 104 ----------------- 8 files changed, 75 insertions(+), 190 deletions(-) rename BasicSolver/2D-mpi/{include_ICC.mk => include_ICX.mk} (66%) delete mode 100644 BasicSolver/2D-mpi/src/solver.c diff --git a/BasicSolver/2D-mpi/config.mk b/BasicSolver/2D-mpi/config.mk index eb5546a..e8c9ac2 100644 --- a/BasicSolver/2D-mpi/config.mk +++ b/BasicSolver/2D-mpi/config.mk @@ -1,5 +1,5 @@ -# Supported: GCC, CLANG, ICC -TAG ?= ICC +# Supported: GCC, CLANG, ICX +TAG ?= ICX # Supported: true, false ENABLE_MPI ?= true ENABLE_OPENMP ?= false diff --git a/BasicSolver/2D-mpi/include_ICC.mk b/BasicSolver/2D-mpi/include_ICX.mk similarity index 66% rename from BasicSolver/2D-mpi/include_ICC.mk rename to BasicSolver/2D-mpi/include_ICX.mk index 6bedf55..7b2e6ee 100644 --- a/BasicSolver/2D-mpi/include_ICC.mk +++ b/BasicSolver/2D-mpi/include_ICX.mk @@ -1,8 +1,8 @@ ifeq ($(ENABLE_MPI),true) -CC = mpiicc +CC = mpiicx DEFINES = -D_MPI else -CC = icc +CC = icx endif GCC = gcc @@ -13,7 +13,7 @@ OPENMP = -qopenmp endif VERSION = --version -CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) -Wno-unused-command-line-argument LFLAGS = $(OPENMP) DEFINES += -D_GNU_SOURCE# -DDEBUG INCLUDES = diff --git a/BasicSolver/2D-mpi/src/comm-v1.c b/BasicSolver/2D-mpi/src/comm-v1.c index e2de966..24cd1df 100644 --- a/BasicSolver/2D-mpi/src/comm-v1.c +++ b/BasicSolver/2D-mpi/src/comm-v1.c @@ -10,18 +10,6 @@ #include "comm.h" #ifdef _MPI -// subroutines local to this module -static int sum(int* sizes, int position) -{ - int sum = 0; - - for (int i = 0; i < position; i += position) { - sum += sizes[i]; - } - - return sum; -} - static void gatherArray( Comm* c, int cnt, int* rcvCounts, int* displs, double* src, double* dst) { @@ -42,6 +30,52 @@ static void gatherArray( MPI_COMM_WORLD); } #endif // defined _MPI +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int imax, + int jmax) +{ +#ifdef _MPI + int *rcvCounts, *displs; + int cnt = c->jmaxLocal * (imax + 2); + + if (c->rank == 0) { + rcvCounts = (int*)malloc(c->size * sizeof(int)); + displs = (int*)malloc(c->size * sizeof(int)); + for (int i = 0; i < c->size; ++i) { + rcvCounts[i] = 0; + displs[i] = 0; + } + } + + if (c->rank == 0 && c->size == 1) { + cnt = (c->jmaxLocal + 2) * (imax + 2); + } else if (c->rank == 0 || c->rank == (c->size - 1)) { + cnt = (c->jmaxLocal + 1) * (imax + 2); + } + + MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD); + + if (c->rank == 0) { + displs[0] = 0; + int cursor = rcvCounts[0]; + + for (int i = 1; i < c->size; i++) { + displs[i] = cursor; + cursor += rcvCounts[i]; + } + } + + gatherArray(c, cnt, rcvCounts, displs, p, pg); + gatherArray(c, cnt, rcvCounts, displs, u, ug); + gatherArray(c, cnt, rcvCounts, displs, v, vg); +#endif +} // exported subroutines int commIsBoundary(Comm* c, int direction) @@ -128,49 +162,6 @@ void commShift(Comm* c, double* f, double* g) #endif } -void commCollectResult(Comm* c, - double* ug, - double* vg, - double* pg, - double* u, - double* v, - double* p, - int jmax, - int imax) -{ -#ifdef _MPI - int *rcvCounts, *displs; - int cnt = c->jmaxLocal * (imax + 2); - - if (c->rank == 0) { - rcvCounts = (int*)malloc(c->size * sizeof(int)); - displs = (int*)malloc(c->size * sizeof(int)); - } - - if (c->rank == 0 && c->size == 1) { - cnt = (c->jmaxLocal + 2) * (imax + 2); - } else if (c->rank == 0 || c->rank == (c->size - 1)) { - cnt = (c->jmaxLocal + 1) * (imax + 2); - } - - MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD); - - if (c->rank == 0) { - displs[0] = 0; - int cursor = rcvCounts[0]; - - for (int i = 1; i < c->size; i++) { - displs[i] = cursor; - cursor += rcvCounts[i]; - } - } - - gatherArray(c, cnt, rcvCounts, displs, p, pg); - gatherArray(c, cnt, rcvCounts, displs, u, ug); - gatherArray(c, cnt, rcvCounts, displs, v, vg); -#endif -} - void commPartition(Comm* c, int jmax, int imax) { #ifdef _MPI @@ -211,7 +202,8 @@ void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLo newcomm->jmaxLocal = jmaxLocal / 2; newcomm->neighbours[BOTTOM] = newcomm->rank == 0 ? -1 : newcomm->rank - 1; - newcomm->neighbours[TOP] = newcomm->rank == (newcomm->size - 1) ? -1 : newcomm->rank + 1; + newcomm->neighbours[TOP] = newcomm->rank == (newcomm->size - 1) ? -1 + : newcomm->rank + 1; newcomm->neighbours[LEFT] = -1; newcomm->neighbours[RIGHT] = -1; @@ -221,7 +213,6 @@ void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLo newcomm->dims[IDIM] = 1; newcomm->dims[JDIM] = newcomm->size; - #endif newcomm->imaxLocal = imaxLocal; newcomm->jmaxLocal = jmaxLocal; diff --git a/BasicSolver/2D-mpi/src/main.c b/BasicSolver/2D-mpi/src/main.c index 8f93767..75c58b3 100644 --- a/BasicSolver/2D-mpi/src/main.c +++ b/BasicSolver/2D-mpi/src/main.c @@ -16,6 +16,24 @@ #include "solver.h" #include "timing.h" +static FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; +} + +static void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} + static void writeResults(Discretization* s) { #ifdef _MPI @@ -86,7 +104,7 @@ int main(int argc, char** argv) timeStart = getTimeStamp(); while (t <= te) { - + if (tau > 0.0) computeTimestep(&d); setBoundaryConditions(&d); setSpecialBoundaryCondition(&d); diff --git a/BasicSolver/2D-mpi/src/progress.c b/BasicSolver/2D-mpi/src/progress.c index 4f3b22e..2358e43 100644 --- a/BasicSolver/2D-mpi/src/progress.c +++ b/BasicSolver/2D-mpi/src/progress.c @@ -49,22 +49,3 @@ void stopProgress() printf("\n"); fflush(stdout); } - -FILE* initResidualWriter() -{ - FILE* fp; - fp = fopen("residual.dat", "w"); - - if (fp == NULL) { - printf("Error!\n"); - exit(EXIT_FAILURE); - } - - return fp; - -} - -void writeResidual(FILE* fp, double ts, double res) -{ - fprintf(fp, "%f, %f\n", ts, res); -} diff --git a/BasicSolver/2D-mpi/src/progress.h b/BasicSolver/2D-mpi/src/progress.h index 6803315..b1937cc 100644 --- a/BasicSolver/2D-mpi/src/progress.h +++ b/BasicSolver/2D-mpi/src/progress.h @@ -10,6 +10,4 @@ extern void initProgress(double); extern void printProgress(double); extern void stopProgress(); -extern FILE* initResidualWriter(void); -extern void writeResidual(FILE*, double, double); #endif diff --git a/BasicSolver/2D-mpi/src/solver-mg.c b/BasicSolver/2D-mpi/src/solver-mg.c index 23d64f3..7357278 100644 --- a/BasicSolver/2D-mpi/src/solver-mg.c +++ b/BasicSolver/2D-mpi/src/solver-mg.c @@ -109,7 +109,7 @@ static void setBoundaryCondition(Solver* s, double* p, int imaxLocal, int jmaxLo #endif } -static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm) +static void smooth(Solver* s, double* p, double* rhs, int level, Comm* comm) { int imaxLocal = comm->imaxLocal; int jmaxLocal = comm->jmaxLocal; @@ -228,10 +228,11 @@ static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm // restrict restrictMG(s, level, comm); + // Create a new comm object withupdated imaxLocal and jmaxLocal + // along with their updated bufferTypes, sdispls, rdispls Comm newcomm; commUpdateDatatypes(s->comm, &newcomm, comm->imaxLocal, comm->jmaxLocal); - // MGSolver on residual and error. multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm); diff --git a/BasicSolver/2D-mpi/src/solver.c b/BasicSolver/2D-mpi/src/solver.c deleted file mode 100644 index 654f59d..0000000 --- a/BasicSolver/2D-mpi/src/solver.c +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include - -#include "allocate.h" -#include "comm.h" -#include "discretization.h" -#include "parameter.h" -#include "solver.h" -#include "util.h" - -void initSolver(Solver* s, Discretization* d, Parameter* p) -{ - s->grid = &d->grid; - s->eps = p->eps; - s->omega = p->omg; - s->itermax = p->itermax; - s->comm = &d->comm; -} - -void solve(Solver* s, double* p, double* rhs) -{ - int imax = s->grid->imax; - int jmax = s->grid->jmax; - int imaxLocal = s->comm->imaxLocal; - int jmaxLocal = s->comm->jmaxLocal; - double eps = s->eps; - int itermax = s->itermax; - double dx2 = s->grid->dx * s->grid->dx; - double dy2 = s->grid->dy * s->grid->dy; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); - double epssq = eps * eps; - int pass, jsw, isw; - int it = 0; - double res = 1.0; - - while ((res >= epssq) && (it < itermax)) { - jsw = 1; - for (pass = 0; pass < 2; pass++) { - isw = jsw; - commExchange(s->comm, p); - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = isw; i < imaxLocal + 1; i += 2) { - - double r = RHS(i, j) - - ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + - (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - - P(i, j) -= (factor * r); - res += (r * r); - } - isw = 3 - isw; - } - jsw = 3 - jsw; - } - - if (commIsBoundary(s->comm, BOTTOM)) { // set bottom bc - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, 0) = P(i, 1); - } - } - - if (commIsBoundary(s->comm, TOP)) { // set top bc - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, jmaxLocal + 1) = P(i, jmaxLocal); - } - } - - if (commIsBoundary(s->comm, LEFT)) { // set left bc - for (int j = 1; j < jmaxLocal + 1; j++) { - P(0, j) = P(1, j); - } - } - - if (commIsBoundary(s->comm, RIGHT)) { // set right bc - for (int j = 1; j < jmaxLocal + 1; j++) { - P(imaxLocal + 1, j) = P(imaxLocal, j); - } - } - - commReduction(&res, SUM); - res = res / (double)(imax * jmax); -#ifdef DEBUG - if (commIsMaster(s->comm)) { - printf("%d Residuum: %e\n", it, res); - } -#endif - it++; - } - -#ifdef VERBOSE - if (commIsMaster(s->comm)) { - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); - } -#endif -}