From 213e633a4d24be2afe6008870fd222044c2b6e67 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sun, 5 Feb 2023 07:34:23 +0100 Subject: [PATCH] Initial checkin --- BasicSolver/2D-mpi-v1/Makefile | 71 + BasicSolver/2D-mpi-v1/README.md | 48 + BasicSolver/2D-mpi-v1/canal.par | 46 + BasicSolver/2D-mpi-v1/config.mk | 10 + BasicSolver/2D-mpi-v1/dcavity.par | 46 + BasicSolver/2D-mpi-v1/include_CLANG.mk | 16 + BasicSolver/2D-mpi-v1/include_GCC.mk | 14 + BasicSolver/2D-mpi-v1/include_ICC.mk | 14 + BasicSolver/2D-mpi-v1/src/affinity.c | 61 + BasicSolver/2D-mpi-v1/src/affinity.h | 14 + BasicSolver/2D-mpi-v1/src/allocate.c | 35 + BasicSolver/2D-mpi-v1/src/allocate.h | 13 + BasicSolver/2D-mpi-v1/src/likwid-marker.h | 54 + BasicSolver/2D-mpi-v1/src/main.c | 79 + BasicSolver/2D-mpi-v1/src/parameter.c | 111 + BasicSolver/2D-mpi-v1/src/parameter.h | 26 + BasicSolver/2D-mpi-v1/src/progress.c | 60 + BasicSolver/2D-mpi-v1/src/progress.h | 14 + BasicSolver/2D-mpi-v1/src/solver.c | 689 + BasicSolver/2D-mpi-v1/src/solver.h | 49 + BasicSolver/2D-mpi-v1/src/timing.c | 24 + BasicSolver/2D-mpi-v1/src/timing.h | 14 + BasicSolver/2D-mpi-v1/src/util.h | 23 + BasicSolver/2D-mpi-v1/surface.plot | 7 + BasicSolver/2D-mpi-v1/vector.plot | 5 + BasicSolver/2D-mpi-v2/Makefile | 71 + BasicSolver/2D-mpi-v2/README.md | 48 + BasicSolver/2D-mpi-v2/canal.par | 46 + BasicSolver/2D-mpi-v2/config.mk | 10 + BasicSolver/2D-mpi-v2/dcavity.par | 46 + BasicSolver/2D-mpi-v2/include_CLANG.mk | 16 + BasicSolver/2D-mpi-v2/include_GCC.mk | 14 + BasicSolver/2D-mpi-v2/include_ICC.mk | 14 + BasicSolver/2D-mpi-v2/src/affinity.c | 61 + BasicSolver/2D-mpi-v2/src/affinity.h | 14 + BasicSolver/2D-mpi-v2/src/allocate.c | 35 + BasicSolver/2D-mpi-v2/src/allocate.h | 13 + BasicSolver/2D-mpi-v2/src/likwid-marker.h | 54 + BasicSolver/2D-mpi-v2/src/main.c | 80 + BasicSolver/2D-mpi-v2/src/parameter.c | 108 + BasicSolver/2D-mpi-v2/src/parameter.h | 26 + BasicSolver/2D-mpi-v2/src/progress.c | 60 + BasicSolver/2D-mpi-v2/src/progress.h | 14 + BasicSolver/2D-mpi-v2/src/solver.c | 900 ++ BasicSolver/2D-mpi-v2/src/solver.h | 56 + BasicSolver/2D-mpi-v2/src/timing.c | 24 + BasicSolver/2D-mpi-v2/src/timing.h | 14 + BasicSolver/2D-mpi-v2/src/util.h | 22 + BasicSolver/2D-mpi-v2/surface.plot | 7 + BasicSolver/2D-mpi-v2/vector.plot | 5 + BasicSolver/2D-mpi-v3/Makefile | 71 + BasicSolver/2D-mpi-v3/README.md | 48 + BasicSolver/2D-mpi-v3/canal.par | 46 + BasicSolver/2D-mpi-v3/config.mk | 10 + BasicSolver/2D-mpi-v3/dcavity.par | 46 + BasicSolver/2D-mpi-v3/include_CLANG.mk | 16 + BasicSolver/2D-mpi-v3/include_GCC.mk | 14 + BasicSolver/2D-mpi-v3/include_ICC.mk | 14 + BasicSolver/2D-mpi-v3/src/affinity.c | 61 + BasicSolver/2D-mpi-v3/src/affinity.h | 14 + BasicSolver/2D-mpi-v3/src/allocate.c | 35 + BasicSolver/2D-mpi-v3/src/allocate.h | 13 + BasicSolver/2D-mpi-v3/src/likwid-marker.h | 54 + BasicSolver/2D-mpi-v3/src/main.c | 77 + BasicSolver/2D-mpi-v3/src/parameter.c | 111 + BasicSolver/2D-mpi-v3/src/parameter.h | 26 + BasicSolver/2D-mpi-v3/src/progress.c | 60 + BasicSolver/2D-mpi-v3/src/progress.h | 14 + BasicSolver/2D-mpi-v3/src/solver.c | 832 + BasicSolver/2D-mpi-v3/src/solver.h | 58 + BasicSolver/2D-mpi-v3/src/timing.c | 24 + BasicSolver/2D-mpi-v3/src/timing.h | 14 + BasicSolver/2D-mpi-v3/src/util.h | 22 + BasicSolver/2D-mpi-v3/surface.plot | 7 + BasicSolver/2D-mpi-v3/vector.plot | 5 + BasicSolver/2D-mpi/Makefile | 71 + BasicSolver/2D-mpi/README.md | 48 + BasicSolver/2D-mpi/canal.par | 46 + BasicSolver/2D-mpi/config.mk | 10 + BasicSolver/2D-mpi/dcavity.par | 46 + BasicSolver/2D-mpi/exe-CLANG | Bin 0 -> 69936 bytes BasicSolver/2D-mpi/include_CLANG.mk | 16 + BasicSolver/2D-mpi/include_GCC.mk | 14 + BasicSolver/2D-mpi/include_ICC.mk | 14 + BasicSolver/2D-mpi/pressure.dat | 12505 +++++++++++++++ BasicSolver/2D-mpi/src/affinity.c | 61 + BasicSolver/2D-mpi/src/affinity.h | 14 + BasicSolver/2D-mpi/src/allocate.c | 35 + BasicSolver/2D-mpi/src/allocate.h | 13 + BasicSolver/2D-mpi/src/comm.c | 326 + BasicSolver/2D-mpi/src/comm.h | 44 + BasicSolver/2D-mpi/src/likwid-marker.h | 54 + BasicSolver/2D-mpi/src/main.c | 95 + BasicSolver/2D-mpi/src/parameter.c | 108 + BasicSolver/2D-mpi/src/parameter.h | 26 + BasicSolver/2D-mpi/src/progress.c | 60 + BasicSolver/2D-mpi/src/progress.h | 14 + BasicSolver/2D-mpi/src/solver.c | 546 + BasicSolver/2D-mpi/src/solver.h | 47 + BasicSolver/2D-mpi/src/timing.c | 24 + BasicSolver/2D-mpi/src/timing.h | 14 + BasicSolver/2D-mpi/src/util.h | 22 + BasicSolver/2D-mpi/surface.plot | 7 + BasicSolver/2D-mpi/vector.plot | 5 + BasicSolver/2D-mpi/velocity.dat | 12794 ++++++++++++++++ BasicSolver/2D-mpi/velocity.png | Bin 0 -> 9306 bytes BasicSolver/2D-seq-pt/Makefile | 71 + BasicSolver/2D-seq-pt/README.md | 78 + BasicSolver/2D-seq-pt/animate.plot | 7 + BasicSolver/2D-seq-pt/canal.par | 59 + BasicSolver/2D-seq-pt/config.mk | 12 + BasicSolver/2D-seq-pt/dcavity.par | 46 + BasicSolver/2D-seq-pt/include_CLANG.mk | 17 + BasicSolver/2D-seq-pt/include_GCC.mk | 14 + BasicSolver/2D-seq-pt/include_ICC.mk | 14 + BasicSolver/2D-seq-pt/src/affinity.c | 61 + BasicSolver/2D-seq-pt/src/affinity.h | 14 + BasicSolver/2D-seq-pt/src/allocate.c | 35 + BasicSolver/2D-seq-pt/src/allocate.h | 13 + BasicSolver/2D-seq-pt/src/grid.h | 16 + BasicSolver/2D-seq-pt/src/likwid-marker.h | 54 + BasicSolver/2D-seq-pt/src/main.c | 71 + BasicSolver/2D-seq-pt/src/parameter.c | 129 + BasicSolver/2D-seq-pt/src/parameter.h | 29 + BasicSolver/2D-seq-pt/src/progress.c | 51 + BasicSolver/2D-seq-pt/src/progress.h | 14 + BasicSolver/2D-seq-pt/src/solver.c | 500 + BasicSolver/2D-seq-pt/src/solver.h | 47 + BasicSolver/2D-seq-pt/src/timing.c | 24 + BasicSolver/2D-seq-pt/src/timing.h | 14 + BasicSolver/2D-seq-pt/src/trace.c | 208 + BasicSolver/2D-seq-pt/src/trace.h | 32 + BasicSolver/2D-seq-pt/src/util.h | 23 + BasicSolver/2D-seq-pt/surface.plot | 7 + BasicSolver/2D-seq-pt/vector.plot | 5 + BasicSolver/2D-seq/Makefile | 71 + BasicSolver/2D-seq/README.md | 78 + BasicSolver/2D-seq/canal.par | 46 + BasicSolver/2D-seq/config.mk | 12 + BasicSolver/2D-seq/dcavity.par | 46 + BasicSolver/2D-seq/include_CLANG.mk | 17 + BasicSolver/2D-seq/include_GCC.mk | 14 + BasicSolver/2D-seq/include_ICC.mk | 14 + BasicSolver/2D-seq/src/affinity.c | 61 + BasicSolver/2D-seq/src/affinity.h | 14 + BasicSolver/2D-seq/src/allocate.c | 35 + BasicSolver/2D-seq/src/allocate.h | 13 + BasicSolver/2D-seq/src/likwid-marker.h | 54 + BasicSolver/2D-seq/src/main.c | 66 + BasicSolver/2D-seq/src/parameter.c | 111 + BasicSolver/2D-seq/src/parameter.h | 26 + BasicSolver/2D-seq/src/progress.c | 51 + BasicSolver/2D-seq/src/progress.h | 14 + BasicSolver/2D-seq/src/solver.c | 502 + BasicSolver/2D-seq/src/solver.h | 44 + BasicSolver/2D-seq/src/timing.c | 24 + BasicSolver/2D-seq/src/timing.h | 14 + BasicSolver/2D-seq/src/util.h | 23 + BasicSolver/2D-seq/surface.plot | 7 + BasicSolver/2D-seq/vector.plot | 5 + .../index/affinity.c.B7C05C30E5FF4519.idx | Bin 0 -> 368 bytes .../index/allocate.c.ECF0527ECC6DF781.idx | Bin 0 -> 812 bytes .../index/allocate.h.E842D58079DB450B.idx | Bin 0 -> 390 bytes .../clangd/index/main.c.DC3EC91E37026717.idx | Bin 0 -> 1246 bytes .../index/parameter.c.C63CC045C4D27867.idx | Bin 0 -> 3254 bytes .../index/parameter.h.4BBCC8C05260A054.idx | Bin 0 -> 2124 bytes .../index/progress.c.6A59AB8463A0FDB4.idx | Bin 0 -> 1134 bytes .../index/progress.h.00B7B5207E4C4A3D.idx | Bin 0 -> 388 bytes .../index/solver.c.C754E0A082732D4F.idx | Bin 0 -> 7350 bytes .../index/solver.h.4DC1D59CDF9898D1.idx | Bin 0 -> 4504 bytes .../index/timing.c.3A026594A4A42B70.idx | Bin 0 -> 810 bytes .../index/timing.h.65881B97289F66D9.idx | Bin 0 -> 376 bytes .../clangd/index/util.h.47A029E807954814.idx | Bin 0 -> 264 bytes .../index/vtkWriter.c.D6920E8109B5D2E8.idx | Bin 0 -> 2210 bytes .../index/vtkWriter.h.27987C0C4DAEF54F.idx | Bin 0 -> 540 bytes BasicSolver/3D-mpi/Makefile | 71 + BasicSolver/3D-mpi/README.md | 78 + BasicSolver/3D-mpi/canal.par | 52 + BasicSolver/3D-mpi/config.mk | 12 + BasicSolver/3D-mpi/dcavity.par | 52 + BasicSolver/3D-mpi/include_CLANG.mk | 17 + BasicSolver/3D-mpi/include_GCC.mk | 14 + BasicSolver/3D-mpi/include_ICC.mk | 14 + BasicSolver/3D-mpi/src/allocate.c | 35 + BasicSolver/3D-mpi/src/allocate.h | 13 + BasicSolver/3D-mpi/src/comm.c | 503 + BasicSolver/3D-mpi/src/comm.h | 57 + BasicSolver/3D-mpi/src/grid.h | 16 + BasicSolver/3D-mpi/src/likwid-marker.h | 54 + BasicSolver/3D-mpi/src/main.c | 117 + BasicSolver/3D-mpi/src/parameter.c | 126 + BasicSolver/3D-mpi/src/parameter.h | 26 + BasicSolver/3D-mpi/src/progress.c | 51 + BasicSolver/3D-mpi/src/progress.h | 14 + BasicSolver/3D-mpi/src/solver.c | 845 + BasicSolver/3D-mpi/src/solver.h | 45 + BasicSolver/3D-mpi/src/test.c | 118 + BasicSolver/3D-mpi/src/test.h | 13 + BasicSolver/3D-mpi/src/timing.c | 24 + BasicSolver/3D-mpi/src/timing.h | 14 + BasicSolver/3D-mpi/src/util.h | 22 + BasicSolver/3D-mpi/src/vtkWriter.c | 165 + BasicSolver/3D-mpi/src/vtkWriter.h | 33 + BasicSolver/3D-seq/Makefile | 71 + BasicSolver/3D-seq/README.md | 78 + BasicSolver/3D-seq/canal.par | 52 + BasicSolver/3D-seq/config.mk | 12 + BasicSolver/3D-seq/dcavity.par | 52 + BasicSolver/3D-seq/include_CLANG.mk | 17 + BasicSolver/3D-seq/include_GCC.mk | 14 + BasicSolver/3D-seq/include_ICC.mk | 14 + BasicSolver/3D-seq/src/allocate.c | 38 + BasicSolver/3D-seq/src/allocate.h | 13 + BasicSolver/3D-seq/src/grid.h | 16 + BasicSolver/3D-seq/src/likwid-marker.h | 54 + BasicSolver/3D-seq/src/main.c | 133 + BasicSolver/3D-seq/src/parameter.c | 126 + BasicSolver/3D-seq/src/parameter.h | 26 + BasicSolver/3D-seq/src/progress.c | 51 + BasicSolver/3D-seq/src/progress.h | 14 + BasicSolver/3D-seq/src/solver.c | 722 + BasicSolver/3D-seq/src/solver.h | 44 + BasicSolver/3D-seq/src/timing.c | 22 + BasicSolver/3D-seq/src/timing.h | 13 + BasicSolver/3D-seq/src/util.h | 22 + BasicSolver/3D-seq/src/vtkWriter.c | 132 + BasicSolver/3D-seq/src/vtkWriter.h | 29 + PoissonSolver/2D-mpi/Makefile | 62 + PoissonSolver/2D-mpi/README.md | 48 + PoissonSolver/2D-mpi/config.mk | 9 + PoissonSolver/2D-mpi/include_CLANG.mk | 16 + PoissonSolver/2D-mpi/include_GCC.mk | 14 + PoissonSolver/2D-mpi/include_ICC.mk | 14 + PoissonSolver/2D-mpi/poisson.par | 21 + PoissonSolver/2D-mpi/src/affinity.c | 63 + PoissonSolver/2D-mpi/src/affinity.h | 14 + PoissonSolver/2D-mpi/src/allocate.c | 37 + PoissonSolver/2D-mpi/src/allocate.h | 13 + PoissonSolver/2D-mpi/src/likwid-marker.h | 53 + PoissonSolver/2D-mpi/src/main.c | 44 + PoissonSolver/2D-mpi/src/parameter.c | 72 + PoissonSolver/2D-mpi/src/parameter.h | 20 + PoissonSolver/2D-mpi/src/solver.c | 278 + PoissonSolver/2D-mpi/src/solver.h | 29 + PoissonSolver/2D-mpi/src/timing.c | 27 + PoissonSolver/2D-mpi/src/timing.h | 14 + PoissonSolver/2D-mpi/src/util.h | 21 + PoissonSolver/2D-mpi/surface.plot | 7 + PoissonSolver/2D-seq/Makefile | 62 + PoissonSolver/2D-seq/README.md | 48 + PoissonSolver/2D-seq/config.mk | 9 + PoissonSolver/2D-seq/include_CLANG.mk | 16 + PoissonSolver/2D-seq/include_GCC.mk | 14 + PoissonSolver/2D-seq/include_ICC.mk | 14 + PoissonSolver/2D-seq/poisson.par | 21 + PoissonSolver/2D-seq/src/affinity.c | 63 + PoissonSolver/2D-seq/src/affinity.h | 14 + PoissonSolver/2D-seq/src/allocate.c | 37 + PoissonSolver/2D-seq/src/allocate.h | 13 + PoissonSolver/2D-seq/src/likwid-marker.h | 53 + PoissonSolver/2D-seq/src/main.c | 36 + PoissonSolver/2D-seq/src/parameter.c | 72 + PoissonSolver/2D-seq/src/parameter.h | 20 + PoissonSolver/2D-seq/src/solver.c | 146 + PoissonSolver/2D-seq/src/solver.h | 22 + PoissonSolver/2D-seq/src/timing.c | 27 + PoissonSolver/2D-seq/src/timing.h | 14 + PoissonSolver/2D-seq/src/util.h | 21 + PoissonSolver/2D-seq/surface.plot | 7 + 269 files changed, 41070 insertions(+) create mode 100644 BasicSolver/2D-mpi-v1/Makefile create mode 100644 BasicSolver/2D-mpi-v1/README.md create mode 100644 BasicSolver/2D-mpi-v1/canal.par create mode 100644 BasicSolver/2D-mpi-v1/config.mk create mode 100644 BasicSolver/2D-mpi-v1/dcavity.par create mode 100644 BasicSolver/2D-mpi-v1/include_CLANG.mk create mode 100644 BasicSolver/2D-mpi-v1/include_GCC.mk create mode 100644 BasicSolver/2D-mpi-v1/include_ICC.mk create mode 100644 BasicSolver/2D-mpi-v1/src/affinity.c create mode 100644 BasicSolver/2D-mpi-v1/src/affinity.h create mode 100644 BasicSolver/2D-mpi-v1/src/allocate.c create mode 100644 BasicSolver/2D-mpi-v1/src/allocate.h create mode 100644 BasicSolver/2D-mpi-v1/src/likwid-marker.h create mode 100644 BasicSolver/2D-mpi-v1/src/main.c create mode 100644 BasicSolver/2D-mpi-v1/src/parameter.c create mode 100644 BasicSolver/2D-mpi-v1/src/parameter.h create mode 100644 BasicSolver/2D-mpi-v1/src/progress.c create mode 100644 BasicSolver/2D-mpi-v1/src/progress.h create mode 100644 BasicSolver/2D-mpi-v1/src/solver.c create mode 100644 BasicSolver/2D-mpi-v1/src/solver.h create mode 100644 BasicSolver/2D-mpi-v1/src/timing.c create mode 100644 BasicSolver/2D-mpi-v1/src/timing.h create mode 100644 BasicSolver/2D-mpi-v1/src/util.h create mode 100644 BasicSolver/2D-mpi-v1/surface.plot create mode 100644 BasicSolver/2D-mpi-v1/vector.plot create mode 100644 BasicSolver/2D-mpi-v2/Makefile create mode 100644 BasicSolver/2D-mpi-v2/README.md create mode 100644 BasicSolver/2D-mpi-v2/canal.par create mode 100644 BasicSolver/2D-mpi-v2/config.mk create mode 100644 BasicSolver/2D-mpi-v2/dcavity.par create mode 100644 BasicSolver/2D-mpi-v2/include_CLANG.mk create mode 100644 BasicSolver/2D-mpi-v2/include_GCC.mk create mode 100644 BasicSolver/2D-mpi-v2/include_ICC.mk create mode 100644 BasicSolver/2D-mpi-v2/src/affinity.c create mode 100644 BasicSolver/2D-mpi-v2/src/affinity.h create mode 100644 BasicSolver/2D-mpi-v2/src/allocate.c create mode 100644 BasicSolver/2D-mpi-v2/src/allocate.h create mode 100644 BasicSolver/2D-mpi-v2/src/likwid-marker.h create mode 100644 BasicSolver/2D-mpi-v2/src/main.c create mode 100644 BasicSolver/2D-mpi-v2/src/parameter.c create mode 100644 BasicSolver/2D-mpi-v2/src/parameter.h create mode 100644 BasicSolver/2D-mpi-v2/src/progress.c create mode 100644 BasicSolver/2D-mpi-v2/src/progress.h create mode 100644 BasicSolver/2D-mpi-v2/src/solver.c create mode 100644 BasicSolver/2D-mpi-v2/src/solver.h create mode 100644 BasicSolver/2D-mpi-v2/src/timing.c create mode 100644 BasicSolver/2D-mpi-v2/src/timing.h create mode 100644 BasicSolver/2D-mpi-v2/src/util.h create mode 100644 BasicSolver/2D-mpi-v2/surface.plot create mode 100644 BasicSolver/2D-mpi-v2/vector.plot create mode 100644 BasicSolver/2D-mpi-v3/Makefile create mode 100644 BasicSolver/2D-mpi-v3/README.md create mode 100644 BasicSolver/2D-mpi-v3/canal.par create mode 100644 BasicSolver/2D-mpi-v3/config.mk create mode 100644 BasicSolver/2D-mpi-v3/dcavity.par create mode 100644 BasicSolver/2D-mpi-v3/include_CLANG.mk create mode 100644 BasicSolver/2D-mpi-v3/include_GCC.mk create mode 100644 BasicSolver/2D-mpi-v3/include_ICC.mk create mode 100644 BasicSolver/2D-mpi-v3/src/affinity.c create mode 100644 BasicSolver/2D-mpi-v3/src/affinity.h create mode 100644 BasicSolver/2D-mpi-v3/src/allocate.c create mode 100644 BasicSolver/2D-mpi-v3/src/allocate.h create mode 100644 BasicSolver/2D-mpi-v3/src/likwid-marker.h create mode 100644 BasicSolver/2D-mpi-v3/src/main.c create mode 100644 BasicSolver/2D-mpi-v3/src/parameter.c create mode 100644 BasicSolver/2D-mpi-v3/src/parameter.h create mode 100644 BasicSolver/2D-mpi-v3/src/progress.c create mode 100644 BasicSolver/2D-mpi-v3/src/progress.h create mode 100644 BasicSolver/2D-mpi-v3/src/solver.c create mode 100644 BasicSolver/2D-mpi-v3/src/solver.h create mode 100644 BasicSolver/2D-mpi-v3/src/timing.c create mode 100644 BasicSolver/2D-mpi-v3/src/timing.h create mode 100644 BasicSolver/2D-mpi-v3/src/util.h create mode 100644 BasicSolver/2D-mpi-v3/surface.plot create mode 100644 BasicSolver/2D-mpi-v3/vector.plot create mode 100644 BasicSolver/2D-mpi/Makefile create mode 100644 BasicSolver/2D-mpi/README.md create mode 100644 BasicSolver/2D-mpi/canal.par create mode 100644 BasicSolver/2D-mpi/config.mk create mode 100644 BasicSolver/2D-mpi/dcavity.par create mode 100755 BasicSolver/2D-mpi/exe-CLANG create mode 100644 BasicSolver/2D-mpi/include_CLANG.mk create mode 100644 BasicSolver/2D-mpi/include_GCC.mk create mode 100644 BasicSolver/2D-mpi/include_ICC.mk create mode 100644 BasicSolver/2D-mpi/pressure.dat create mode 100644 BasicSolver/2D-mpi/src/affinity.c create mode 100644 BasicSolver/2D-mpi/src/affinity.h create mode 100644 BasicSolver/2D-mpi/src/allocate.c create mode 100644 BasicSolver/2D-mpi/src/allocate.h create mode 100644 BasicSolver/2D-mpi/src/comm.c create mode 100644 BasicSolver/2D-mpi/src/comm.h create mode 100644 BasicSolver/2D-mpi/src/likwid-marker.h create mode 100644 BasicSolver/2D-mpi/src/main.c create mode 100644 BasicSolver/2D-mpi/src/parameter.c create mode 100644 BasicSolver/2D-mpi/src/parameter.h create mode 100644 BasicSolver/2D-mpi/src/progress.c create mode 100644 BasicSolver/2D-mpi/src/progress.h create mode 100644 BasicSolver/2D-mpi/src/solver.c create mode 100644 BasicSolver/2D-mpi/src/solver.h create mode 100644 BasicSolver/2D-mpi/src/timing.c create mode 100644 BasicSolver/2D-mpi/src/timing.h create mode 100644 BasicSolver/2D-mpi/src/util.h create mode 100644 BasicSolver/2D-mpi/surface.plot create mode 100644 BasicSolver/2D-mpi/vector.plot create mode 100644 BasicSolver/2D-mpi/velocity.dat create mode 100644 BasicSolver/2D-mpi/velocity.png create mode 100644 BasicSolver/2D-seq-pt/Makefile create mode 100644 BasicSolver/2D-seq-pt/README.md create mode 100644 BasicSolver/2D-seq-pt/animate.plot create mode 100644 BasicSolver/2D-seq-pt/canal.par create mode 100644 BasicSolver/2D-seq-pt/config.mk create mode 100644 BasicSolver/2D-seq-pt/dcavity.par create mode 100644 BasicSolver/2D-seq-pt/include_CLANG.mk create mode 100644 BasicSolver/2D-seq-pt/include_GCC.mk create mode 100644 BasicSolver/2D-seq-pt/include_ICC.mk create mode 100644 BasicSolver/2D-seq-pt/src/affinity.c create mode 100644 BasicSolver/2D-seq-pt/src/affinity.h create mode 100644 BasicSolver/2D-seq-pt/src/allocate.c create mode 100644 BasicSolver/2D-seq-pt/src/allocate.h create mode 100644 BasicSolver/2D-seq-pt/src/grid.h create mode 100644 BasicSolver/2D-seq-pt/src/likwid-marker.h create mode 100644 BasicSolver/2D-seq-pt/src/main.c create mode 100644 BasicSolver/2D-seq-pt/src/parameter.c create mode 100644 BasicSolver/2D-seq-pt/src/parameter.h create mode 100644 BasicSolver/2D-seq-pt/src/progress.c create mode 100644 BasicSolver/2D-seq-pt/src/progress.h create mode 100644 BasicSolver/2D-seq-pt/src/solver.c create mode 100644 BasicSolver/2D-seq-pt/src/solver.h create mode 100644 BasicSolver/2D-seq-pt/src/timing.c create mode 100644 BasicSolver/2D-seq-pt/src/timing.h create mode 100644 BasicSolver/2D-seq-pt/src/trace.c create mode 100644 BasicSolver/2D-seq-pt/src/trace.h create mode 100644 BasicSolver/2D-seq-pt/src/util.h create mode 100644 BasicSolver/2D-seq-pt/surface.plot create mode 100644 BasicSolver/2D-seq-pt/vector.plot create mode 100644 BasicSolver/2D-seq/Makefile create mode 100644 BasicSolver/2D-seq/README.md create mode 100644 BasicSolver/2D-seq/canal.par create mode 100644 BasicSolver/2D-seq/config.mk create mode 100644 BasicSolver/2D-seq/dcavity.par create mode 100644 BasicSolver/2D-seq/include_CLANG.mk create mode 100644 BasicSolver/2D-seq/include_GCC.mk create mode 100644 BasicSolver/2D-seq/include_ICC.mk create mode 100644 BasicSolver/2D-seq/src/affinity.c create mode 100644 BasicSolver/2D-seq/src/affinity.h create mode 100644 BasicSolver/2D-seq/src/allocate.c create mode 100644 BasicSolver/2D-seq/src/allocate.h create mode 100644 BasicSolver/2D-seq/src/likwid-marker.h create mode 100644 BasicSolver/2D-seq/src/main.c create mode 100644 BasicSolver/2D-seq/src/parameter.c create mode 100644 BasicSolver/2D-seq/src/parameter.h create mode 100644 BasicSolver/2D-seq/src/progress.c create mode 100644 BasicSolver/2D-seq/src/progress.h create mode 100644 BasicSolver/2D-seq/src/solver.c create mode 100644 BasicSolver/2D-seq/src/solver.h create mode 100644 BasicSolver/2D-seq/src/timing.c create mode 100644 BasicSolver/2D-seq/src/timing.h create mode 100644 BasicSolver/2D-seq/src/util.h create mode 100644 BasicSolver/2D-seq/surface.plot create mode 100644 BasicSolver/2D-seq/vector.plot create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/affinity.c.B7C05C30E5FF4519.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/allocate.c.ECF0527ECC6DF781.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/allocate.h.E842D58079DB450B.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/main.c.DC3EC91E37026717.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/parameter.c.C63CC045C4D27867.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/parameter.h.4BBCC8C05260A054.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/progress.c.6A59AB8463A0FDB4.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/progress.h.00B7B5207E4C4A3D.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/solver.c.C754E0A082732D4F.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/solver.h.4DC1D59CDF9898D1.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/timing.c.3A026594A4A42B70.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/timing.h.65881B97289F66D9.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/util.h.47A029E807954814.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/vtkWriter.c.D6920E8109B5D2E8.idx create mode 100644 BasicSolver/3D-mpi/.cache/clangd/index/vtkWriter.h.27987C0C4DAEF54F.idx create mode 100644 BasicSolver/3D-mpi/Makefile create mode 100644 BasicSolver/3D-mpi/README.md create mode 100644 BasicSolver/3D-mpi/canal.par create mode 100644 BasicSolver/3D-mpi/config.mk create mode 100644 BasicSolver/3D-mpi/dcavity.par create mode 100644 BasicSolver/3D-mpi/include_CLANG.mk create mode 100644 BasicSolver/3D-mpi/include_GCC.mk create mode 100644 BasicSolver/3D-mpi/include_ICC.mk create mode 100644 BasicSolver/3D-mpi/src/allocate.c create mode 100644 BasicSolver/3D-mpi/src/allocate.h create mode 100644 BasicSolver/3D-mpi/src/comm.c create mode 100644 BasicSolver/3D-mpi/src/comm.h create mode 100644 BasicSolver/3D-mpi/src/grid.h create mode 100644 BasicSolver/3D-mpi/src/likwid-marker.h create mode 100644 BasicSolver/3D-mpi/src/main.c create mode 100644 BasicSolver/3D-mpi/src/parameter.c create mode 100644 BasicSolver/3D-mpi/src/parameter.h create mode 100644 BasicSolver/3D-mpi/src/progress.c create mode 100644 BasicSolver/3D-mpi/src/progress.h create mode 100644 BasicSolver/3D-mpi/src/solver.c create mode 100644 BasicSolver/3D-mpi/src/solver.h create mode 100644 BasicSolver/3D-mpi/src/test.c create mode 100644 BasicSolver/3D-mpi/src/test.h create mode 100644 BasicSolver/3D-mpi/src/timing.c create mode 100644 BasicSolver/3D-mpi/src/timing.h create mode 100644 BasicSolver/3D-mpi/src/util.h create mode 100644 BasicSolver/3D-mpi/src/vtkWriter.c create mode 100644 BasicSolver/3D-mpi/src/vtkWriter.h create mode 100644 BasicSolver/3D-seq/Makefile create mode 100644 BasicSolver/3D-seq/README.md create mode 100644 BasicSolver/3D-seq/canal.par create mode 100644 BasicSolver/3D-seq/config.mk create mode 100644 BasicSolver/3D-seq/dcavity.par create mode 100644 BasicSolver/3D-seq/include_CLANG.mk create mode 100644 BasicSolver/3D-seq/include_GCC.mk create mode 100644 BasicSolver/3D-seq/include_ICC.mk create mode 100644 BasicSolver/3D-seq/src/allocate.c create mode 100644 BasicSolver/3D-seq/src/allocate.h create mode 100644 BasicSolver/3D-seq/src/grid.h create mode 100644 BasicSolver/3D-seq/src/likwid-marker.h create mode 100644 BasicSolver/3D-seq/src/main.c create mode 100644 BasicSolver/3D-seq/src/parameter.c create mode 100644 BasicSolver/3D-seq/src/parameter.h create mode 100644 BasicSolver/3D-seq/src/progress.c create mode 100644 BasicSolver/3D-seq/src/progress.h create mode 100644 BasicSolver/3D-seq/src/solver.c create mode 100644 BasicSolver/3D-seq/src/solver.h create mode 100644 BasicSolver/3D-seq/src/timing.c create mode 100644 BasicSolver/3D-seq/src/timing.h create mode 100644 BasicSolver/3D-seq/src/util.h create mode 100644 BasicSolver/3D-seq/src/vtkWriter.c create mode 100644 BasicSolver/3D-seq/src/vtkWriter.h create mode 100644 PoissonSolver/2D-mpi/Makefile create mode 100644 PoissonSolver/2D-mpi/README.md create mode 100644 PoissonSolver/2D-mpi/config.mk create mode 100644 PoissonSolver/2D-mpi/include_CLANG.mk create mode 100644 PoissonSolver/2D-mpi/include_GCC.mk create mode 100644 PoissonSolver/2D-mpi/include_ICC.mk create mode 100644 PoissonSolver/2D-mpi/poisson.par create mode 100644 PoissonSolver/2D-mpi/src/affinity.c create mode 100644 PoissonSolver/2D-mpi/src/affinity.h create mode 100644 PoissonSolver/2D-mpi/src/allocate.c create mode 100644 PoissonSolver/2D-mpi/src/allocate.h create mode 100644 PoissonSolver/2D-mpi/src/likwid-marker.h create mode 100644 PoissonSolver/2D-mpi/src/main.c create mode 100644 PoissonSolver/2D-mpi/src/parameter.c create mode 100644 PoissonSolver/2D-mpi/src/parameter.h create mode 100644 PoissonSolver/2D-mpi/src/solver.c create mode 100644 PoissonSolver/2D-mpi/src/solver.h create mode 100644 PoissonSolver/2D-mpi/src/timing.c create mode 100644 PoissonSolver/2D-mpi/src/timing.h create mode 100644 PoissonSolver/2D-mpi/src/util.h create mode 100644 PoissonSolver/2D-mpi/surface.plot create mode 100644 PoissonSolver/2D-seq/Makefile create mode 100644 PoissonSolver/2D-seq/README.md create mode 100644 PoissonSolver/2D-seq/config.mk create mode 100644 PoissonSolver/2D-seq/include_CLANG.mk create mode 100644 PoissonSolver/2D-seq/include_GCC.mk create mode 100644 PoissonSolver/2D-seq/include_ICC.mk create mode 100644 PoissonSolver/2D-seq/poisson.par create mode 100644 PoissonSolver/2D-seq/src/affinity.c create mode 100644 PoissonSolver/2D-seq/src/affinity.h create mode 100644 PoissonSolver/2D-seq/src/allocate.c create mode 100644 PoissonSolver/2D-seq/src/allocate.h create mode 100644 PoissonSolver/2D-seq/src/likwid-marker.h create mode 100644 PoissonSolver/2D-seq/src/main.c create mode 100644 PoissonSolver/2D-seq/src/parameter.c create mode 100644 PoissonSolver/2D-seq/src/parameter.h create mode 100644 PoissonSolver/2D-seq/src/solver.c create mode 100644 PoissonSolver/2D-seq/src/solver.h create mode 100644 PoissonSolver/2D-seq/src/timing.c create mode 100644 PoissonSolver/2D-seq/src/timing.h create mode 100644 PoissonSolver/2D-seq/src/util.h create mode 100644 PoissonSolver/2D-seq/surface.plot diff --git a/BasicSolver/2D-mpi-v1/Makefile b/BasicSolver/2D-mpi-v1/Makefile new file mode 100644 index 0000000..57f99f4 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/Makefile @@ -0,0 +1,71 @@ +#======================================================================================= +# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(wildcard $(SRC_DIR)/*.c) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm format + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/BasicSolver/2D-mpi-v1/README.md b/BasicSolver/2D-mpi-v1/README.md new file mode 100644 index 0000000..b0a80a6 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/README.md @@ -0,0 +1,48 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution. + + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. diff --git a/BasicSolver/2D-mpi-v1/canal.par b/BasicSolver/2D-mpi-v1/canal.par new file mode 100644 index 0000000..c90294c --- /dev/null +++ b/BasicSolver/2D-mpi-v1/canal.par @@ -0,0 +1,46 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 100.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 50 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 100.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.00001 # stopping tolerance for pressure iteration +omg 1.8 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-mpi-v1/config.mk b/BasicSolver/2D-mpi-v1/config.mk new file mode 100644 index 0000000..46cef95 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/config.mk @@ -0,0 +1,10 @@ +# Supported: GCC, CLANG, ICC +TAG ?= CLANG +ENABLE_OPENMP ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/2D-mpi-v1/dcavity.par b/BasicSolver/2D-mpi-v1/dcavity.par new file mode 100644 index 0000000..b5c22ce --- /dev/null +++ b/BasicSolver/2D-mpi-v1/dcavity.par @@ -0,0 +1,46 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcLeft 1 # flags for boundary conditions +bcRight 1 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 500.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +imax 100 # number of interior cells in x-direction +jmax 100 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 25.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-mpi-v1/include_CLANG.mk b/BasicSolver/2D-mpi-v1/include_CLANG.mk new file mode 100644 index 0000000..1d17c52 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/include_CLANG.mk @@ -0,0 +1,16 @@ +CC = mpicc +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +CFLAGS = -Ofast -std=c99 $(OPENMP) +#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE# -DDEBUG +INCLUDES = -I/usr/local/include diff --git a/BasicSolver/2D-mpi-v1/include_GCC.mk b/BasicSolver/2D-mpi-v1/include_GCC.mk new file mode 100644 index 0000000..427e798 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/include_GCC.mk @@ -0,0 +1,14 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/2D-mpi-v1/include_ICC.mk b/BasicSolver/2D-mpi-v1/include_ICC.mk new file mode 100644 index 0000000..f85d836 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/include_ICC.mk @@ -0,0 +1,14 @@ +CC = mpiicc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/2D-mpi-v1/src/affinity.c b/BasicSolver/2D-mpi-v1/src/affinity.c new file mode 100644 index 0000000..b501665 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/affinity.c @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifdef __linux__ +#ifdef _OPENMP +#include +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_THREADS 128 +#define gettid() syscall(SYS_gettid) + +static int getProcessorID(cpu_set_t* cpu_set) +{ + int processorId; + + for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { + if (CPU_ISSET(processorId, cpu_set)) { + break; + } + } + return processorId; +} + +int affinity_getProcessorId() +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); + + return getProcessorID(&cpu_set); +} + +void affinity_pinThread(int processorId) +{ + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); +} + +void affinity_pinProcess(int processorId) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} +#endif /*_OPENMP*/ +#endif /*__linux__*/ diff --git a/BasicSolver/2D-mpi-v1/src/affinity.h b/BasicSolver/2D-mpi-v1/src/affinity.h new file mode 100644 index 0000000..d844fe5 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/affinity.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef AFFINITY_H +#define AFFINITY_H + +extern int affinity_getProcessorId(); +extern void affinity_pinProcess(int); +extern void affinity_pinThread(int); + +#endif /*AFFINITY_H*/ diff --git a/BasicSolver/2D-mpi-v1/src/allocate.c b/BasicSolver/2D-mpi-v1/src/allocate.c new file mode 100644 index 0000000..81e1e9d --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/allocate.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +void* allocate(int alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/BasicSolver/2D-mpi-v1/src/allocate.h b/BasicSolver/2D-mpi-v1/src/allocate.h new file mode 100644 index 0000000..54cfe06 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(int alignment, size_t bytesize); + +#endif diff --git a/BasicSolver/2D-mpi-v1/src/likwid-marker.h b/BasicSolver/2D-mpi-v1/src/likwid-marker.h new file mode 100644 index 0000000..eb7cc78 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/BasicSolver/2D-mpi-v1/src/main.c b/BasicSolver/2D-mpi-v1/src/main.c new file mode 100644 index 0000000..d685367 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/main.c @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include +#include + +#include "parameter.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" +#include + +int main(int argc, char** argv) +{ + int rank; + double S, E; + Parameter params; + Solver solver; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + initParameter(¶ms); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(¶ms, argv[1]); + if (rank == 0) { + printParameter(¶ms); + } + initSolver(&solver, ¶ms); + initProgress(solver.te); + + double tau = solver.tau; + double te = solver.te; + double t = 0.0; + + S = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) { + computeTimestep(&solver); + } + + setBoundaryConditions(&solver); + setSpecialBoundaryCondition(&solver); + computeFG(&solver); + computeRHS(&solver); + solve(&solver); + adaptUV(&solver); + /* exit(EXIT_SUCCESS); */ + t += solver.dt; + +#ifdef VERBOSE + if (rank == 0) { + printf("TIME %f , TIMESTEP %f\n", t, solver.dt); + } +#else + printProgress(t); +#endif + } + E = getTimeStamp(); + stopProgress(); + if (rank == 0) { + printf("Solution took %.2fs\n", E - S); + } + collectResult(&solver); + + MPI_Finalize(); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/2D-mpi-v1/src/parameter.c b/BasicSolver/2D-mpi-v1/src/parameter.c new file mode 100644 index 0000000..d691627 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/parameter.c @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_STRING(name); + PARSE_INT(bcLeft); + PARSE_INT(bcRight); + PARSE_INT(bcBottom); + PARSE_INT(bcTop); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(p_init); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + param->bcLeft, + param->bcRight, + param->bcBottom, + param->bcTop); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength); + printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); +} diff --git a/BasicSolver/2D-mpi-v1/src/parameter.h b/BasicSolver/2D-mpi-v1/src/parameter.h new file mode 100644 index 0000000..f4c331a --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/parameter.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + double xlength, ylength; + int imax, jmax; + int itermax; + double eps, omg; + double re, tau, gamma; + double te, dt; + double gx, gy; + char* name; + int bcLeft, bcRight, bcBottom, bcTop; + double u_init, v_init, p_init; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/BasicSolver/2D-mpi-v1/src/progress.c b/BasicSolver/2D-mpi-v1/src/progress.c new file mode 100644 index 0000000..31a8a90 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/progress.c @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; +static int _rank = -1; + +void initProgress(double end) +{ + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); + _end = end; + _current = 0; + + if (_rank == 0) { + printf("[ ]"); + fflush(stdout); + } +} + +void printProgress(double current) +{ + if (_rank == 0) { + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); + } +} + +void stopProgress() +{ + if (_rank == 0) { + printf("\n"); + fflush(stdout); + } +} diff --git a/BasicSolver/2D-mpi-v1/src/progress.h b/BasicSolver/2D-mpi-v1/src/progress.h new file mode 100644 index 0000000..9ef2d96 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/progress.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ + +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(); + +#endif diff --git a/BasicSolver/2D-mpi-v1/src/solver.c b/BasicSolver/2D-mpi-v1/src/solver.c new file mode 100644 index 0000000..6bc575d --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/solver.c @@ -0,0 +1,689 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +#define P(i, j) p[(j) * (imax + 2) + (i)] +#define F(i, j) f[(j) * (imax + 2) + (i)] +#define G(i, j) g[(j) * (imax + 2) + (i)] +#define U(i, j) u[(j) * (imax + 2) + (i)] +#define V(i, j) v[(j) * (imax + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imax + 2) + (i)] + +static int sizeOfRank(int rank, int size, int N) +{ + return N / size + ((N % size > rank) ? 1 : 0); +} + +static void print(Solver* solver, double* grid) +{ + int imax = solver->imax; + + for (int i = 0; i < solver->size; i++) { + if (i == solver->rank) { + printf("### RANK %d " + "#######################################################\n", + solver->rank); + for (int j = 0; j < solver->jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < solver->imax + 2; i++) { + printf("%12.8f ", grid[j * (imax + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + MPI_Barrier(MPI_COMM_WORLD); + } +} + +static void exchange(Solver* solver, double* grid) +{ + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* exchange ghost cells with top neighbor */ + if (solver->rank + 1 < solver->size) { + int top = solver->rank + 1; + double* src = grid + (solver->jmaxLocal) * (solver->imax + 2) + 1; + double* dst = grid + (solver->jmaxLocal + 1) * (solver->imax + 2) + 1; + + MPI_Isend(src, solver->imax, MPI_DOUBLE, top, 1, MPI_COMM_WORLD, &requests[0]); + MPI_Irecv(dst, solver->imax, MPI_DOUBLE, top, 2, MPI_COMM_WORLD, &requests[1]); + } + + /* exchange ghost cells with bottom neighbor */ + if (solver->rank > 0) { + int bottom = solver->rank - 1; + double* src = grid + (solver->imax + 2) + 1; + double* dst = grid + 1; + + MPI_Isend(src, solver->imax, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]); + MPI_Irecv(dst, solver->imax, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]); + } + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +} + +static void shift(Solver* solver) +{ + MPI_Request requests[2] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL }; + double* g = solver->g; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + if (solver->rank > 0) { + int bottom = solver->rank - 1; + MPI_Irecv(g + 1, + solver->imax, + MPI_DOUBLE, + bottom, + 0, + MPI_COMM_WORLD, + &requests[0]); + } + + if (solver->rank + 1 < solver->size) { + int top = solver->rank + 1; + double* buf = g + (solver->jmaxLocal) * (solver->imax + 2) + 1; + /* send ghost cells to top neighbor */ + MPI_Isend(buf, solver->imax, MPI_DOUBLE, top, 0, MPI_COMM_WORLD, &requests[1]); + } + + MPI_Waitall(2, requests, MPI_STATUSES_IGNORE); +} + +void collectResult(Solver* solver) +{ + double* Pall = NULL; + double* Uall = NULL; + double* Vall = NULL; + int *rcvCounts, *displs; + + if (solver->rank == 0) { + Pall = allocate(64, (solver->imax + 2) * (solver->jmax + 2) * sizeof(double)); + Uall = allocate(64, (solver->imax + 2) * (solver->jmax + 2) * sizeof(double)); + Vall = allocate(64, (solver->imax + 2) * (solver->jmax + 2) * sizeof(double)); + rcvCounts = (int*)malloc(solver->size * sizeof(int)); + displs = (int*)malloc(solver->size * sizeof(int)); + rcvCounts[0] = solver->jmaxLocal * (solver->imax + 2); + displs[0] = 0; + int cursor = rcvCounts[0]; + + for (int i = 1; i < solver->size; i++) { + rcvCounts[i] = sizeOfRank(i, solver->size, solver->jmax) * (solver->imax + 2); + displs[i] = cursor; + cursor += rcvCounts[i]; + } + } + + int cnt = solver->jmaxLocal * (solver->imax + 2); + double* sendbuffer = solver->p + (solver->imax + 2); + MPI_Gatherv(sendbuffer, + cnt, + MPI_DOUBLE, + Pall, + rcvCounts, + displs, + MPI_DOUBLE, + 0, + MPI_COMM_WORLD); + sendbuffer = solver->u + (solver->imax + 2); + MPI_Gatherv(sendbuffer, + cnt, + MPI_DOUBLE, + Uall, + rcvCounts, + displs, + MPI_DOUBLE, + 0, + MPI_COMM_WORLD); + sendbuffer = solver->v + (solver->imax + 2); + MPI_Gatherv(sendbuffer, + cnt, + MPI_DOUBLE, + Vall, + rcvCounts, + displs, + MPI_DOUBLE, + 0, + MPI_COMM_WORLD); + + if (solver->rank == 0) { + writeResult(solver, Pall, Uall, Vall); + } +} + +static void printConfig(Solver* solver) +{ + if (solver->rank == 0) { + printf("Parameters for #%s#\n", solver->problem); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + solver->bcLeft, + solver->bcRight, + solver->bcBottom, + solver->bcTop); + printf("\tReynolds number: %.2f\n", solver->re); + printf("\tGx Gy: %.2f %.2f\n", solver->gx, solver->gy); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", + solver->xlength, + solver->ylength); + printf("\tCells (x, y): %d, %d\n", solver->imax, solver->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", solver->dt, solver->te); + printf("\tdt bound: %.6f\n", solver->dtBound); + printf("\tTau factor: %.2f\n", solver->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", solver->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", solver->eps); + printf("\tgamma factor: %f\n", solver->gamma); + printf("\tomega (SOR relaxation): %f\n", solver->omega); + printf("Communication parameters:\n"); + } + for (int i = 0; i < solver->size; i++) { + if (i == solver->rank) { + printf("\tRank %d of %d\n", solver->rank, solver->size); + printf("\tLocal domain size: %dx%d\n", solver->imax, solver->jmaxLocal); + fflush(stdout); + } + } +} + +void initSolver(Solver* solver, Parameter* params) +{ + MPI_Comm_rank(MPI_COMM_WORLD, &(solver->rank)); + MPI_Comm_size(MPI_COMM_WORLD, &(solver->size)); + solver->problem = params->name; + solver->bcLeft = params->bcLeft; + solver->bcRight = params->bcRight; + solver->bcBottom = params->bcBottom; + solver->bcTop = params->bcTop; + solver->imax = params->imax; + solver->jmax = params->jmax; + solver->jmaxLocal = sizeOfRank(solver->rank, solver->size, solver->jmax); + solver->xlength = params->xlength; + solver->ylength = params->ylength; + solver->dx = params->xlength / params->imax; + solver->dy = params->ylength / params->jmax; + solver->eps = params->eps; + solver->omega = params->omg; + solver->itermax = params->itermax; + solver->re = params->re; + solver->gx = params->gx; + solver->gy = params->gy; + solver->dt = params->dt; + solver->te = params->te; + solver->tau = params->tau; + solver->gamma = params->gamma; + + int imax = solver->imax; + int jmaxLocal = solver->jmaxLocal; + size_t bytesize = (imax + 2) * (jmaxLocal + 2) * sizeof(double); + solver->u = allocate(64, bytesize); + solver->v = allocate(64, bytesize); + solver->p = allocate(64, bytesize); + solver->rhs = allocate(64, bytesize); + solver->f = allocate(64, bytesize); + solver->g = allocate(64, bytesize); + + for (int i = 0; i < (imax + 2) * (jmaxLocal + 2); i++) { + solver->u[i] = params->u_init; + solver->v[i] = params->v_init; + solver->p[i] = params->p_init; + solver->rhs[i] = 0.0; + solver->f[i] = 0.0; + solver->g[i] = 0.0; + } + + double dx = solver->dx; + double dy = solver->dy; + double inv_sqr_sum = 1.0 / (dx * dx) + 1.0 / (dy * dy); + solver->dtBound = 0.5 * solver->re * 1.0 / inv_sqr_sum; +#ifdef VERBOSE + printConfig(solver); +#endif +} + +void computeRHS(Solver* solver) +{ + int imax = solver->imax; + int jmaxLocal = solver->jmaxLocal; + double idx = 1.0 / solver->dx; + double idy = 1.0 / solver->dy; + double idt = 1.0 / solver->dt; + double* rhs = solver->rhs; + double* f = solver->f; + double* g = solver->g; + + shift(solver); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) * + idt; + } + } +} + +void solve(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + int jmaxLocal = solver->jmaxLocal; + double eps = solver->eps; + int itermax = solver->itermax; + double dx2 = solver->dx * solver->dx; + double dy2 = solver->dy * solver->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* p = solver->p; + double* rhs = solver->rhs; + double epssq = eps * eps; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + exchange(solver, p); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + } + + if (solver->rank == 0) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (solver->rank == (solver->size - 1)) { + for (int i = 1; i < imax + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } + + MPI_Allreduce(MPI_IN_PLACE, &res, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (solver->rank == 0) { + printf("%d Residuum: %e\n", it, res); + } +#endif + it++; + } + +#ifdef VERBOSE + if (solver->rank == 0) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif +} + +static double maxElement(Solver* solver, double* m) +{ + int size = (solver->imax + 2) * (solver->jmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + MPI_Allreduce(MPI_IN_PLACE, &maxval, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + return maxval; +} + +void normalizePressure(Solver* solver) +{ + int size = (solver->imax + 2) * (solver->jmaxLocal + 2); + double* p = solver->p; + double avgP = 0.0; + + for (int i = 0; i < size; i++) { + avgP += p[i]; + } + MPI_Allreduce(MPI_IN_PLACE, &avgP, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + avgP /= (solver->imax + 2) * (solver->jmax + 2); + + for (int i = 0; i < size; i++) { + p[i] = p[i] - avgP; + } +} + +void computeTimestep(Solver* solver) +{ + double dt = solver->dtBound; + double dx = solver->dx; + double dy = solver->dy; + double umax = maxElement(solver, solver->u); + double vmax = maxElement(solver, solver->v); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + + solver->dt = dt * solver->tau; +} + +void setBoundaryConditions(Solver* solver) +{ + int imax = solver->imax; + int jmaxLocal = solver->jmaxLocal; + double* u = solver->u; + double* v = solver->v; + + // Left boundary + switch (solver->bcLeft) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = -V(1, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = V(1, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = U(1, j); + V(0, j) = V(1, j); + } + break; + case PERIODIC: + break; + } + + // Right boundary + switch (solver->bcRight) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imax, j) = 0.0; + V(imax + 1, j) = -V(imax, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imax, j) = 0.0; + V(imax + 1, j) = V(imax, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imax, j) = U(imax - 1, j); + V(imax + 1, j) = V(imax, j); + } + break; + case PERIODIC: + break; + } + + // Bottom boundary + if (solver->rank == 0) { + switch (solver->bcBottom) { + case NOSLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = -U(i, 1); + } + break; + case SLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = U(i, 1); + } + break; + case OUTFLOW: + for (int i = 1; i < imax + 1; i++) { + U(i, 0) = U(i, 1); + V(i, 0) = V(i, 1); + } + break; + case PERIODIC: + break; + } + } + + // Top boundary + if (solver->rank == (solver->size - 1)) { + switch (solver->bcTop) { + case NOSLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = -U(i, jmaxLocal); + } + break; + case SLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + } + break; + case OUTFLOW: + for (int i = 1; i < imax + 1; i++) { + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + V(i, jmaxLocal) = V(i, jmaxLocal - 1); + } + break; + case PERIODIC: + break; + } + } +} + +void setSpecialBoundaryCondition(Solver* solver) +{ + int imax = solver->imax; + int jmaxLocal = solver->jmaxLocal; + double* u = solver->u; + + if (strcmp(solver->problem, "dcavity") == 0) { + if (solver->rank == (solver->size - 1)) { + for (int i = 1; i < imax; i++) { + U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal); + } + } + } else if (strcmp(solver->problem, "canal") == 0) { + double ylength = solver->ylength; + double dy = solver->dy; + int rest = solver->jmax % solver->size; + int yc = solver->rank * (solver->jmax / solver->size) + MIN(rest, solver->rank); + double ys = dy * (yc + 0.5); + double y; + + /* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */ + + for (int j = 1; j < jmaxLocal + 1; j++) { + y = ys + dy * (j - 0.5); + U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } + /* print(solver, solver->u); */ +} + +void computeFG(Solver* solver) +{ + double* u = solver->u; + double* v = solver->v; + double* f = solver->f; + double* g = solver->g; + int imax = solver->imax; + int jmaxLocal = solver->jmaxLocal; + double gx = solver->gx; + double gy = solver->gy; + double gamma = solver->gamma; + double dt = solver->dt; + double inverseRe = 1.0 / solver->re; + double inverseDx = 1.0 / solver->dx; + double inverseDy = 1.0 / solver->dy; + double du2dx, dv2dy, duvdx, duvdy; + double du2dx2, du2dy2, dv2dx2, dv2dy2; + + exchange(solver, u); + exchange(solver, v); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + du2dx = inverseDx * 0.25 * + ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - + (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + + fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); + + duvdy = inverseDy * 0.25 * + ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - + (V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + + fabs(V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) - U(i, j - 1))); + + du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); + du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); + F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - + (U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + + fabs(U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) - V(i - 1, j))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - + (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + + fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); + + dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); + dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); + G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); + } + } + + /* ----------------------------- boundary of F --------------------------- + */ + for (int j = 1; j < jmaxLocal + 1; j++) { + F(0, j) = U(0, j); + F(imax, j) = U(imax, j); + } + + /* ----------------------------- boundary of G --------------------------- + */ + if (solver->rank == 0) { + for (int i = 1; i < imax + 1; i++) { + G(i, 0) = V(i, 0); + } + } + + if (solver->rank == (solver->size - 1)) { + for (int i = 1; i < imax + 1; i++) { + G(i, jmaxLocal) = V(i, jmaxLocal); + } + } +} + +void adaptUV(Solver* solver) +{ + int imax = solver->imax; + int jmaxLocal = solver->jmaxLocal; + double* p = solver->p; + double* u = solver->u; + double* v = solver->v; + double* f = solver->f; + double* g = solver->g; + double factorX = solver->dt / solver->dx; + double factorY = solver->dt / solver->dy; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; + V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; + } + } +} + +void writeResult(Solver* solver, double* p, double* u, double* v) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double dx = solver->dx; + double dy = solver->dy; + double x = 0.0, y = 0.0; + + FILE* fp; + fp = fopen("pressure.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax + 1; j++) { + y = (double)(j - 0.5) * dy; + for (int i = 1; i < imax + 1; i++) { + x = (double)(i - 0.5) * dx; + fprintf(fp, "%.2f %.2f %f\n", x, y, P(i, j)); + } + fprintf(fp, "\n"); + } + + fclose(fp); + + fp = fopen("velocity.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax + 1; j++) { + y = dy * (j - 0.5); + for (int i = 1; i < imax + 1; i++) { + x = dx * (i - 0.5); + double vel_u = (U(i, j) + U(i - 1, j)) / 2.0; + double vel_v = (V(i, j) + V(i, j - 1)) / 2.0; + double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); + } + } + + fclose(fp); +} diff --git a/BasicSolver/2D-mpi-v1/src/solver.h b/BasicSolver/2D-mpi-v1/src/solver.h new file mode 100644 index 0000000..00f8fd1 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/solver.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "parameter.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + double dx, dy; + int imax, jmax; + int jmaxLocal; + double xlength, ylength; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop; + /* mpi */ + int rank; + int size; +} Solver; + +void initSolver(Solver*, Parameter*); +void computeRHS(Solver*); +void solve(Solver*); +void normalizePressure(Solver*); +void computeTimestep(Solver*); +void setBoundaryConditions(Solver*); +void setSpecialBoundaryCondition(Solver*); +void computeFG(Solver*); +void adaptUV(Solver*); +void collectResult(Solver*); +void writeResult(Solver*, double*, double*, double*); +#endif diff --git a/BasicSolver/2D-mpi-v1/src/timing.c b/BasicSolver/2D-mpi-v1/src/timing.c new file mode 100644 index 0000000..c4025a4 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/timing.c @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution() +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-mpi-v1/src/timing.h b/BasicSolver/2D-mpi-v1/src/timing.h new file mode 100644 index 0000000..db95329 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/timing.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(); +extern double getTimeResolution(); +extern double getTimeStamp_(); + +#endif // __TIMING_H_ diff --git a/BasicSolver/2D-mpi-v1/src/util.h b/BasicSolver/2D-mpi-v1/src/util.h new file mode 100644 index 0000000..61a1dff --- /dev/null +++ b/BasicSolver/2D-mpi-v1/src/util.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "------------------------------------------------------------------------" \ + "----\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#endif // __UTIL_H_ diff --git a/BasicSolver/2D-mpi-v1/surface.plot b/BasicSolver/2D-mpi-v1/surface.plot new file mode 100644 index 0000000..4f7ccd9 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/surface.plot @@ -0,0 +1,7 @@ +set terminal png size 1024,768 enhanced font ,12 +set output 'p.png' +set datafile separator whitespace + +set grid +set hidden3d +splot 'pressure.dat' using 1:2:3 with lines diff --git a/BasicSolver/2D-mpi-v1/vector.plot b/BasicSolver/2D-mpi-v1/vector.plot new file mode 100644 index 0000000..0934ab2 --- /dev/null +++ b/BasicSolver/2D-mpi-v1/vector.plot @@ -0,0 +1,5 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'velocity.png' +set datafile separator whitespace + +plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/BasicSolver/2D-mpi-v2/Makefile b/BasicSolver/2D-mpi-v2/Makefile new file mode 100644 index 0000000..57f99f4 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/Makefile @@ -0,0 +1,71 @@ +#======================================================================================= +# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(wildcard $(SRC_DIR)/*.c) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm format + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/BasicSolver/2D-mpi-v2/README.md b/BasicSolver/2D-mpi-v2/README.md new file mode 100644 index 0000000..b0a80a6 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/README.md @@ -0,0 +1,48 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution. + + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. diff --git a/BasicSolver/2D-mpi-v2/canal.par b/BasicSolver/2D-mpi-v2/canal.par new file mode 100644 index 0000000..80dfa19 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/canal.par @@ -0,0 +1,46 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcN 1 # flags for boundary conditions +bcE 3 # 1 = no-slip 3 = outflow +bcS 1 # 2 = free-slip 4 = periodic +bcW 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 100.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 50 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 100.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.00001 # stopping tolerance for pressure iteration +omg 1.8 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-mpi-v2/config.mk b/BasicSolver/2D-mpi-v2/config.mk new file mode 100644 index 0000000..e187c7d --- /dev/null +++ b/BasicSolver/2D-mpi-v2/config.mk @@ -0,0 +1,10 @@ +# Supported: GCC, CLANG, ICC +TAG ?= CLANG +ENABLE_OPENMP ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +# OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/2D-mpi-v2/dcavity.par b/BasicSolver/2D-mpi-v2/dcavity.par new file mode 100644 index 0000000..d20d83d --- /dev/null +++ b/BasicSolver/2D-mpi-v2/dcavity.par @@ -0,0 +1,46 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcN 1 # flags for boundary conditions +bcE 1 # 1 = no-slip 3 = outflow +bcS 1 # 2 = free-slip 4 = periodic +bcW 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 1000.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +imax 100 # number of interior cells in x-direction +jmax 100 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 10.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-mpi-v2/include_CLANG.mk b/BasicSolver/2D-mpi-v2/include_CLANG.mk new file mode 100644 index 0000000..1d17c52 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/include_CLANG.mk @@ -0,0 +1,16 @@ +CC = mpicc +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +CFLAGS = -Ofast -std=c99 $(OPENMP) +#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE# -DDEBUG +INCLUDES = -I/usr/local/include diff --git a/BasicSolver/2D-mpi-v2/include_GCC.mk b/BasicSolver/2D-mpi-v2/include_GCC.mk new file mode 100644 index 0000000..427e798 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/include_GCC.mk @@ -0,0 +1,14 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/2D-mpi-v2/include_ICC.mk b/BasicSolver/2D-mpi-v2/include_ICC.mk new file mode 100644 index 0000000..f85d836 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/include_ICC.mk @@ -0,0 +1,14 @@ +CC = mpiicc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/2D-mpi-v2/src/affinity.c b/BasicSolver/2D-mpi-v2/src/affinity.c new file mode 100644 index 0000000..b501665 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/affinity.c @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifdef __linux__ +#ifdef _OPENMP +#include +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_THREADS 128 +#define gettid() syscall(SYS_gettid) + +static int getProcessorID(cpu_set_t* cpu_set) +{ + int processorId; + + for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { + if (CPU_ISSET(processorId, cpu_set)) { + break; + } + } + return processorId; +} + +int affinity_getProcessorId() +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); + + return getProcessorID(&cpu_set); +} + +void affinity_pinThread(int processorId) +{ + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); +} + +void affinity_pinProcess(int processorId) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} +#endif /*_OPENMP*/ +#endif /*__linux__*/ diff --git a/BasicSolver/2D-mpi-v2/src/affinity.h b/BasicSolver/2D-mpi-v2/src/affinity.h new file mode 100644 index 0000000..d844fe5 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/affinity.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef AFFINITY_H +#define AFFINITY_H + +extern int affinity_getProcessorId(); +extern void affinity_pinProcess(int); +extern void affinity_pinThread(int); + +#endif /*AFFINITY_H*/ diff --git a/BasicSolver/2D-mpi-v2/src/allocate.c b/BasicSolver/2D-mpi-v2/src/allocate.c new file mode 100644 index 0000000..81e1e9d --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/allocate.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +void* allocate(int alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/BasicSolver/2D-mpi-v2/src/allocate.h b/BasicSolver/2D-mpi-v2/src/allocate.h new file mode 100644 index 0000000..54cfe06 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(int alignment, size_t bytesize); + +#endif diff --git a/BasicSolver/2D-mpi-v2/src/likwid-marker.h b/BasicSolver/2D-mpi-v2/src/likwid-marker.h new file mode 100644 index 0000000..c3770c0 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/BasicSolver/2D-mpi-v2/src/main.c b/BasicSolver/2D-mpi-v2/src/main.c new file mode 100644 index 0000000..c600033 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/main.c @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include +#include + +#include "parameter.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" + +int main(int argc, char** argv) +{ + int rank; + double S, E; + Parameter params; + Solver solver; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + initParameter(¶ms); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(¶ms, argv[1]); + if (rank == 0) { + printParameter(¶ms); + } + initSolver(&solver, ¶ms); + /* debugExchange(&solver); */ + /* debugBC(&solver); */ + /* exit(EXIT_SUCCESS); */ + initProgress(solver.te); + + double tau = solver.tau; + double te = solver.te; + double t = 0.0; + + S = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) { + computeTimestep(&solver); + } + + setBoundaryConditions(&solver); + setSpecialBoundaryCondition(&solver); + computeFG(&solver); + computeRHS(&solver); + solve(&solver); + adaptUV(&solver); + t += solver.dt; + +#ifdef VERBOSE + if (rank == 0) { + printf("TIME %f , TIMESTEP %f\n", t, solver.dt); + } +#else + printProgress(t); +#endif + } + E = getTimeStamp(); + stopProgress(); + if (rank == 0) { + printf("Solution took %.2fs\n", E - S); + } + collectResult(&solver); + + MPI_Finalize(); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/2D-mpi-v2/src/parameter.c b/BasicSolver/2D-mpi-v2/src/parameter.c new file mode 100644 index 0000000..8f8a7e3 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/parameter.c @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.8; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_STRING(name); + PARSE_INT(bcN); + PARSE_INT(bcS); + PARSE_INT(bcE); + PARSE_INT(bcW); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(p_init); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions N:%d E:%d S:%d W:%d\n", + param->bcN, + param->bcE, + param->bcS, + param->bcW); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength); + printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); +} diff --git a/BasicSolver/2D-mpi-v2/src/parameter.h b/BasicSolver/2D-mpi-v2/src/parameter.h new file mode 100644 index 0000000..a6c6756 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/parameter.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + double xlength, ylength; + int imax, jmax; + int itermax; + double eps, omg; + double re, tau, gamma; + double te, dt; + double gx, gy; + char* name; + int bcN, bcS, bcE, bcW; + double u_init, v_init, p_init; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/BasicSolver/2D-mpi-v2/src/progress.c b/BasicSolver/2D-mpi-v2/src/progress.c new file mode 100644 index 0000000..31a8a90 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/progress.c @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; +static int _rank = -1; + +void initProgress(double end) +{ + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); + _end = end; + _current = 0; + + if (_rank == 0) { + printf("[ ]"); + fflush(stdout); + } +} + +void printProgress(double current) +{ + if (_rank == 0) { + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); + } +} + +void stopProgress() +{ + if (_rank == 0) { + printf("\n"); + fflush(stdout); + } +} diff --git a/BasicSolver/2D-mpi-v2/src/progress.h b/BasicSolver/2D-mpi-v2/src/progress.h new file mode 100644 index 0000000..9ef2d96 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/progress.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ + +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(); + +#endif diff --git a/BasicSolver/2D-mpi-v2/src/solver.c b/BasicSolver/2D-mpi-v2/src/solver.c new file mode 100644 index 0000000..28f266e --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/solver.c @@ -0,0 +1,900 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +#define P(i, j) p[(j) * (imaxLocal + 2) + (i)] +#define F(i, j) f[(j) * (imaxLocal + 2) + (i)] +#define G(i, j) g[(j) * (imaxLocal + 2) + (i)] +#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] +#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)] + +#define NDIMS 2 +#define IDIM 0 +#define JDIM 1 + +static int sizeOfRank(int rank, int size, int N) +{ + return N / size + ((N % size > rank) ? 1 : 0); +} + +void print(Solver* solver, double* grid) +{ + int imaxLocal = solver->imaxLocal; + + for (int i = 0; i < solver->size; i++) { + if (i == solver->rank) { + printf( + "### RANK %d #######################################################\n", + solver->rank); + for (int j = 0; j < solver->jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < solver->imaxLocal + 2; i++) { + printf("%12.8f ", grid[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + MPI_Barrier(MPI_COMM_WORLD); + } +} + +static void exchange(Solver* solver, double* grid) +{ + double* buf[8]; + MPI_Request requests[8]; + for (int i = 0; i < 8; i++) + requests[i] = MPI_REQUEST_NULL; + + buf[0] = grid + 1; // recv bottom + buf[1] = grid + (solver->imaxLocal + 2) + 1; // send bottom + buf[2] = grid + (solver->jmaxLocal + 1) * (solver->imaxLocal + 2) + 1; // recv top + buf[3] = grid + (solver->jmaxLocal) * (solver->imaxLocal + 2) + 1; // send top + buf[4] = grid + (solver->imaxLocal + 2); // recv left + buf[5] = grid + (solver->imaxLocal + 2) + 1; // send left + buf[6] = grid + (solver->imaxLocal + 2) + (solver->imaxLocal + 1); // recv right + buf[7] = grid + (solver->imaxLocal + 2) + (solver->imaxLocal); // send right + + for (int i = 0; i < 2; i++) { + int tag = 0; + if (solver->jNeighbours[i] != MPI_PROC_NULL) { + tag = solver->jNeighbours[i]; + } + /* exchange ghost cells with bottom/top neighbor */ + MPI_Irecv(buf[i * 2], + 1, + solver->jBufferType, + solver->jNeighbours[i], + tag, + solver->comm, + &requests[i * 2]); + MPI_Isend(buf[(i * 2) + 1], + 1, + solver->jBufferType, + solver->jNeighbours[i], + solver->rank, + solver->comm, + &requests[i * 2 + 1]); + + tag = 0; + if (solver->iNeighbours[i] != MPI_PROC_NULL) { + tag = solver->iNeighbours[i]; + } + /* exchange ghost cells with left/right neighbor */ + MPI_Irecv(buf[i * 2 + 4], + 1, + solver->iBufferType, + solver->iNeighbours[i], + tag, + solver->comm, + &requests[i * 2 + 4]); + MPI_Isend(buf[i * 2 + 5], + 1, + solver->iBufferType, + solver->iNeighbours[i], + solver->rank, + solver->comm, + &requests[(i * 2) + 5]); + } + + MPI_Waitall(8, requests, MPI_STATUSES_IGNORE); +} + +static void shift(Solver* solver) +{ + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + double* f = solver->f; + double* g = solver->g; + + /* shift G */ + double* buf = g + 1; + /* receive ghost cells from bottom neighbor */ + MPI_Irecv(buf, + 1, + solver->jBufferType, + solver->jNeighbours[0], + 0, + solver->comm, + &requests[0]); + + buf = g + (solver->jmaxLocal) * (solver->imaxLocal + 2) + 1; + /* send ghost cells to top neighbor */ + MPI_Isend(buf, + 1, + solver->jBufferType, + solver->jNeighbours[1], + 0, + solver->comm, + &requests[1]); + + /* shift F */ + buf = f + (solver->imaxLocal + 2); + /* receive ghost cells from left neighbor */ + MPI_Irecv(buf, + 1, + solver->iBufferType, + solver->iNeighbours[0], + 1, + solver->comm, + &requests[2]); + + buf = f + (solver->imaxLocal + 2) + (solver->imaxLocal); + /* send ghost cells to right neighbor */ + MPI_Isend(buf, + 1, + solver->iBufferType, + solver->iNeighbours[1], + 1, + solver->comm, + &requests[3]); + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +} + +void debugExchange(Solver* solver) +{ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + + for (int j = 0; j < jmaxLocal + 2; j++) { + for (int i = 0; i < solver->imaxLocal + 2; i++) { + solver->p[j * (imaxLocal + 2) + i] = solver->rank + 0.01 * i + 0.0001 * j; + } + } + collectResult(solver); + /* print(solver, solver->p); */ +} + +void debugBC(Solver* solver) +{ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double* v = solver->v; + + // Northern boundary + if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal + 1) = 10.0 + solver->rank; + } + } + + // Eastern boundary + if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + V(imaxLocal + 1, j) = 20.0 + solver->rank; + } + } + + // Southern boundary + if (solver->coords[JDIM] == 0) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 30.0 + solver->rank; + } + } + + // Western boundary + if (solver->coords[IDIM] == 0) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + V(0, j) = 40.0 + solver->rank; + } + } + print(solver, solver->v); +} + +static void assembleResult(Solver* solver, + double* src, + double* dst, + int imaxLocal[], + int jmaxLocal[], + int offset[]) +{ + MPI_Request* requests; + int numRequests = 1; + + if (solver->rank == 0) { + numRequests = solver->size + 1; + } else { + numRequests = 1; + } + + requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); + + /* all ranks send their bulk array */ + MPI_Datatype bulkType; + const int ndims = 2; + int oldSizes[ndims] = { solver->jmaxLocal + 2, solver->imaxLocal + 2 }; + int newSizes[ndims] = { solver->jmaxLocal, solver->imaxLocal }; + int starts[ndims] = { 1, 1 }; + MPI_Type_create_subarray(2, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + + MPI_Isend(src, 1, bulkType, 0, 0, solver->comm, &requests[0]); + + /* rank 0 assembles the subdomains */ + if (solver->rank == 0) { + for (int i = 0; i < solver->size; i++) { + MPI_Datatype domainType; + MPI_Type_vector(jmaxLocal[i], + imaxLocal[i], + solver->imax, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst + offset[i], + 1, + domainType, + i, + 0, + solver->comm, + &requests[i + 1]); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} + +static int sum(int* sizes, int position) +{ + int sum = 0; + + for (int i = 0; i < position; i++) { + sum += sizes[i]; + } + + return sum; +} + +void collectResult(Solver* solver) +{ + double* Pall = NULL; + double* Uall = NULL; + double* Vall = NULL; + int offset[solver->size]; + int imaxLocal[solver->size]; + int jmaxLocal[solver->size]; + + MPI_Gather(&solver->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&solver->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (solver->rank == 0) { + Pall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); + Uall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); + Vall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); + + for (int i = 0; i < solver->size; i++) { + int coords[2]; + MPI_Cart_coords(solver->comm, i, 2, coords); + int ioffset = sum(imaxLocal, coords[0]); + int joffset = sum(jmaxLocal, coords[1]); + offset[i] = (joffset * solver->imax) + ioffset; + printf("Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Offset(i,j): %d %d\n", + i, + coords[0], + coords[1], + imaxLocal[i], + jmaxLocal[i], + ioffset, + joffset); + } + } + + /* collect P */ + assembleResult(solver, solver->p, Pall, imaxLocal, jmaxLocal, offset); + + /* collect U */ + assembleResult(solver, solver->u, Uall, imaxLocal, jmaxLocal, offset); + + /* collect V */ + assembleResult(solver, solver->v, Vall, imaxLocal, jmaxLocal, offset); + + /* write to disk */ + if (solver->rank == 0) writeResult(solver, Pall, Uall, Vall); +} + +static void printConfig(Solver* solver) +{ + if (solver->rank == 0) { + printf("Parameters for #%s#\n", solver->problem); + printf("Boundary conditions N:%d E:%d S:%d W:%d\n", + solver->bcN, + solver->bcE, + solver->bcS, + solver->bcW); + printf("\tReynolds number: %.2f\n", solver->re); + printf("\tGx Gy: %.2f %.2f\n", solver->gx, solver->gy); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", + solver->xlength, + solver->ylength); + printf("\tCells (x, y): %d, %d\n", solver->imax, solver->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", solver->dt, solver->te); + printf("\tdt bound: %.6f\n", solver->dtBound); + printf("\tTau factor: %.2f\n", solver->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", solver->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", solver->eps); + printf("\tgamma factor: %f\n", solver->gamma); + printf("\tomega (SOR relaxation): %f\n", solver->omega); + printf("Communication parameters:\n"); + } + for (int i = 0; i < solver->size; i++) { + if (i == solver->rank) { + printf("\tRank %d of %d\n", solver->rank, solver->size); + printf("\tNeighbours (b, t, l, r): %d, %d, %d, %d\n", + solver->jNeighbours[0], + solver->jNeighbours[1], + solver->iNeighbours[0], + solver->iNeighbours[1]); + printf("\tCoordinates %d,%d\n", solver->coords[0], solver->coords[1]); + printf("\tLocal domain size: %dx%d\n", solver->imaxLocal, solver->jmaxLocal); + fflush(stdout); + } + } +} + +void initSolver(Solver* solver, Parameter* params) +{ + solver->problem = params->name; + solver->bcN = params->bcN; + solver->bcS = params->bcS; + solver->bcW = params->bcW; + solver->bcE = params->bcE; + solver->imax = params->imax; + solver->jmax = params->jmax; + solver->xlength = params->xlength; + solver->ylength = params->ylength; + solver->dx = params->xlength / params->imax; + solver->dy = params->ylength / params->jmax; + solver->eps = params->eps; + solver->omega = params->omg; + solver->itermax = params->itermax; + solver->re = params->re; + solver->gx = params->gx; + solver->gy = params->gy; + solver->dt = params->dt; + solver->te = params->te; + solver->tau = params->tau; + solver->gamma = params->gamma; + + /* setup communication */ + MPI_Comm_rank(MPI_COMM_WORLD, &(solver->rank)); + MPI_Comm_size(MPI_COMM_WORLD, &(solver->size)); + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + MPI_Dims_create(solver->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &solver->comm); + MPI_Cart_shift(solver->comm, + IDIM, + 1, + &solver->iNeighbours[0], + &solver->iNeighbours[1]); + MPI_Cart_shift(solver->comm, + JDIM, + 1, + &solver->jNeighbours[0], + &solver->jNeighbours[1]); + MPI_Cart_get(solver->comm, NDIMS, solver->dims, periods, solver->coords); + + solver->imaxLocal = sizeOfRank(solver->rank, dims[IDIM], solver->imax); + solver->jmaxLocal = sizeOfRank(solver->rank, dims[JDIM], solver->jmax); + + MPI_Type_contiguous(solver->imaxLocal, MPI_DOUBLE, &solver->jBufferType); + MPI_Type_commit(&solver->jBufferType); + + MPI_Type_vector(solver->jmaxLocal, + 1, + solver->imaxLocal + 2, + MPI_DOUBLE, + &solver->iBufferType); + MPI_Type_commit(&solver->iBufferType); + + /* allocate arrays */ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + size_t bytesize = (imaxLocal + 2) * (jmaxLocal + 2) * sizeof(double); + solver->u = allocate(64, bytesize); + solver->v = allocate(64, bytesize); + solver->p = allocate(64, bytesize); + solver->rhs = allocate(64, bytesize); + solver->f = allocate(64, bytesize); + solver->g = allocate(64, bytesize); + + for (int i = 0; i < (imaxLocal + 2) * (jmaxLocal + 2); i++) { + solver->u[i] = params->u_init; + solver->v[i] = params->v_init; + solver->p[i] = params->p_init; + solver->rhs[i] = 0.0; + solver->f[i] = 0.0; + solver->g[i] = 0.0; + } + + double dx = solver->dx; + double dy = solver->dy; + double inv_sqr_sum = 1.0 / (dx * dx) + 1.0 / (dy * dy); + solver->dtBound = 0.5 * solver->re * 1.0 / inv_sqr_sum; +#ifdef VERBOSE + printConfig(solver); +#endif +} + +void computeRHS(Solver* solver) +{ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double idx = 1.0 / solver->dx; + double idy = 1.0 / solver->dy; + double idt = 1.0 / solver->dt; + double* rhs = solver->rhs; + double* f = solver->f; + double* g = solver->g; + + shift(solver); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) * + idt; + } + } +} + +int solve(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double eps = solver->eps; + int itermax = solver->itermax; + double dx2 = solver->dx * solver->dx; + double dy2 = solver->dy * solver->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* p = solver->p; + double* rhs = solver->rhs; + double epssq = eps * eps; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + exchange(solver, p); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + } + + if (solver->coords[JDIM] == 0) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (solver->coords[IDIM] == 0) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } + + MPI_Allreduce(MPI_IN_PLACE, &res, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (solver->rank == 0) { + printf("%d Residuum: %e\n", it, res); + } +#endif + it++; + } + +#ifdef VERBOSE + if (solver->rank == 0) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + if (res < eps) { + return 0; + } else { + return 1; + } +} + +static double maxElement(Solver* solver, double* m) +{ + int size = (solver->imaxLocal + 2) * (solver->jmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + MPI_Allreduce(MPI_IN_PLACE, &maxval, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + return maxval; +} + +void computeTimestep(Solver* solver) +{ + double dt = solver->dtBound; + double dx = solver->dx; + double dy = solver->dy; + double umax = maxElement(solver, solver->u); + double vmax = maxElement(solver, solver->v); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + + solver->dt = dt * solver->tau; +} + +void setBoundaryConditions(Solver* solver) +{ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double* u = solver->u; + double* v = solver->v; + + // Northern boundary + if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc + switch (solver->bcN) { + case NOSLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = -U(i, jmaxLocal); + } + break; + case SLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + } + break; + case OUTFLOW: + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + V(i, jmaxLocal) = V(i, jmaxLocal - 1); + } + break; + case PERIODIC: + break; + } + } + + // Southern boundary + if (solver->coords[JDIM] == 0) { // set bottom bc + switch (solver->bcS) { + case NOSLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = -U(i, 1); + } + break; + case SLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = U(i, 1); + } + break; + case OUTFLOW: + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0) = U(i, 1); + V(i, 0) = V(i, 1); + } + break; + case PERIODIC: + break; + } + } + + // Eastern boundary + if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc + switch (solver->bcE) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = 0.0; + V(imaxLocal + 1, j) = -V(imaxLocal, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = 0.0; + V(imaxLocal + 1, j) = V(imaxLocal, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = U(imaxLocal - 1, j); + V(imaxLocal + 1, j) = V(imaxLocal, j); + } + break; + case PERIODIC: + break; + } + } + + // Western boundary + if (solver->coords[IDIM] == 0) { // set left bc + switch (solver->bcW) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = -V(1, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = V(1, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = U(1, j); + V(0, j) = V(1, j); + } + break; + case PERIODIC: + break; + } + } +} + +void setSpecialBoundaryCondition(Solver* solver) +{ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double* u = solver->u; + + if (strcmp(solver->problem, "dcavity") == 0) { + if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal); + } + } + } else if (strcmp(solver->problem, "canal") == 0) { + if (solver->coords[IDIM] == 0) { // set left bc + double ylength = solver->ylength; + double dy = solver->dy; + int rest = solver->jmax % solver->size; + int yc = solver->rank * (solver->jmax / solver->size) + + MIN(rest, solver->rank); + double ys = dy * (yc + 0.5); + double y; + + /* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */ + + for (int j = 1; j < jmaxLocal + 1; j++) { + y = ys + dy * (j - 0.5); + U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } + } + /* print(solver, solver->u); */ +} + +void computeFG(Solver* solver) +{ + double* u = solver->u; + double* v = solver->v; + double* f = solver->f; + double* g = solver->g; + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double gx = solver->gx; + double gy = solver->gy; + double gamma = solver->gamma; + double dt = solver->dt; + double inverseRe = 1.0 / solver->re; + double inverseDx = 1.0 / solver->dx; + double inverseDy = 1.0 / solver->dy; + double du2dx, dv2dy, duvdx, duvdy; + double du2dx2, du2dy2, dv2dx2, dv2dy2; + + exchange(solver, u); + exchange(solver, v); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + du2dx = inverseDx * 0.25 * + ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - + (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + + fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); + + duvdy = inverseDy * 0.25 * + ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - + (V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + + fabs(V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) - U(i, j - 1))); + + du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); + du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); + F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - + (U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + + fabs(U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) - V(i - 1, j))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - + (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + + fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); + + dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); + dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); + G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); + } + } + + /* ----------------------------- boundary of F --------------------------- */ + if (solver->coords[IDIM] == 0) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + F(0, j) = U(0, j); + } + } + + if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + F(imaxLocal, j) = U(imaxLocal, j); + } + } + + /* ----------------------------- boundary of G --------------------------- */ + if (solver->coords[JDIM] == 0) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, 0) = V(i, 0); + } + } + + if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, jmaxLocal) = V(i, jmaxLocal); + } + } +} + +void adaptUV(Solver* solver) +{ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double* p = solver->p; + double* u = solver->u; + double* v = solver->v; + double* f = solver->f; + double* g = solver->g; + double factorX = solver->dt / solver->dx; + double factorY = solver->dt / solver->dy; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; + V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; + } + } +} + +void writeResult(Solver* solver, double* p, double* u, double* v) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double dx = solver->dx; + double dy = solver->dy; + double x = 0.0, y = 0.0; + + FILE* fp; + fp = fopen("pressure.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax; j++) { + y = (double)(j - 0.5) * dy; + for (int i = 1; i < imax; i++) { + x = (double)(i - 0.5) * dx; + fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]); + } + fprintf(fp, "\n"); + } + + fclose(fp); + + fp = fopen("velocity.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax; j++) { + y = dy * (j - 0.5); + for (int i = 1; i < imax; i++) { + x = dx * (i - 0.5); + double vel_u = (u[j * (imax) + i] + u[j * (imax) + (i - 1)]) / 2.0; + double vel_v = (v[j * (imax) + i] + v[(j - 1) * (imax) + i]) / 2.0; + double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); + } + } + + fclose(fp); +} diff --git a/BasicSolver/2D-mpi-v2/src/solver.h b/BasicSolver/2D-mpi-v2/src/solver.h new file mode 100644 index 0000000..1d3f7c9 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/solver.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "parameter.h" +#include + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + double dx, dy; + int imax, jmax; + double xlength, ylength; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcN, bcS, bcW, bcE; + /* mpi */ + int rank; + int size; + MPI_Comm comm; + MPI_Datatype iBufferType, jBufferType; + int iNeighbours[2], jNeighbours[2]; + int coords[2], dims[2]; + int imaxLocal, jmaxLocal; +} Solver; + +void initSolver(Solver*, Parameter*); +void computeRHS(Solver*); +int solve(Solver*); +void computeTimestep(Solver*); +void setBoundaryConditions(Solver*); +void setSpecialBoundaryCondition(Solver*); +void computeFG(Solver*); +void adaptUV(Solver*); +void collectResult(Solver*); +void writeResult(Solver*, double*, double*, double*); +void debugExchange(Solver*); +void debugBC(Solver*); +void print(Solver*, double*); +#endif diff --git a/BasicSolver/2D-mpi-v2/src/timing.c b/BasicSolver/2D-mpi-v2/src/timing.c new file mode 100644 index 0000000..c4025a4 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/timing.c @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution() +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-mpi-v2/src/timing.h b/BasicSolver/2D-mpi-v2/src/timing.h new file mode 100644 index 0000000..db95329 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/timing.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(); +extern double getTimeResolution(); +extern double getTimeStamp_(); + +#endif // __TIMING_H_ diff --git a/BasicSolver/2D-mpi-v2/src/util.h b/BasicSolver/2D-mpi-v2/src/util.h new file mode 100644 index 0000000..657b009 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/src/util.h @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#endif // __UTIL_H_ diff --git a/BasicSolver/2D-mpi-v2/surface.plot b/BasicSolver/2D-mpi-v2/surface.plot new file mode 100644 index 0000000..4f7ccd9 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/surface.plot @@ -0,0 +1,7 @@ +set terminal png size 1024,768 enhanced font ,12 +set output 'p.png' +set datafile separator whitespace + +set grid +set hidden3d +splot 'pressure.dat' using 1:2:3 with lines diff --git a/BasicSolver/2D-mpi-v2/vector.plot b/BasicSolver/2D-mpi-v2/vector.plot new file mode 100644 index 0000000..0934ab2 --- /dev/null +++ b/BasicSolver/2D-mpi-v2/vector.plot @@ -0,0 +1,5 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'velocity.png' +set datafile separator whitespace + +plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/BasicSolver/2D-mpi-v3/Makefile b/BasicSolver/2D-mpi-v3/Makefile new file mode 100644 index 0000000..57f99f4 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/Makefile @@ -0,0 +1,71 @@ +#======================================================================================= +# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(wildcard $(SRC_DIR)/*.c) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm format + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/BasicSolver/2D-mpi-v3/README.md b/BasicSolver/2D-mpi-v3/README.md new file mode 100644 index 0000000..b0a80a6 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/README.md @@ -0,0 +1,48 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution. + + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. diff --git a/BasicSolver/2D-mpi-v3/canal.par b/BasicSolver/2D-mpi-v3/canal.par new file mode 100644 index 0000000..80dfa19 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/canal.par @@ -0,0 +1,46 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcN 1 # flags for boundary conditions +bcE 3 # 1 = no-slip 3 = outflow +bcS 1 # 2 = free-slip 4 = periodic +bcW 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 100.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 50 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 100.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.00001 # stopping tolerance for pressure iteration +omg 1.8 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-mpi-v3/config.mk b/BasicSolver/2D-mpi-v3/config.mk new file mode 100644 index 0000000..496668e --- /dev/null +++ b/BasicSolver/2D-mpi-v3/config.mk @@ -0,0 +1,10 @@ +# Supported: GCC, CLANG, ICC +TAG ?= CLANG +ENABLE_OPENMP ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/2D-mpi-v3/dcavity.par b/BasicSolver/2D-mpi-v3/dcavity.par new file mode 100644 index 0000000..b4013d6 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/dcavity.par @@ -0,0 +1,46 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 1 # 2 = free-slip 4 = periodic +bcRight 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 10.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +imax 100 # number of interior cells in x-direction +jmax 100 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 5.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-mpi-v3/include_CLANG.mk b/BasicSolver/2D-mpi-v3/include_CLANG.mk new file mode 100644 index 0000000..1d17c52 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/include_CLANG.mk @@ -0,0 +1,16 @@ +CC = mpicc +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +CFLAGS = -Ofast -std=c99 $(OPENMP) +#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE# -DDEBUG +INCLUDES = -I/usr/local/include diff --git a/BasicSolver/2D-mpi-v3/include_GCC.mk b/BasicSolver/2D-mpi-v3/include_GCC.mk new file mode 100644 index 0000000..427e798 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/include_GCC.mk @@ -0,0 +1,14 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/2D-mpi-v3/include_ICC.mk b/BasicSolver/2D-mpi-v3/include_ICC.mk new file mode 100644 index 0000000..f85d836 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/include_ICC.mk @@ -0,0 +1,14 @@ +CC = mpiicc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/2D-mpi-v3/src/affinity.c b/BasicSolver/2D-mpi-v3/src/affinity.c new file mode 100644 index 0000000..b501665 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/affinity.c @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifdef __linux__ +#ifdef _OPENMP +#include +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_THREADS 128 +#define gettid() syscall(SYS_gettid) + +static int getProcessorID(cpu_set_t* cpu_set) +{ + int processorId; + + for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { + if (CPU_ISSET(processorId, cpu_set)) { + break; + } + } + return processorId; +} + +int affinity_getProcessorId() +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); + + return getProcessorID(&cpu_set); +} + +void affinity_pinThread(int processorId) +{ + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); +} + +void affinity_pinProcess(int processorId) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} +#endif /*_OPENMP*/ +#endif /*__linux__*/ diff --git a/BasicSolver/2D-mpi-v3/src/affinity.h b/BasicSolver/2D-mpi-v3/src/affinity.h new file mode 100644 index 0000000..d844fe5 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/affinity.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef AFFINITY_H +#define AFFINITY_H + +extern int affinity_getProcessorId(); +extern void affinity_pinProcess(int); +extern void affinity_pinThread(int); + +#endif /*AFFINITY_H*/ diff --git a/BasicSolver/2D-mpi-v3/src/allocate.c b/BasicSolver/2D-mpi-v3/src/allocate.c new file mode 100644 index 0000000..81e1e9d --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/allocate.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +void* allocate(int alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/BasicSolver/2D-mpi-v3/src/allocate.h b/BasicSolver/2D-mpi-v3/src/allocate.h new file mode 100644 index 0000000..54cfe06 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(int alignment, size_t bytesize); + +#endif diff --git a/BasicSolver/2D-mpi-v3/src/likwid-marker.h b/BasicSolver/2D-mpi-v3/src/likwid-marker.h new file mode 100644 index 0000000..c3770c0 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/BasicSolver/2D-mpi-v3/src/main.c b/BasicSolver/2D-mpi-v3/src/main.c new file mode 100644 index 0000000..34de302 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/main.c @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "parameter.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" +#include + +int main(int argc, char** argv) +{ + int rank; + double S, E; + Parameter params; + Solver solver; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + initParameter(¶ms); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(¶ms, argv[1]); + if (rank == 0) { + printParameter(¶ms); + } + initSolver(&solver, ¶ms); + initProgress(solver.te); + + double tau = solver.tau; + double te = solver.te; + double t = 0.0; + + S = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) { + computeTimestep(&solver); + } + + setBoundaryConditions(&solver); + setSpecialBoundaryCondition(&solver); + computeFG(&solver); + computeRHS(&solver); + solve(&solver); + adaptUV(&solver); + t += solver.dt; + +#ifdef VERBOSE + if (rank == 0) { + printf("TIME %f , TIMESTEP %f\n", t, solver.dt); + } +#else + printProgress(t); +#endif + } + E = getTimeStamp(); + stopProgress(); + if (rank == 0) { + printf("Solution took %.2fs\n", E - S); + } + collectResult(&solver); + + MPI_Finalize(); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/2D-mpi-v3/src/parameter.c b/BasicSolver/2D-mpi-v3/src/parameter.c new file mode 100644 index 0000000..d691627 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/parameter.c @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_STRING(name); + PARSE_INT(bcLeft); + PARSE_INT(bcRight); + PARSE_INT(bcBottom); + PARSE_INT(bcTop); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(p_init); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + param->bcLeft, + param->bcRight, + param->bcBottom, + param->bcTop); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength); + printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); +} diff --git a/BasicSolver/2D-mpi-v3/src/parameter.h b/BasicSolver/2D-mpi-v3/src/parameter.h new file mode 100644 index 0000000..f4c331a --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/parameter.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + double xlength, ylength; + int imax, jmax; + int itermax; + double eps, omg; + double re, tau, gamma; + double te, dt; + double gx, gy; + char* name; + int bcLeft, bcRight, bcBottom, bcTop; + double u_init, v_init, p_init; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/BasicSolver/2D-mpi-v3/src/progress.c b/BasicSolver/2D-mpi-v3/src/progress.c new file mode 100644 index 0000000..31a8a90 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/progress.c @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; +static int _rank = -1; + +void initProgress(double end) +{ + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); + _end = end; + _current = 0; + + if (_rank == 0) { + printf("[ ]"); + fflush(stdout); + } +} + +void printProgress(double current) +{ + if (_rank == 0) { + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); + } +} + +void stopProgress() +{ + if (_rank == 0) { + printf("\n"); + fflush(stdout); + } +} diff --git a/BasicSolver/2D-mpi-v3/src/progress.h b/BasicSolver/2D-mpi-v3/src/progress.h new file mode 100644 index 0000000..9ef2d96 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/progress.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ + +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(); + +#endif diff --git a/BasicSolver/2D-mpi-v3/src/solver.c b/BasicSolver/2D-mpi-v3/src/solver.c new file mode 100644 index 0000000..3378e94 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/solver.c @@ -0,0 +1,832 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +#define P(i, j) p[(j) * (imaxLocal + 2) + (i)] +#define F(i, j) f[(j) * (imaxLocal + 2) + (i)] +#define G(i, j) g[(j) * (imaxLocal + 2) + (i)] +#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] +#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)] + +#define IDIM 0 +#define JDIM 1 + +static int sizeOfRank(int rank, int size, int N) +{ + return N / size + ((N % size > rank) ? 1 : 0); +} + +void print(Solver* solver, double* grid) +{ + int imaxLocal = solver->imaxLocal; + + for (int i = 0; i < solver->size; i++) { + if (i == solver->rank) { + printf( + "### RANK %d #######################################################\n", + solver->rank); + for (int j = 0; j < solver->jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < solver->imaxLocal + 2; i++) { + printf("%12.8f ", grid[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + MPI_Barrier(MPI_COMM_WORLD); + } +} + +static void exchange(Solver* solver, double* grid) +{ + int counts[4] = { 1, 1, 1, 1 }; + + MPI_Neighbor_alltoallw(grid, + counts, + solver->sdispls, + solver->bufferTypes, + grid, + counts, + solver->rdispls, + solver->bufferTypes, + solver->comm); +} + +static void shift(Solver* solver) +{ + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + double* f = solver->f; + double* g = solver->g; + + /* shift G */ + double* buf = g + 1; + /* receive ghost cells from bottom neighbor */ + MPI_Irecv(buf, + 1, + solver->bufferTypes[2], + solver->jNeighbours[0], + 0, + solver->comm, + &requests[0]); + + buf = g + (solver->jmaxLocal) * (solver->imaxLocal + 2) + 1; + /* send ghost cells to top neighbor */ + MPI_Isend(buf, + 1, + solver->bufferTypes[2], + solver->jNeighbours[1], + 0, + solver->comm, + &requests[1]); + + /* shift F */ + buf = f + (solver->imaxLocal + 2); + /* receive ghost cells from left neighbor */ + MPI_Irecv(buf, + 1, + solver->bufferTypes[0], + solver->iNeighbours[0], + 1, + solver->comm, + &requests[2]); + + buf = f + (solver->imaxLocal + 2) + (solver->imaxLocal); + /* send ghost cells to right neighbor */ + MPI_Isend(buf, + 1, + solver->bufferTypes[0], + solver->iNeighbours[1], + 1, + solver->comm, + &requests[3]); + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +} + +void debugExchange(Solver* solver) +{ + for (int i = 0; i < (solver->imaxLocal + 2) * (solver->jmaxLocal + 2); i++) { + solver->p[i] = solver->rank; + } + exchange(solver, solver->p); + print(solver, solver->p); +} + +static void assembleResult(Solver* solver, + double* src, + double* dst, + int imaxLocal[], + int jmaxLocal[], + int offset[]) +{ + MPI_Request* requests; + int numRequests = 1; + + if (solver->rank == 0) { + numRequests = solver->size + 1; + } else { + numRequests = 1; + } + + requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); + + /* all ranks send their bulk array */ + MPI_Datatype bulkType; + const int ndims = 2; + int oldSizes[ndims] = { solver->jmaxLocal + 2, solver->imaxLocal + 2 }; + int newSizes[ndims] = { solver->jmaxLocal, solver->imaxLocal }; + int starts[ndims] = { 1, 1 }; + MPI_Type_create_subarray(2, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + + MPI_Isend(src, 1, bulkType, 0, 0, solver->comm, &requests[0]); + + /* rank 0 assembles the subdomains */ + if (solver->rank == 0) { + for (int i = 0; i < solver->size; i++) { + MPI_Datatype domainType; + MPI_Type_vector(jmaxLocal[i], + imaxLocal[i], + solver->imax, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst + offset[i], + 1, + domainType, + i, + 0, + solver->comm, + &requests[i + 1]); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} + +static int sum(int* sizes, int position) +{ + int sum = 0; + + for (int i = 0; i < position; i++) { + sum += sizes[i]; + } + + return sum; +} + +void collectResult(Solver* solver) +{ + double* Pall = NULL; + double* Uall = NULL; + double* Vall = NULL; + int offset[solver->size]; + int imaxLocal[solver->size]; + int jmaxLocal[solver->size]; + + MPI_Gather(&solver->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&solver->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (solver->rank == 0) { + Pall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); + Uall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); + Vall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); + + for (int i = 0; i < solver->size; i++) { + int coords[2]; + MPI_Cart_coords(solver->comm, i, 2, coords); + int ioffset = sum(imaxLocal, coords[0]); + int joffset = sum(jmaxLocal, coords[1]); + offset[i] = (joffset * solver->imax) + ioffset; + printf("Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Offset(i,j): %d %d\n", + i, + coords[0], + coords[1], + imaxLocal[i], + jmaxLocal[i], + ioffset, + joffset); + } + } + + /* collect P */ + assembleResult(solver, solver->p, Pall, imaxLocal, jmaxLocal, offset); + + /* collect U */ + assembleResult(solver, solver->u, Uall, imaxLocal, jmaxLocal, offset); + + /* collect V */ + assembleResult(solver, solver->v, Vall, imaxLocal, jmaxLocal, offset); + + /* write to disk */ + if (solver->rank == 0) writeResult(solver, Pall, Uall, Vall); +} + +static void printConfig(Solver* solver) +{ + if (solver->rank == 0) { + printf("Parameters for #%s#\n", solver->problem); + printf("Boundary conditions Top:%d Bottom:%d Left:%d Right:%d\n", + solver->bcTop, + solver->bcBottom, + solver->bcLeft, + solver->bcRight); + printf("\tReynolds number: %.2f\n", solver->re); + printf("\tGx Gy: %.2f %.2f\n", solver->gx, solver->gy); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", + solver->xlength, + solver->ylength); + printf("\tCells (x, y): %d, %d\n", solver->imax, solver->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", solver->dt, solver->te); + printf("\tdt bound: %.6f\n", solver->dtBound); + printf("\tTau factor: %.2f\n", solver->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", solver->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", solver->eps); + printf("\tgamma factor: %f\n", solver->gamma); + printf("\tomega (SOR relaxation): %f\n", solver->omega); + printf("Communication parameters:\n"); + } + for (int i = 0; i < solver->size; i++) { + if (i == solver->rank) { + printf("\tRank %d of %d\n", solver->rank, solver->size); + printf("\tNeighbours (b, t, l, r): %d, %d, %d, %d\n", + solver->jNeighbours[0], + solver->jNeighbours[1], + solver->iNeighbours[0], + solver->iNeighbours[1]); + printf("\tCoordinates %d,%d\n", solver->coords[0], solver->coords[1]); + printf("\tLocal domain size: %dx%d\n", solver->imaxLocal, solver->jmaxLocal); + fflush(stdout); + } + } +} + +void initSolver(Solver* solver, Parameter* params) +{ + solver->problem = params->name; + solver->bcTop = params->bcTop; + solver->bcBottom = params->bcBottom; + solver->bcLeft = params->bcLeft; + solver->bcRight = params->bcRight; + solver->imax = params->imax; + solver->jmax = params->jmax; + solver->xlength = params->xlength; + solver->ylength = params->ylength; + solver->dx = params->xlength / params->imax; + solver->dy = params->ylength / params->jmax; + solver->eps = params->eps; + solver->omega = params->omg; + solver->itermax = params->itermax; + solver->re = params->re; + solver->gx = params->gx; + solver->gy = params->gy; + solver->dt = params->dt; + solver->te = params->te; + solver->tau = params->tau; + solver->gamma = params->gamma; + + /* setup communication */ + MPI_Comm_rank(MPI_COMM_WORLD, &(solver->rank)); + MPI_Comm_size(MPI_COMM_WORLD, &(solver->size)); + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + MPI_Dims_create(solver->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &solver->comm); + MPI_Cart_shift(solver->comm, + IDIM, + 1, + &solver->iNeighbours[0], + &solver->iNeighbours[1]); + MPI_Cart_shift(solver->comm, + JDIM, + 1, + &solver->jNeighbours[0], + &solver->jNeighbours[1]); + MPI_Cart_get(solver->comm, NDIMS, solver->dims, periods, solver->coords); + + solver->imaxLocal = sizeOfRank(solver->rank, dims[IDIM], solver->imax); + solver->jmaxLocal = sizeOfRank(solver->rank, dims[JDIM], solver->jmax); + + MPI_Datatype jBufferType; + MPI_Type_contiguous(solver->imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(solver->jmaxLocal, + 1, + solver->imaxLocal + 2, + MPI_DOUBLE, + &iBufferType); + MPI_Type_commit(&iBufferType); + + // in the order of the dimensions i->0, j->1 + // first negative direction, then positive direction + size_t dblsize = sizeof(double); + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + solver->bufferTypes[0] = iBufferType; // left + solver->bufferTypes[1] = iBufferType; // right + solver->bufferTypes[2] = jBufferType; // bottom + solver->bufferTypes[3] = jBufferType; // top + + solver->sdispls[0] = ((imaxLocal + 2) + 1) * dblsize; // send left + solver->sdispls[1] = ((imaxLocal + 2) + imaxLocal) * dblsize; // send right + solver->sdispls[2] = ((imaxLocal + 2) + 1) * dblsize; // send bottom + solver->sdispls[3] = ((jmaxLocal) * (imaxLocal + 2) + 1) * dblsize; // send top + + solver->rdispls[0] = (imaxLocal + 2) * dblsize; // recv left + solver->rdispls[1] = ((imaxLocal + 2) + (imaxLocal + 1)) * dblsize; // recv right + solver->rdispls[2] = 1 * dblsize; // recv bottom + solver->rdispls[3] = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * dblsize; // recv top + + /* allocate arrays */ + size_t bytesize = (imaxLocal + 2) * (jmaxLocal + 2) * sizeof(double); + solver->u = allocate(64, bytesize); + solver->v = allocate(64, bytesize); + solver->p = allocate(64, bytesize); + solver->rhs = allocate(64, bytesize); + solver->f = allocate(64, bytesize); + solver->g = allocate(64, bytesize); + + for (int i = 0; i < (imaxLocal + 2) * (jmaxLocal + 2); i++) { + solver->u[i] = params->u_init; + solver->v[i] = params->v_init; + solver->p[i] = params->p_init; + solver->rhs[i] = 0.0; + solver->f[i] = 0.0; + solver->g[i] = 0.0; + } + + double dx = solver->dx; + double dy = solver->dy; + double inv_sqr_sum = 1.0 / (dx * dx) + 1.0 / (dy * dy); + solver->dtBound = 0.5 * solver->re * 1.0 / inv_sqr_sum; +#ifdef VERBOSE + printConfig(solver); +#endif +} + +void computeRHS(Solver* solver) +{ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double idx = 1.0 / solver->dx; + double idy = 1.0 / solver->dy; + double idt = 1.0 / solver->dt; + double* rhs = solver->rhs; + double* f = solver->f; + double* g = solver->g; + + shift(solver); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) * + idt; + } + } +} + +int solve(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double eps = solver->eps; + int itermax = solver->itermax; + double dx2 = solver->dx * solver->dx; + double dy2 = solver->dy * solver->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* p = solver->p; + double* rhs = solver->rhs; + double epssq = eps * eps; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + exchange(solver, p); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + } + + if (solver->coords[JDIM] == 0) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (solver->coords[IDIM] == 0) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } + + MPI_Allreduce(MPI_IN_PLACE, &res, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (solver->rank == 0) { + printf("%d Residuum: %e\n", it, res); + } +#endif + it++; + } + +#ifdef VERBOSE + if (solver->rank == 0) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + if (res < eps) { + return 0; + } else { + return 1; + } +} + +static double maxElement(Solver* solver, double* m) +{ + int size = (solver->imaxLocal + 2) * (solver->jmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + MPI_Allreduce(MPI_IN_PLACE, &maxval, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + return maxval; +} + +void computeTimestep(Solver* solver) +{ + double dt = solver->dtBound; + double dx = solver->dx; + double dy = solver->dy; + double umax = maxElement(solver, solver->u); + double vmax = maxElement(solver, solver->v); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + + solver->dt = dt * solver->tau; +} + +void setBoundaryConditions(Solver* solver) +{ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double* u = solver->u; + double* v = solver->v; + + // Northern boundary + if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc + switch (solver->bcTop) { + case NOSLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = -U(i, jmaxLocal); + } + break; + case SLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + } + break; + case OUTFLOW: + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + V(i, jmaxLocal) = V(i, jmaxLocal - 1); + } + break; + case PERIODIC: + break; + } + } + + // Southern boundary + if (solver->coords[JDIM] == 0) { // set bottom bc + switch (solver->bcBottom) { + case NOSLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = -U(i, 1); + } + break; + case SLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = U(i, 1); + } + break; + case OUTFLOW: + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0) = U(i, 1); + V(i, 0) = V(i, 1); + } + break; + case PERIODIC: + break; + } + } + + // Eastern boundary + if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc + switch (solver->bcRight) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = 0.0; + V(imaxLocal + 1, j) = -V(imaxLocal, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = 0.0; + V(imaxLocal + 1, j) = V(imaxLocal, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = U(imaxLocal - 1, j); + V(imaxLocal + 1, j) = V(imaxLocal, j); + } + break; + case PERIODIC: + break; + } + } + + // Western boundary + if (solver->coords[IDIM] == 0) { // set left bc + switch (solver->bcLeft) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = -V(1, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = V(1, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = U(1, j); + V(0, j) = V(1, j); + } + break; + case PERIODIC: + break; + } + } +} + +void setSpecialBoundaryCondition(Solver* solver) +{ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double* u = solver->u; + + if (strcmp(solver->problem, "dcavity") == 0) { + if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal); + } + } + } else if (strcmp(solver->problem, "canal") == 0) { + if (solver->coords[IDIM] == 0) { // set left bc + double ylength = solver->ylength; + double dy = solver->dy; + int rest = solver->jmax % solver->size; + int yc = solver->rank * (solver->jmax / solver->size) + + MIN(rest, solver->rank); + double ys = dy * (yc + 0.5); + double y; + + /* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */ + + for (int j = 1; j < jmaxLocal + 1; j++) { + y = ys + dy * (j - 0.5); + U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } + } + /* print(solver, solver->u); */ +} + +void computeFG(Solver* solver) +{ + double* u = solver->u; + double* v = solver->v; + double* f = solver->f; + double* g = solver->g; + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double gx = solver->gx; + double gy = solver->gy; + double gamma = solver->gamma; + double dt = solver->dt; + double inverseRe = 1.0 / solver->re; + double inverseDx = 1.0 / solver->dx; + double inverseDy = 1.0 / solver->dy; + double du2dx, dv2dy, duvdx, duvdy; + double du2dx2, du2dy2, dv2dx2, dv2dy2; + + exchange(solver, u); + exchange(solver, v); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + du2dx = inverseDx * 0.25 * + ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - + (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + + fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); + + duvdy = inverseDy * 0.25 * + ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - + (V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + + fabs(V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) - U(i, j - 1))); + + du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); + du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); + F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - + (U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + + fabs(U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) - V(i - 1, j))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - + (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + + fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); + + dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); + dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); + G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); + } + } + + /* ----------------------------- boundary of F --------------------------- */ + if (solver->coords[IDIM] == 0) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + F(0, j) = U(0, j); + } + } + + if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + F(imaxLocal, j) = U(imaxLocal, j); + } + } + + /* ----------------------------- boundary of G --------------------------- */ + if (solver->coords[JDIM] == 0) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, 0) = V(i, 0); + } + } + + if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, jmaxLocal) = V(i, jmaxLocal); + } + } +} + +void adaptUV(Solver* solver) +{ + int imaxLocal = solver->imaxLocal; + int jmaxLocal = solver->jmaxLocal; + double* p = solver->p; + double* u = solver->u; + double* v = solver->v; + double* f = solver->f; + double* g = solver->g; + double factorX = solver->dt / solver->dx; + double factorY = solver->dt / solver->dy; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; + V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; + } + } +} + +void writeResult(Solver* solver, double* p, double* u, double* v) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double dx = solver->dx; + double dy = solver->dy; + double x = 0.0, y = 0.0; + + FILE* fp; + fp = fopen("pressure.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax; j++) { + y = (double)(j - 0.5) * dy; + for (int i = 1; i < imax; i++) { + x = (double)(i - 0.5) * dx; + fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]); + } + fprintf(fp, "\n"); + } + + fclose(fp); + + fp = fopen("velocity.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax; j++) { + y = dy * (j - 0.5); + for (int i = 1; i < imax; i++) { + x = dx * (i - 0.5); + double vel_u = (u[j * (imax) + i] + u[j * (imax) + (i - 1)]) / 2.0; + double vel_v = (v[j * (imax) + i] + v[(j - 1) * (imax) + i]) / 2.0; + double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); + } + } + + fclose(fp); +} diff --git a/BasicSolver/2D-mpi-v3/src/solver.h b/BasicSolver/2D-mpi-v3/src/solver.h new file mode 100644 index 0000000..76ccce0 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/solver.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "parameter.h" +#include + +#define NDIMS 2 + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + double dx, dy; + int imax, jmax; + double xlength, ylength; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop; + /* mpi */ + int rank; + int size; + MPI_Comm comm; + MPI_Datatype bufferTypes[NDIMS * 2]; + MPI_Aint sdispls[NDIMS * 2], rdispls[NDIMS * 2]; + int iNeighbours[NDIMS], jNeighbours[NDIMS]; + int coords[NDIMS], dims[NDIMS]; + int imaxLocal, jmaxLocal; +} Solver; + +void initSolver(Solver*, Parameter*); +void computeRHS(Solver*); +int solve(Solver*); +void computeTimestep(Solver*); +void setBoundaryConditions(Solver*); +void setSpecialBoundaryCondition(Solver*); +void computeFG(Solver*); +void adaptUV(Solver*); +void collectResult(Solver*); +void writeResult(Solver*, double*, double*, double*); +void debugExchange(Solver*); +void print(Solver*, double*); +#endif diff --git a/BasicSolver/2D-mpi-v3/src/timing.c b/BasicSolver/2D-mpi-v3/src/timing.c new file mode 100644 index 0000000..c4025a4 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/timing.c @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution() +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-mpi-v3/src/timing.h b/BasicSolver/2D-mpi-v3/src/timing.h new file mode 100644 index 0000000..db95329 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/timing.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(); +extern double getTimeResolution(); +extern double getTimeStamp_(); + +#endif // __TIMING_H_ diff --git a/BasicSolver/2D-mpi-v3/src/util.h b/BasicSolver/2D-mpi-v3/src/util.h new file mode 100644 index 0000000..657b009 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/src/util.h @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#endif // __UTIL_H_ diff --git a/BasicSolver/2D-mpi-v3/surface.plot b/BasicSolver/2D-mpi-v3/surface.plot new file mode 100644 index 0000000..4f7ccd9 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/surface.plot @@ -0,0 +1,7 @@ +set terminal png size 1024,768 enhanced font ,12 +set output 'p.png' +set datafile separator whitespace + +set grid +set hidden3d +splot 'pressure.dat' using 1:2:3 with lines diff --git a/BasicSolver/2D-mpi-v3/vector.plot b/BasicSolver/2D-mpi-v3/vector.plot new file mode 100644 index 0000000..0934ab2 --- /dev/null +++ b/BasicSolver/2D-mpi-v3/vector.plot @@ -0,0 +1,5 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'velocity.png' +set datafile separator whitespace + +plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/BasicSolver/2D-mpi/Makefile b/BasicSolver/2D-mpi/Makefile new file mode 100644 index 0000000..57f99f4 --- /dev/null +++ b/BasicSolver/2D-mpi/Makefile @@ -0,0 +1,71 @@ +#======================================================================================= +# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(wildcard $(SRC_DIR)/*.c) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm format + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/BasicSolver/2D-mpi/README.md b/BasicSolver/2D-mpi/README.md new file mode 100644 index 0000000..b0a80a6 --- /dev/null +++ b/BasicSolver/2D-mpi/README.md @@ -0,0 +1,48 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution. + + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. diff --git a/BasicSolver/2D-mpi/canal.par b/BasicSolver/2D-mpi/canal.par new file mode 100644 index 0000000..80dfa19 --- /dev/null +++ b/BasicSolver/2D-mpi/canal.par @@ -0,0 +1,46 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcN 1 # flags for boundary conditions +bcE 3 # 1 = no-slip 3 = outflow +bcS 1 # 2 = free-slip 4 = periodic +bcW 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 100.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 50 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 100.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.00001 # stopping tolerance for pressure iteration +omg 1.8 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-mpi/config.mk b/BasicSolver/2D-mpi/config.mk new file mode 100644 index 0000000..46cef95 --- /dev/null +++ b/BasicSolver/2D-mpi/config.mk @@ -0,0 +1,10 @@ +# Supported: GCC, CLANG, ICC +TAG ?= CLANG +ENABLE_OPENMP ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/2D-mpi/dcavity.par b/BasicSolver/2D-mpi/dcavity.par new file mode 100644 index 0000000..b4013d6 --- /dev/null +++ b/BasicSolver/2D-mpi/dcavity.par @@ -0,0 +1,46 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 1 # 2 = free-slip 4 = periodic +bcRight 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 10.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +imax 100 # number of interior cells in x-direction +jmax 100 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 5.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-mpi/exe-CLANG b/BasicSolver/2D-mpi/exe-CLANG new file mode 100755 index 0000000000000000000000000000000000000000..aa01ff385a5352d5d5d95bb6195eef41bf5bbae6 GIT binary patch literal 69936 zcmeHweRx#Wx$oY|1fmA_L<7beWmM3pNQt5)2$~@?um|=)Dv5-e7-KSlXu_nK89t7p z(aBVHx0~kJdJgS9-qIeAO?%stb56CCicSI~0TBWU1`v#Bbs|<{tR#Gx+~2#_o{vni z_niCO`^P!Yo`<#9de?f_*Ke)$u6MnA?XdsspGU?Ef|x1@LJHhvaD8cl5E5(%3Br|d zGF(MPk$r*lJI;HTQ0&;BIF`I1MdkE@v7(~bSz0`nh2?J=8z&evnj%CUaa2_JJ!}0G z;=knbtW(h$B+WGHAledTCf^b*nMYPs`22x~d@+vX@*L}Uc_~IGqjWSwx>4RZBch_B z%I9yaU9}=ckL~aLzvJaCG#W*8;*Z84@rUxft9=SMoGYQ@i4{5a#8@+5-A-;a^elkS{)aqqxP(OE=2P zh)0dVO-zWVAoY{4cg3}Mk<;Na))BRe@fHvl{dmq9nQVpbX%$@ z%tD8#PuXy}<3xd~=*tj=B6Ru^gp1(lM;MUHfwRG(%&+~eyhIcpxIpTqh_k^_|GydV zHGZ?M_MvZ9udAyDO~0$)s2#uB=KW;jkLSGk{FPri&iv}ff!h#ggX4XgfWQP6W^+!Q znR+wQc>fT22`Cc~;$8-%!h%hbAI8 zeddhmXlU(2;OFh11wU4fa3jQ`81EApm#*{qJ@wP)C1ewBKptvmBn`O3zsaC&z&#yd z^JgMpB48q5B48q5B48q5B48q5B48q5B48q5B48q5B48q5B48q5B48q5B48q5B48q5 zB48q5B48r$-w}ZlS^bk7{KP7&gL2i7tgj!A^y7X)a5%%C6&#-Azfsnl!*VF>zf%tG zw3hFXwY19n@rqwdm-V?fmI%VDbaUX;bi})2@yAiZt9cYZ3GsQc_?wCU8pLm-YX%WN z*j{6y*$B0;A8U^~96Ig4G&mv#@)RxY?~(u1cm$eQK@)1>DcXeZz+<^(wUd=s-e0v; z5XyJdOqV7%qkKV-CJR(va3pojHR^z@PgsaZxytgSEOw9f1)r!Eq#uQ)m&4^d^0=oh zs0)Ti^A}V{NF&evJ&iD*?6bKIkL4y#VDkk7cz$HVsHty2zGi03-k@l6f62R$97?XmhlP%iwzh$)RDZf{obXKiYO+*F40vr-{Q*e zlJtvFtV?@59bDV!o{_A(yCYs)*3{g4?b{F`gmz0CaY;%Do=NvznHP5E9qj)M(TTIe zAqZ-Buy0t_*AB~=1MQ!t%E3-i9SojCU^Ls8Q6qg8&Kt#Z_o@5j;D_hsp%d{aTJJS> zX~gmhJ(X9iDVbcl)nQ%Ap#Mr}K{W|2YXEK|LJ&5ctdoK3#JAXMm+k zw5w;x_8Pf zD@jRN4XvcbqrdbZTSJh;+Jri)Ll1QG-9jgZ*J%EeXx^z%+lC@9!b)gf1Yr}RuY zYSh-hKoSl+#vd4GS}S&&=#Fuj{Nzp=!vsMU`JZ6B;CL7Dab3&u&x*XriknM2Ze#?i z1mN>$XiF`jgHPo2KaWBoxo*bjvU*v(wDEfcpg#BK)JuKpLFV21CwR`h)2A+}X|ZGK z+4B#tuXNmXdteX+tVEgv`&~G)0@Z20Qf+A&Hc^LGSd2{+(M2r!L2YR@wq?)-L@#a7 ze0~ft=oze)X**~e(q=_IM?=w;Q~w}qb8n=M0IhiE`|m_@qP!S8--DmXcJ*yF@B#;- zf|bc8xT4*91Qe>2)f16RQHrelEFdH46z+1iqT}RmB42yd0$rl@Rhp@aIvDxbMQ~th z>NGV!i^gNOp#lCaQf2K?XhcvR^ul5`_*`ZDX*>zqNnu&uX&(P{EPm-q#6M`1_+Dfs z613YL>hO<;*8keBLYU_d7Dzwsvg?*vYGH%sd;tlqdAsumrJuIzS?TIhzg?d>E7+N$ z2KMTX^c2rsV%masUTTVv{xHun(rC@4>CS)x;0*^F1zPkE2r>Y2!mXb3IN zD_KjOZL(Tu)zE-N{$LoD?A7gwRk_1}j#WnbSF|)*^zjl8$-Kla_>k&U;tXD*UPSVQ zNtIaR-C?ZQ|6`d?8fE(8WoG}aGMQ-mx?&+4d26xOe}}C(a=lUd#CYkm5=#%ZlR#9R zJG-1=CIQF7oq&X*o{bf~oQm$oHDiKz8+-!{+utsl<|P-i7|E|GdSo$*t~ZLVh!o2h)ZoNbh1|@6A z200!nO6SA?7zU+GXJ0ghh#m-v*Py^5&fP;1P&k&sO>dW;{uo1x_>w-5J*(GI0?RM! z&r@ZI@=jLYjcm9I?SLP~(UkniZQK{`X?$h0WO3y7|#Z!AqL)mC4D?F0Po9m=n zojt4n>leYBYM3ed&dWGhli1gzu>ry*o z^btmP#pvCjWpxKeQKtP}3w5IuMbmrWX{=<2_8co2r&=7{Fg<1UNRj>C(U)4m5~&?T zemc{8ETp{$`T zj4N`oe#ehdFSg#uzY|+1EH%3j;DC)LK;^VT89~$`J-Yt*uoNWTJO-9(RI;L0?m`aH0@dT!`cFvTQP~jcKv^;_7kinF_Ej3nM zHZjfQOsrME0u!~}F~yd^_9ZZhOh{l0#@KS%VUr_qkk~#%5|Ch}mBrX*B$W1BVtXxt z?T#4RtZn$F#lTe~woM6am&Vu%5-NR|*uI;<_EAB!3q=WR^NDR?0$Vu7CMU36M{L(8 zu>Cm3R+_*z0xTlukreG|V~p*=gn>FrY=<}-8_n$qyD*w7w~Z5`{oF=OZ3#@%5T+q- zNEpq3A|@??X}rPY-v%LTAoc_?)g&;TguJAZ)+WGOMNIYtrtJpPV+n0?5K|6kx{Rjb zvm|*jFPf(`39weiTYoKa4k8suFwT`R&JANZM}TUin{i5^8_v^e|12^)#@zz$ehJZ# z7*^UtwOLqPGmr`55>ie9C%27WgHYt$-E>63^d%Z`^>X7FRrm>cH&E+eV{Q3rU`okmnHqWV}AJM5JAUyTHAUS|3Y*}G}C zgn4#umXkVp$De|zR-VgR>l{X6+A+j*r5(YOe!;mN6!MeRm8{_U+~cSKz;_@IhLB_w z4I!oWKaRLD=gIrX5^Dq7HH-~{*BJ@UZJ-jR(7Phlx;*VV^n6fyAnhSU*GRK330nrG zX}$SZe+RUr{|a2{)r%4CbRI{G*B_5~P)+EVwDBa0sp)d09lu@(o)e|!BcK_B0}Qc* z6lr5mG~t%SgjA`yE1GawVuD55_-Zua%$$Ux(xm2>q6vM83FD-Vzm6t!B_@oQn*TYP z@Y}?MbZO&Hq6yE%6Y|2ra~Da^h4q|*;On-24J@o9Hr}yEqMQbOe-gbYiM}g|z9osi zDv7=%i9UB%tp1um+HlYd9H+C!2NbQAr5(p~IEE!fX6kUH4Y3&NLJTLJzrV}LDU2l3 zeI#%ZX+KbHV6VQOiinx}qVFr_mOP0b`%}tyymm1Hkt$xs0Hz#@#C!&wFrS%y8`ggH z3uefkzfxA8ZbLrFI=?~6e0nQ_vUZ=fMwV7}sLQOfdKi{Yo4O{eMzUT~BV{hh{zCc? zs`CRdCT2=lsWvHdL8(@ltv*c>0THDrrilk-sWZG7B%SL?s(^JRt?mLWngwg+C%d|l z5P1noiL2^+voLGjRWHtg2c0AZmYNhr+S0dJpYtbh39l94p{OH~)qf&=W4;`V@bHMD zSVS?8$c;r5@`xF+h}(F?tXRagJfgrE`oQmVYx8Bdwz$Zx-Ioiy@eG%?c$O=8K82YN zTvFj%d8e6tMOV03B#mZ(C?*_<`kW{NglRj2a+hTY1Z*6Z&kcWo`Gzs^C9qk?(PMEs z)^`}WC(ef;s*!S!Vx9LQZa4e`I7GZjT=N0r^0aA8hCR?hb$qWC<&oOR-feLGZjAnl zGa4IQ{b>SS`vJiXqhy;ydyODRjpIqqQPm63hD_&HgxJUt(Iab1W`Mn0(N@e-v^51# zXchHCvd_E>=$udlHy%W}!S!vh1X2xjmQ?i86h$u*T`&Zrv^3vb)}?NA#f>?Xi?i4R zC8*~Clph>Uk%B)%#>KkjI6Nw#d!e4yyGWmTVv#=Q9s3{9Ow54Zm_3$7#8*TMs>s=| z=cEUB;Jk7SsFdZr1DuW^e#$-~@syUI1c>j$??b<)?2&OscIEF0>~JCA7CRNOL!oDv zBKEprApRfF$O*Ruja-2xRo3NHS=CFS@5*`xCTu&AWJPu98F=x-jIC;W@Vya)$xbP& z?Q(Fp2#QRnNZE~}`3r*%>9Y1g=`ytGvWQ#lb+z}|V5&sF=Ny5FI(?y@!Ld+vIzKJa zGftqY^hJ8kJ92PGy4uNXPn+HP;&f(R>ETA(gNj-XD{TOl=TH&y(K{?feSQ`T{-%I= z+h@Q#^=KTv8W>4mqO&x!aBI)yvS5cOYpe`e#CtHZtZN}^iY#`apH)zhDROes#t zm%qfyZ%Wrq-j>GBFvv_eb4Bk({vL0c1S+nG-gKW9IWh}Ru!$i(J2XYY0?F;w=h{^PSg{9Jk_a|op9sN4JTS?x};_qLd z{G=2^?9kR&9oh=$j7)CHRZ+p!qWyqv6{>qP);G4GpE7YPStXFmNP>}MSq_N3xr+A0 z3`B&6(QU**#$|@AfCQXrAI&1+m&ukT(y$trFL)3UK!d%uftdR=a&I4J{x`;K)hRP9 zH{H_H#jv2L&sT#E9m8t2IyXog3lN8)Y*7dC(zsR8A0R8&W*y5z+VY`8^|Yz`Wp#Tt z^4O&td~O4}q^QSW(6V5F!st&gG!K9m z>Z8u8-RLNjbtJ}zj({pfY1o1AvOf)6Va7F#xBYmN3DYMtZqZE+b$c5n1wTTzcR_KZ zq;+bL83~`8MZM7HVKXqKbJ(R9V54w`q3keuzfgDFLhXp4m_@qdIE6`G>+^A$0GB8< zXRBT?UJdzKd&)PV>DPdTtZEplt>o#JFA&pls&KmO_4%7QcbDYRx$T zhC@ioc$Xf%ps=x*?t%I82xKD5|2iV~BLeF=(*hl8`7v;mqvXu;Zfu>{^G^hRpoqhY zh&3~!h-cm65qCbUa4T5T*Eg}ZLXrMsSlE!u>#=Q=LvXG`A4%8wE_~MCruBz$F8c#| zE(f>M^u#TT>@-^cY_e!aM7ndCGhfTwg-!ToYOOQp>84AEv90B-o#4S#Y@-) z+urVUR>}m5$y6p_i{w-8u9s*MO3lASUiIir9WFKO8tQS0SO6ze{dkk=-dfY5&lJ&K z;7$!D@eL7yW{7gIV>EEm6&y}&tc#Bx_%p_%b$qC-4h$8~qATMe%xL`Lk?DxEBu82i znU)-xjT$WwJb|AH56eE`P4L%lcSmo3Z2~8RYz(}5g0ZJNRNi= zVBnjAx+0(Tq7zZDPjVvfbfgy{vh;Fgf3uK;U3hy*MoA&Fb)yCh?!=JC-9r|GnvY0r_kh3=p|zmfu) zPMbht0}2Gz<->f!XVQ8h2(IC}R3_vA4kF@F@rX;GgC*C69WM+wDh9gL_S2y@6es#~ zp{8v@Z#$$dd*oDS3*kho6dQ!tO*3Q@!wfO{`flE}&ckf8b&`|~Z9pl2z=_lRbfywX zPd!B))tKZ*0e6K+x8a%%@G>?HtFdWV*`|T`sjc6{5&xMzXP{R)*tv|2y3tH2RDweK z*W!^EHdc!iDmaI@NAO69+XgFV|A%;BU7|(91nm#$4NocHO*n(a?VGTc<>JPH_Vg@p zX#Yq$A#e(Iq)1z^$y+B}frioDCH##ibgNKS4+PIze552!I6oa`xr6vf4LJWKm7Nj_B$x$(}wZz4_onc z3`0;@m0<{t;*V>j3TP+AgYco96nDVi1%D6xeRR-6F*w)ezK63{O5peQ(_ztnJ#uGJ zab;9o?$nK}w&J&t1=)+=f&V%jZg=Hk^d@3BE4tM3;LMV16JTrfrBoN za`^`WP?W5}R&<$$JpuxDTK>AUkR*f?yJ@C-0MlgvtNkZ$@wqZUDLq<{Vf@1}7>5p` zjEVnxD~5lg$T2R8V?1#a5`Zx&5MPV&w~=iu#?MkNhB4(whk$Vb{QsZ$PG>b;5MO{e zkwEN;;VXif@Xa863Hk#G_-2+w@%zr327Y+E3j>a|1nkofto{aq<$X{?p_o`pFbL&u zBEEgigj3lpf{4`<5OiZS9zgNsu>34v+JI5`A#D+`3Fh99%gY$(mD3EQx@po#p)MS% z`u{{HDGf%X?@K zfNVUxL@$0L-KA&Y_*4d=i38yxy=)(`Y;x-vhmiv5mqW@b9c7gK6FAPdxbn}!+o$C3 zgSVeaQ5N%(oPPr5lJFwkzYAx5=gV?FPWR5=yYu(pbcEycQ92~K@{hu3(1o(dcJTd@ zea&WOJ1^->{m4RW-0VrM18;H_qLLkrO?k2w%dB8(2o%)u;FTLIZPwY zuoP-x8Q%-Ohcnl=A%YIN+CPQ}!h(h5YQ7k$*dce#VjcL{rI($Ug$?ozPJfDvn)@KG zX1UcjUBUN9Ttmlb<d+6#tz@KlETu#2ASfzENR50Phk5OEEBEzzN@qh9f$ENRsjVi3Er9Vt~ka@7$G$e0m) z2f`Fbl|yty#7T9Lp7jlv?mr~Bb>#$&S6C5`x%3=Ln*+=u^DYjd&gKp&l!H17Yy(p0as+Hg6|JNPKm+6o&<>S@YzN5UxSYrA z{fuTipIo!hE>ztCkS&Ke$gs_|yKoA04IRlFrTh4~6Qr9Um=dfPz#Bjf15XFgwiXk{ zWx#mZ0cqJ0DYKZ6E*l}thXJN#m{trnwqC{q?hAn&R;msp zgIvr(-W3Nqn}NJYf9x=>Sd?ID=Kyv>+1Z21(}~GL%O{F5fTbzUQFi4I5Xy4?+YJ9+ zhVmXMMCS&E@@51GWwP>>0OiAIb0W&^E}YfLw#x1Y_>hX~f$k2VJO|rq7f_B(Z3Rc} zU>`QxURco=z+FdfSZ{-(?-r zvOTi+E*LB$cqk)QSN>k9c_kp{*4=J-sKp0V_f!!NQ|E9<%~VeF z&S@whsdLw&kz!KksK2CZCUlGTbr?#~2i7fPMpCyh@SW*n`s3Kg&NX9tW5KCD(-P}T zPe0wkqfvukG-RjW7`-fsh6`=CDu=DO9X~mt7ck}{9%e8~{feEr4kmcwfi9h>bUlZm^B_C<|cYy$REbZM?q(2i|N+8#hxxSEi(l-CG;8un%vA%~3oJ z{S)`z_1ox#97SM?UEJ-e`kZ6H`cH3DVn%^CqJ5W|3s8PsD8e$F0Y#i2XfQenSr2j8 z-hfmq&VY4*%*O_YV;?`GMp>B2gSy_<{~q=lE)d^F=mH&)o+^W+OzO-tGU6PnyT_#m za35jLF5rXh=QFo{AFhsDI#D~s;b9COlY+sHVFlO2pLN1|ewIr@%qw~yaE@;{2SIfx z`8z;$Ihx%NdXE7uyX^}p;K#51LqCk#)%NRkr>BY)8icQq~DP%1&7Mwt>Xr5n?GD$!3hX>2vdb&>R^~EgsIhWoQdG$Oa!xu zt$YQuCqYX)jukr&&E65hR~eZ_IT8(}U$OYF#?0Mk`u_rvLtQ)Np=bYmV~fB6dI2#0 ziq^Uu+>>r>c@x>9nrOwA?-2LnE7pX%vCGe%9Np#71j{69$0WW<)Sz6x3mV(PRP29m zQ0YRVbi@{a`Wi$@m*Wi@-w~wdn~@iL_;Dn}(SrReUcwkbnE^$i8zp4yBL-6A#z@I% zvVtRyu_?UV-vuT^pPBui>~?ZbA45@NdOA7Y&cE6RqPyNgl2>uQG5v`O{tNboCqm{& z`?8H~a6n9m)a|4V4xbdYO3ga0YwhV*5RXmHU(ZgKkAjVDa9*S`0ak+mj9>WNq9BEA z<6H^uhhXp2X9~D&p|4F*)c5fcjqIJ=xNSL2J8TE-!0ogHTiWsFRD>eFV>Iwie^#{U zON^rN>K|`z+&$20=w(cp-3qY>&*R9;E;O+NM<5Kd<0J*D02++fVOZot_@WE%ncQ2E zlIPZQI`Oj1Nc{v+jX2klna=WEXzRO39qc62Hh=ZR>cHnNm_`4XLz_br6}za*0$B*{ zvJl5K3non=CQZu1E(mdog*7n)s$Taz#HPCNszcAngeDFfVh1$xGh*O1g4TX|Ct5z8 zrnj^OU#$QfSAT$ zx%5J?ln^Yp{WUB=Lt(H09|lVfU~#tcQ$h@uzsAF20Vrm(ThUdfguTTAP{RHpI!9+mCsH0wq@Z(a z%2p%^eC9MWh@4xoM|85IX`cmJtf8rV3jHw`$EI;kd;-i*p)4!A-x`-F%=l}$jJ5AF zrbJG{Y<5gy!OJOlISXDv!7E_&hQ5;l%QDRfvMggpZ0ED#2Yfc*3J?u{C+TCEmd^f% z5zWTjOX*2Hc--hg;;K_STM9-MyP(S3>RMv^SW0=$(S3)$lVA5ry z6w-?`u4D%p<;r11B9TW<=8==p_BSEraKRGVC2Kq00JQVAzX@K9pmXckDU1yutqcXx z^xY-&{{~#FD=!fHn6CJJd^*R4$A7FFW7==Il0Y8Qddp;rx^UZfQQQ4jU8vn`II#kP zJ4+Mn--fI&jLP~8?Ps*@B?$NA?UZeW%0#}sJ$)80N^l11AO9C<0JM>gd@=TQg@fNG zj@b)g0X(PZzwZWq4s8n=3mw`on3YglAVV&~9`ITlfvex~`6TGu_$cD7%V3I!t>qma zebR{D&Z9#wU|rywLRlca6MT}OMrD;^lkB9z9miqiXlKTJ~IzLifRbOJZ6x#4u zlrC*)7h$Ho8AjGzSXY<#;SM3)5xDvsYJws49Skh2hL!=$85|NbzOHyv-qr&B%zs&nk->Q37z|)<&%l);Yw?H`iFiMO*X!(MLmA#%WagZu_n>EyWK~A!RY&p2j0>=p zokbe>V0`GpCJ*ajt8w#)&hh>`(3gDvsy}!3KK5)GP)OZa6$Z*J?A_D?7BA zBG_HdRs@!$BgKtw#O`vk+i##3ti=RQv!W1iU|p#1;zDO+X)b`QPGc#jy+jpe)T`Vv^~*NeDp#19wF*4poXXd%MEyzI&Q$6Ed87!GO(UVe)cp)c7($ z2*Pp000R{L1)$Szpa#7``Sudi%Yrmrq1Lh55y9`T0Y(5)3W#U{a6ADR8gK{?Tsh>30c+eJ1cq2NC=>(E3dSTZ zr9Fo(ewPxa4B;<``U~2fIrDn|2^yE()!ZCLI^bFG#cScR6`(Lp2j zb{2aZYDK>?EqRLgAym>#NLpO%pvxUlNw?v}fuiB7VmdjYO}O%ab4nXT1btv+$)t>v zuv#H2MgxC7bur`^dY<<($*y_`EdF;aA~rFddm1V&3`NY?IYjEMGQ>jek+|7!(o88p2+`~$^2e~^f z@@Plk0A4QmA0;J4?Ys%sxXEq=8#8W_r24O7?`PTUCn1t)p$pqMigk_#Fp=7o42n-6*#i)rz~*` zx1nene=i5U8zJS<3INUl{d3U#tPZA2Dx^!|O*GRT&l$QSY}I?&E4!;n8zgW>*Z@F1 zB@Ef!I%U*2(6Dg=_FXsOvh_ZB2=;v_${(frug2?LG%IIc3&{`M6V0@cZbw(FMgyP7 z_ycf_ZeoE1$~HQj&EKbk-?0xuI@7U-M73kz01M-ZFzJjAdV@nlW|WD$b=`s8xV=QG zTf*s$!NBip+=ivpzmV1w|NC=vzFO+w_br?_EU`N^+CP$#7{V>4FQ{AE0$lrAg3oZC zFjUKm0#wVs)sgi^d~w4~%ZQsG;+rt+UD$daxPtC9F^y1e{}6v-1p*tNkiaN(h>k(B z{@4M0MCs_k%i@`DV~{~);C#>y8fI%?VQ1IJ;;RAYUD(r-?(gPgW6LbdPRD-L3y00M zG>6W;IoF^yNW@;8bCr>Z?{Z3W=$xGM4V(+oai=1%-7UV3u~o!wMck)|yA^Sd8`l&6 z!T7d6UisJEU)hW8svCcVVw1MON+R8%J&}c)nL144%|C5wmG!v~r&FizXn+?&VmU zduaV4eJi{7b8|Pk3b)+ditf7G>}&$Gz$OhqZiZr9n%07RV89JN$o;Hy&|RU8qg;0z zzOH=)%b32Qa_4^)*uy^aBmMk-#4_nlchypHc|u!{*99~K)8?VHK+`FkU9U*Ea~R8c zEpn1|6Mu4MC^CRP1CKQ@?gNhp;YGfrC_1T=puo5N3=hrIjrXIQP)m>W%w0&JSm~KO zgz!$;P~DvymgqCjU}DfWO6S?I;cN#!bG|m zt`r8@vCVqfd4p2w%w^Ys%BG_4>J`B?bK%wPW$rzI+Xd@z2g$-_TC;O;u2U!mBLFZ;E zGzl!IX{Q{(kOQ?rouj5Cj zzQUfbvS*k*JJ@q4dv>yCH+$}3&%Nxqk3D)7jYQZJceh*VV3ARqt8lw>4BYR@Qs`o<>`(&$i0zw^iC2ylX&uYi#~C-ixC- zT&sM6nwr|GTFO-KsrNRnv-!QYnm}DmZC#zsztUrC^n5qq@%dv}8oa*RwH4s4q}psX zm9=%A>T53+?yX$)$ZXrx>S?w@ueY(&e~ z%Ag!*<i#@d~RzBnnH2Q404|)B5Z~ZhhuVI?4&Qs%`W@|*MKVApb&9bou)UK-Z zdytQsL1nTOh5pA^TeY{ovUZithc?C1*HU_+x4u5Gs#~)~zEtL2wS9s77 zpKT8QFX)=u74y;KyDt`&dg}s=&+ql3)zfdQ@m(x5qKrUYHN%|#Q`v-(0HUmziv^po zw$8I^g@2{6&hTsNE7uAS)1wxHM!u)PCwS{u2#p@WUl|ZqRMyv53e|qW?-5q46;`Yh zR$+jJhpH4H0q3-i2zRn?Uk zdi1ZlmipwgQMt1L8!MOsykrWYXm~VSo^`9db=8<*f%=C$n1BQV;$5iKR@vBCxz0D+ zRx+DnZTE4nhpd$fZMjRA+>2FHSGkr|L488@1sbzCpWr>MNXdG)F6Cu~q?WZvgBz^<n&exDI9ggH&xAN{Na%)`At?)MmceXkkuh8N?f&s^yD1bf0a7@Cs&0)saE zX#IWnJa_np$R*P=@1}m??VrwZrD?cXE{c_6s!0E5y#tZtoa-o= zhoRBn|FO0Tijk`a8LXEIsbMsCz$%Ds6NQukShpX%LP%@Dd!X=@f~D;$L2S8MNa?v% z5FcD3q_z|ZV$)JV$h^cNUMr+fv+%Pv@qo37x5wHB7Dc?Tq;B0WS;mY71hx-ZKHn?86({S0MAkKtS;2wlq z4fhP(D{v>^E=v(orok1#`Qd&7*9O-KcM$Fb+*!EnR3T*w+)TIvxFWa*;OgKWhkFL@ zpHcd-RyO)8s=VIDYF}bRqo>kuurrPo9)CizZ)I&wl=jxwS2R|xdc;U3 z(pUR;MqWp4y|1Dwp|JV2t19ba$qOp|E29l@t*SMObTxXaR`cRqKF_LZ9$f6HU9s{Z zZ(~JeU7g>H#~L1Aysp864%F92v-1d6XN52D5IS4Aj*|Ezu{L0xL-$&2E8hp;P+xau=*Q4-wk+t{)$xr z0FT8}AzXwtCI}Z-)UO00o~nS~Q?b%h30#G=O5wZckQqc_LN$h7HVpJB}VD_ zRMw;-;TNeG-s+;t#>#q+9~pO~`h@RU7)bYee1ST@@EeOo7>Hm=r&L!q_)G2+9wFpt zPKU%)p=F_vDhw zME+XuD&efSlr4c#6xOGpu+0ykDr)N6qHmrzunO~PoiKpZfkl*BMh^=0xxrU2{6GYj z9^oHTEJE8`Xl3gB1;XE_+$%2?wxm$UeSS}a@DftPPl|#?*!)v^_IytLK~ueT0oIB? zrY!YWVkmmS*z&muY)?{XP((#NkAOD@P$Z?%y8?64Cp4#~3QcQ~RwVp6WvRDrHQ?KX zxSsXY#2P%pfyA6YPmSh$6E*z25mKHnzE8;#gp?-VzeMM_&GhfUGr}K0S3F;Sw%|?2&LH8Nu{lK7SJk0sAWikG60jFOy=*^h( zRR8Y{x(73X=sgDAV$j&?Sbpm|p8ipk{|-)L8)WGMI!5^|cu@O#?&h@Bpl!EvdXhnh zX*TG#6Hw6x&NPzeeAmW5vIFflc!`%W0F)Q2(2k|99#0fLt^5AZR zgGdm*1vd*W9}XfxxC@SCFjOqU0h7VC8 zEQPbd{l9E0ohcypNHqOlw;N(gm=CuAPKI;A{m1%JaREK4qS8=mNEhZ>4bue2Dl;r9 zrq3{n>)50;y5d?2OG#98SVUacVI)^~C`gJ6w3TR(be3p1rn!&?OhkgN5{=>-%NVLc zs30vbuFi2XsuM8E&{UwglD5v(z?gPp(6J0-mBR|3$NyKl2 +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_THREADS 128 +#define gettid() syscall(SYS_gettid) + +static int getProcessorID(cpu_set_t* cpu_set) +{ + int processorId; + + for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { + if (CPU_ISSET(processorId, cpu_set)) { + break; + } + } + return processorId; +} + +int affinity_getProcessorId() +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); + + return getProcessorID(&cpu_set); +} + +void affinity_pinThread(int processorId) +{ + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); +} + +void affinity_pinProcess(int processorId) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} +#endif /*_OPENMP*/ +#endif /*__linux__*/ diff --git a/BasicSolver/2D-mpi/src/affinity.h b/BasicSolver/2D-mpi/src/affinity.h new file mode 100644 index 0000000..d844fe5 --- /dev/null +++ b/BasicSolver/2D-mpi/src/affinity.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef AFFINITY_H +#define AFFINITY_H + +extern int affinity_getProcessorId(); +extern void affinity_pinProcess(int); +extern void affinity_pinThread(int); + +#endif /*AFFINITY_H*/ diff --git a/BasicSolver/2D-mpi/src/allocate.c b/BasicSolver/2D-mpi/src/allocate.c new file mode 100644 index 0000000..81e1e9d --- /dev/null +++ b/BasicSolver/2D-mpi/src/allocate.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +void* allocate(int alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/BasicSolver/2D-mpi/src/allocate.h b/BasicSolver/2D-mpi/src/allocate.h new file mode 100644 index 0000000..54cfe06 --- /dev/null +++ b/BasicSolver/2D-mpi/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(int alignment, size_t bytesize); + +#endif diff --git a/BasicSolver/2D-mpi/src/comm.c b/BasicSolver/2D-mpi/src/comm.c new file mode 100644 index 0000000..eeab2a7 --- /dev/null +++ b/BasicSolver/2D-mpi/src/comm.c @@ -0,0 +1,326 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "comm.h" + +// subroutines local to this module +static int sizeOfRank(int rank, int size, int N) +{ + return N / size + ((N % size > rank) ? 1 : 0); +} + +static void setupCommunication(Comm* c, int direction, int layer) +{ + MPI_Datatype type; + size_t dblsize = sizeof(double); + int imaxLocal = c->imaxLocal; + int jmaxLocal = c->jmaxLocal; + int sizes[NDIMS]; + int subSizes[NDIMS]; + int starts[NDIMS]; + int offset = 0; + + sizes[IDIM] = imaxLocal + 2; + sizes[JDIM] = jmaxLocal + 2; + + if (layer == HALO) { + offset = 1; + } + + switch (direction) { + case LEFT: + subSizes[IDIM] = 1; + subSizes[JDIM] = jmaxLocal; + starts[IDIM] = 1 - offset; + starts[JDIM] = 1; + break; + case RIGHT: + subSizes[IDIM] = 1; + subSizes[JDIM] = jmaxLocal; + starts[IDIM] = imaxLocal + offset; + starts[JDIM] = 1; + break; + case BOTTOM: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = 1; + starts[IDIM] = 1; + starts[JDIM] = 1 - offset; + break; + case TOP: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = 1; + starts[IDIM] = 1; + starts[JDIM] = jmaxLocal + offset; + break; + } + + MPI_Type_create_subarray(NDIMS, + sizes, + subSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &type); + MPI_Type_commit(&type); + + if (layer == HALO) { + c->rbufferTypes[direction] = type; + } else if (layer == BULK) { + c->sbufferTypes[direction] = type; + } +} + +static void assembleResult(Comm* c, + double* src, + double* dst, + int imaxLocal[], + int jmaxLocal[], + int offset[], + int jmax, + int imax) +{ + MPI_Request* requests; + int numRequests = 1; + + if (c->rank == 0) { + numRequests = c->size + 1; + } else { + numRequests = 1; + } + + requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); + + /* all ranks send their bulk array */ + MPI_Datatype bulkType; + int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; + int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; + int starts[NDIMS] = { 1, 1 }; + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + + MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]); + + /* rank 0 assembles the subdomains */ + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + MPI_Datatype domainType; + int oldSizes[NDIMS] = { jmax, imax }; + int newSizes[NDIMS] = { jmaxLocal[i], imaxLocal[i] }; + int starts[NDIMS] = { offset[i * NDIMS + JDIM], offset[i * NDIMS + IDIM] }; + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} + +static int sum(int* sizes, int position) +{ + int sum = 0; + + for (int i = 0; i < position; i++) { + sum += sizes[i]; + } + + return sum; +} + +// exported subroutines +void commReduction(double* v, int op) +{ + if (op == MAX) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + } else if (op == SUM) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + } +} + +int commIsBoundary(Comm* c, int direction) +{ + switch (direction) { + case LEFT: + return c->coords[IDIM] == 0; + break; + case RIGHT: + return c->coords[IDIM] == (c->dims[IDIM] - 1); + break; + case BOTTOM: + return c->coords[JDIM] == 0; + break; + case TOP: + return c->coords[JDIM] == (c->dims[JDIM] - 1); + break; + } + + return 0; +} + +void commExchange(Comm* c, double* grid) +{ + int counts[NDIRS] = { 1, 1, 1, 1 }; + MPI_Aint displs[NDIRS] = { 0, 0, 0, 0 }; + + MPI_Neighbor_alltoallw(grid, + counts, + displs, + c->sbufferTypes, + grid, + counts, + displs, + c->rbufferTypes, + c->comm); +} + +void commShift(Comm* c, double* f, double* g) +{ + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + MPI_Irecv(g, + 1, + c->rbufferTypes[BOTTOM], + c->neighbours[BOTTOM], + 0, + c->comm, + &requests[0]); + + /* send ghost cells to top neighbor */ + MPI_Isend(g, 1, c->sbufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]); + + /* shift F */ + /* receive ghost cells from left neighbor */ + MPI_Irecv(f, 1, c->rbufferTypes[LEFT], c->neighbours[LEFT], 1, c->comm, &requests[2]); + + /* send ghost cells to right neighbor */ + MPI_Isend(f, + 1, + c->sbufferTypes[RIGHT], + c->neighbours[RIGHT], + 1, + c->comm, + &requests[3]); + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +} + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int jmax, + int imax) +{ + int offset[c->size * NDIMS]; + int imaxLocal[c->size]; + int jmaxLocal[c->size]; + + MPI_Gather(&c->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&c->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + int coords[NDIMS]; + MPI_Cart_coords(c->comm, i, NDIMS, coords); + offset[i * NDIMS + IDIM] = sum(imaxLocal, coords[IDIM]); + offset[i * NDIMS + JDIM] = sum(jmaxLocal, coords[JDIM]); + printf("Rank: %d, Coords(j,i): %d %d, Size(j,i): %d %d " + "Offset(j,i): %d %d\n", + i, + coords[JDIM], + coords[IDIM], + jmaxLocal[i], + imaxLocal[i], + offset[i * NDIMS + JDIM], + offset[i * NDIMS + IDIM]); + } + } + + /* collect P */ + assembleResult(c, p, pg, imaxLocal, jmaxLocal, offset, jmax, imax); + + /* collect U */ + assembleResult(c, u, ug, imaxLocal, jmaxLocal, offset, jmax, imax); + + /* collect V */ + assembleResult(c, v, vg, imaxLocal, jmaxLocal, offset, jmax, imax); +} + +void commPrintConfig(Comm* c) +{ + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + if (commIsMaster(c)) { + printf("Communication setup:\n"); + } + + for (int i = 0; i < c->size; i++) { + if (i == c->rank) { + printf("\tRank %d of %d\n", c->rank, c->size); + printf("\tNeighbours (bottom, top, left, right): %d %d, %d, %d\n", + c->neighbours[BOTTOM], + c->neighbours[TOP], + c->neighbours[LEFT], + c->neighbours[RIGHT]); + printf("\tCoordinates (j,i) %d %d\n", c->coords[JDIM], c->coords[IDIM]); + printf("\tLocal domain size (j,i) %dx%d\n", c->jmaxLocal, c->imaxLocal); + fflush(stdout); + } + } + MPI_Barrier(MPI_COMM_WORLD); +} + +void commInit(Comm* c, int jmax, int imax) +{ + /* setup communication */ + MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank)); + MPI_Comm_size(MPI_COMM_WORLD, &(c->size)); + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + MPI_Dims_create(c->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm); + MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]); + MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]); + MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords); + + c->imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax); + c->jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax); + + // setup buffer types for communication + setupCommunication(c, LEFT, BULK); + setupCommunication(c, LEFT, HALO); + setupCommunication(c, RIGHT, BULK); + setupCommunication(c, RIGHT, HALO); + setupCommunication(c, BOTTOM, BULK); + setupCommunication(c, BOTTOM, HALO); + setupCommunication(c, TOP, BULK); + setupCommunication(c, TOP, HALO); +} diff --git a/BasicSolver/2D-mpi/src/comm.h b/BasicSolver/2D-mpi/src/comm.h new file mode 100644 index 0000000..11782f1 --- /dev/null +++ b/BasicSolver/2D-mpi/src/comm.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __COMM_H_ +#define __COMM_H_ +#include + +enum direction { LEFT = 0, RIGHT, BOTTOM, TOP, NDIRS }; +enum dimension { JDIM = 0, IDIM, NDIMS }; +enum layer { HALO = 0, BULK }; +enum op { MAX = 0, SUM }; + +typedef struct { + int rank; + int size; + MPI_Comm comm; + MPI_Datatype sbufferTypes[NDIRS]; + MPI_Datatype rbufferTypes[NDIRS]; + int neighbours[NDIRS]; + int coords[NDIMS], dims[NDIMS]; + int imaxLocal, jmaxLocal; +} Comm; + +extern void commInit(Comm* c, int jmax, int imax); +extern void commPrintConfig(Comm*); +extern void commExchange(Comm*, double*); +extern void commShift(Comm* c, double* f, double* g); +extern void commReduction(double* v, int op); +extern int commIsBoundary(Comm* c, int direction); +extern void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int jmax, + int imax); + +static inline int commIsMaster(Comm* c) { return c->rank == 0; } +#endif // __COMM_H_ diff --git a/BasicSolver/2D-mpi/src/likwid-marker.h b/BasicSolver/2D-mpi/src/likwid-marker.h new file mode 100644 index 0000000..c3770c0 --- /dev/null +++ b/BasicSolver/2D-mpi/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/BasicSolver/2D-mpi/src/main.c b/BasicSolver/2D-mpi/src/main.c new file mode 100644 index 0000000..41bb084 --- /dev/null +++ b/BasicSolver/2D-mpi/src/main.c @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "parameter.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" +#include + +int main(int argc, char** argv) +{ + int rank; + double S, E; + Parameter params; + Solver solver; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + initParameter(¶ms); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(¶ms, argv[1]); + if (rank == 0) { + printParameter(¶ms); + } + initSolver(&solver, ¶ms); + /* debugExchange(&solver); */ + /* exit(EXIT_SUCCESS); */ + initProgress(solver.te); + + double tau = solver.tau; + double te = solver.te; + double t = 0.0; + + S = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) { + computeTimestep(&solver); + } + + setBoundaryConditions(&solver); + setSpecialBoundaryCondition(&solver); + computeFG(&solver); + computeRHS(&solver); + solve(&solver); + adaptUV(&solver); + t += solver.dt; + +#ifdef VERBOSE + if (rank == 0) { + printf("TIME %f , TIMESTEP %f\n", t, solver.dt); + } +#else + printProgress(t); +#endif + } + E = getTimeStamp(); + stopProgress(); + if (rank == 0) { + printf("Solution took %.2fs\n", E - S); + } + size_t bytesize = solver.imax * solver.jmax * sizeof(double); + + double* ug = allocate(64, bytesize); + double* vg = allocate(64, bytesize); + double* pg = allocate(64, bytesize); + + commCollectResult(&solver.comm, + ug, + vg, + pg, + solver.u, + solver.v, + solver.p, + solver.jmax, + solver.imax); + writeResult(&solver, ug, vg, pg); + + MPI_Finalize(); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/2D-mpi/src/parameter.c b/BasicSolver/2D-mpi/src/parameter.c new file mode 100644 index 0000000..84304aa --- /dev/null +++ b/BasicSolver/2D-mpi/src/parameter.c @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.8; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_STRING(name); + PARSE_INT(bcLeft); + PARSE_INT(bcRight); + PARSE_INT(bcBottom); + PARSE_INT(bcTop); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(p_init); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + param->bcLeft, + param->bcRight, + param->bcBottom, + param->bcTop); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength); + printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); +} diff --git a/BasicSolver/2D-mpi/src/parameter.h b/BasicSolver/2D-mpi/src/parameter.h new file mode 100644 index 0000000..f4c331a --- /dev/null +++ b/BasicSolver/2D-mpi/src/parameter.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + double xlength, ylength; + int imax, jmax; + int itermax; + double eps, omg; + double re, tau, gamma; + double te, dt; + double gx, gy; + char* name; + int bcLeft, bcRight, bcBottom, bcTop; + double u_init, v_init, p_init; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/BasicSolver/2D-mpi/src/progress.c b/BasicSolver/2D-mpi/src/progress.c new file mode 100644 index 0000000..31a8a90 --- /dev/null +++ b/BasicSolver/2D-mpi/src/progress.c @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; +static int _rank = -1; + +void initProgress(double end) +{ + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); + _end = end; + _current = 0; + + if (_rank == 0) { + printf("[ ]"); + fflush(stdout); + } +} + +void printProgress(double current) +{ + if (_rank == 0) { + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); + } +} + +void stopProgress() +{ + if (_rank == 0) { + printf("\n"); + fflush(stdout); + } +} diff --git a/BasicSolver/2D-mpi/src/progress.h b/BasicSolver/2D-mpi/src/progress.h new file mode 100644 index 0000000..9ef2d96 --- /dev/null +++ b/BasicSolver/2D-mpi/src/progress.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ + +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(); + +#endif diff --git a/BasicSolver/2D-mpi/src/solver.c b/BasicSolver/2D-mpi/src/solver.c new file mode 100644 index 0000000..cbff349 --- /dev/null +++ b/BasicSolver/2D-mpi/src/solver.c @@ -0,0 +1,546 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +#define P(i, j) p[(j) * (imaxLocal + 2) + (i)] +#define F(i, j) f[(j) * (imaxLocal + 2) + (i)] +#define G(i, j) g[(j) * (imaxLocal + 2) + (i)] +#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] +#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)] + +static void printConfig(Solver* s) +{ + if (commIsMaster(&s->comm)) { + printf("Parameters for #%s#\n", s->problem); + printf("BC Left:%d Right:%d Bottom:%d Top:%d\n", + s->bcLeft, + s->bcRight, + s->bcBottom, + s->bcTop); + printf("\tReynolds number: %.2f\n", s->re); + printf("\tGx Gy: %.2f %.2f\n", s->gx, s->gy); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", s->xlength, s->ylength); + printf("\tCells (x, y): %d, %d\n", s->imax, s->jmax); + printf("\tCell size (dx, dy): %f, %f\n", s->dx, s->dy); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te); + printf("\tdt bound: %.6f\n", s->dtBound); + printf("\tTau factor: %.2f\n", s->tau); + printf("Iterative s parameters:\n"); + printf("\tMax iterations: %d\n", s->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", s->eps); + printf("\tgamma factor: %f\n", s->gamma); + printf("\tomega (SOR relaxation): %f\n", s->omega); + } + commPrintConfig(&s->comm); +} + +void initSolver(Solver* s, Parameter* params) +{ + s->problem = params->name; + s->bcLeft = params->bcLeft; + s->bcRight = params->bcRight; + s->bcBottom = params->bcBottom; + s->bcTop = params->bcTop; + s->imax = params->imax; + s->jmax = params->jmax; + s->xlength = params->xlength; + s->ylength = params->ylength; + s->dx = params->xlength / params->imax; + s->dy = params->ylength / params->jmax; + s->eps = params->eps; + s->omega = params->omg; + s->itermax = params->itermax; + s->re = params->re; + s->gx = params->gx; + s->gy = params->gy; + s->dt = params->dt; + s->te = params->te; + s->tau = params->tau; + s->gamma = params->gamma; + + commInit(&s->comm, s->jmax, s->imax); + + /* allocate arrays */ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + size_t size = (imaxLocal + 2) * (jmaxLocal + 2); + + s->u = allocate(64, size * sizeof(double)); + s->v = allocate(64, size * sizeof(double)); + s->p = allocate(64, size * sizeof(double)); + s->rhs = allocate(64, size * sizeof(double)); + s->f = allocate(64, size * sizeof(double)); + s->g = allocate(64, size * sizeof(double)); + + for (int i = 0; i < size; i++) { + s->u[i] = params->u_init; + s->v[i] = params->v_init; + s->p[i] = params->p_init; + s->rhs[i] = 0.0; + s->f[i] = 0.0; + s->g[i] = 0.0; + } + + double dx = s->dx; + double dy = s->dy; + + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy); + s->dtBound = 0.5 * s->re * 1.0 / invSqrSum; +#ifdef VERBOSE + printConfig(s); +#endif +} + +void computeRHS(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + double idx = 1.0 / s->dx; + double idy = 1.0 / s->dy; + double idt = 1.0 / s->dt; + double* rhs = s->rhs; + double* f = s->f; + double* g = s->g; + + commShift(&s->comm, f, g); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) * + idt; + } + } +} + +int solve(Solver* s) +{ + int imax = s->imax; + int jmax = s->jmax; + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->dx * s->dx; + double dy2 = s->dy * s->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* p = s->p; + double* rhs = s->rhs; + double epssq = eps * eps; + int it = 0; + double res = 1.0; + commExchange(&s->comm, p); + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + } + + if (commIsBoundary(&s->comm, BOTTOM)) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (commIsBoundary(&s->comm, TOP)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (commIsBoundary(&s->comm, LEFT)) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (commIsBoundary(&s->comm, RIGHT)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } + + commReduction(&res, SUM); + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (commIsMaster(&s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + it++; + } + +#ifdef VERBOSE + if (commIsMaster(&s->comm)) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + if (res < eps) { + return 0; + } else { + return 1; + } +} + +static double maxElement(Solver* s, double* m) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int size = (imaxLocal + 2) * (jmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + commReduction(&maxval, MAX); + return maxval; +} + +void computeTimestep(Solver* s) +{ + double dt = s->dtBound; + double dx = s->dx; + double dy = s->dy; + double umax = maxElement(s, s->u); + double vmax = maxElement(s, s->v); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + + s->dt = dt * s->tau; +} + +void setBoundaryConditions(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + double* u = s->u; + double* v = s->v; + + if (commIsBoundary(&s->comm, TOP)) { + switch (s->bcTop) { + case NOSLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = -U(i, jmaxLocal); + } + break; + case SLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + } + break; + case OUTFLOW: + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + V(i, jmaxLocal) = V(i, jmaxLocal - 1); + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, BOTTOM)) { + switch (s->bcBottom) { + case NOSLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = -U(i, 1); + } + break; + case SLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = U(i, 1); + } + break; + case OUTFLOW: + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0) = U(i, 1); + V(i, 0) = V(i, 1); + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, RIGHT)) { + switch (s->bcRight) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = 0.0; + V(imaxLocal + 1, j) = -V(imaxLocal, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = 0.0; + V(imaxLocal + 1, j) = V(imaxLocal, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = U(imaxLocal - 1, j); + V(imaxLocal + 1, j) = V(imaxLocal, j); + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, LEFT)) { + switch (s->bcLeft) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = -V(1, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = V(1, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = U(1, j); + V(0, j) = V(1, j); + } + break; + case PERIODIC: + break; + } + } +} + +void setSpecialBoundaryCondition(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + double* u = s->u; + + if (strcmp(s->problem, "dcavity") == 0) { + if (commIsBoundary(&s->comm, TOP)) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal); + } + } + } else if (strcmp(s->problem, "canal") == 0) { + if (commIsBoundary(&s->comm, LEFT)) { + double ylength = s->ylength; + double dy = s->dy; + int rest = s->jmax % s->comm.size; + int yc = s->comm.rank * (s->jmax / s->comm.size) + MIN(rest, s->comm.rank); + double ys = dy * (yc + 0.5); + double y; + + /* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */ + + for (int j = 1; j < jmaxLocal + 1; j++) { + y = ys + dy * (j - 0.5); + U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } + } + /* print(solver, solver->u); */ +} + +void computeFG(Solver* s) +{ + double* u = s->u; + double* v = s->v; + double* f = s->f; + double* g = s->g; + + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + + double gx = s->gx; + double gy = s->gy; + double gamma = s->gamma; + double dt = s->dt; + double inverseRe = 1.0 / s->re; + double inverseDx = 1.0 / s->dx; + double inverseDy = 1.0 / s->dy; + double du2dx, dv2dy, duvdx, duvdy; + double du2dx2, du2dy2, dv2dx2, dv2dy2; + + commExchange(&s->comm, u); + commExchange(&s->comm, v); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + du2dx = inverseDx * 0.25 * + ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - + (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + + fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); + + duvdy = inverseDy * 0.25 * + ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - + (V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + + fabs(V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) - U(i, j - 1))); + + du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); + du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); + F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - + (U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + + fabs(U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) - V(i - 1, j))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - + (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + + fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); + + dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); + dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); + G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); + } + } + + /* ----------------------------- boundary of F --------------------------- */ + if (commIsBoundary(&s->comm, LEFT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(0, j) = U(0, j); + } + } + + if (commIsBoundary(&s->comm, RIGHT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(imaxLocal, j) = U(imaxLocal, j); + } + } + + /* ----------------------------- boundary of G --------------------------- */ + if (commIsBoundary(&s->comm, BOTTOM)) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, 0) = V(i, 0); + } + } + + if (commIsBoundary(&s->comm, TOP)) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, jmaxLocal) = V(i, jmaxLocal); + } + } +} + +void adaptUV(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + + double* p = s->p; + double* u = s->u; + double* v = s->v; + double* f = s->f; + double* g = s->g; + + double factorX = s->dt / s->dx; + double factorY = s->dt / s->dy; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; + V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; + } + } +} + +void writeResult(Solver* s, double* u, double* v, double* p) +{ + int imax = s->imax; + int jmax = s->jmax; + double dx = s->dx; + double dy = s->dy; + double x = 0.0, y = 0.0; + + FILE* fp; + fp = fopen("pressure.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax; j++) { + y = (double)(j - 0.5) * dy; + for (int i = 1; i < imax; i++) { + x = (double)(i - 0.5) * dx; + fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]); + } + fprintf(fp, "\n"); + } + + fclose(fp); + + fp = fopen("velocity.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax; j++) { + y = dy * (j - 0.5); + for (int i = 1; i < imax; i++) { + x = dx * (i - 0.5); + double vel_u = (u[j * (imax) + i] + u[j * (imax) + (i - 1)]) / 2.0; + double vel_v = (v[j * (imax) + i] + v[(j - 1) * (imax) + i]) / 2.0; + double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); + } + } + + fclose(fp); +} diff --git a/BasicSolver/2D-mpi/src/solver.h b/BasicSolver/2D-mpi/src/solver.h new file mode 100644 index 0000000..b13dbe2 --- /dev/null +++ b/BasicSolver/2D-mpi/src/solver.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "comm.h" +#include "parameter.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + double dx, dy; + int imax, jmax; + double xlength, ylength; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop; + /* communication */ + Comm comm; +} Solver; + +void initSolver(Solver*, Parameter*); +void computeRHS(Solver*); +int solve(Solver*); +void normalizePressure(Solver*); +void computeTimestep(Solver*); +void setBoundaryConditions(Solver*); +void setSpecialBoundaryCondition(Solver*); +void computeFG(Solver*); +void adaptUV(Solver*); +void writeResult(Solver* s, double* u, double* v, double* p); +#endif diff --git a/BasicSolver/2D-mpi/src/timing.c b/BasicSolver/2D-mpi/src/timing.c new file mode 100644 index 0000000..c4025a4 --- /dev/null +++ b/BasicSolver/2D-mpi/src/timing.c @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution() +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-mpi/src/timing.h b/BasicSolver/2D-mpi/src/timing.h new file mode 100644 index 0000000..db95329 --- /dev/null +++ b/BasicSolver/2D-mpi/src/timing.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(); +extern double getTimeResolution(); +extern double getTimeStamp_(); + +#endif // __TIMING_H_ diff --git a/BasicSolver/2D-mpi/src/util.h b/BasicSolver/2D-mpi/src/util.h new file mode 100644 index 0000000..657b009 --- /dev/null +++ b/BasicSolver/2D-mpi/src/util.h @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#endif // __UTIL_H_ diff --git a/BasicSolver/2D-mpi/surface.plot b/BasicSolver/2D-mpi/surface.plot new file mode 100644 index 0000000..4f7ccd9 --- /dev/null +++ b/BasicSolver/2D-mpi/surface.plot @@ -0,0 +1,7 @@ +set terminal png size 1024,768 enhanced font ,12 +set output 'p.png' +set datafile separator whitespace + +set grid +set hidden3d +splot 'pressure.dat' using 1:2:3 with lines diff --git a/BasicSolver/2D-mpi/vector.plot b/BasicSolver/2D-mpi/vector.plot new file mode 100644 index 0000000..0934ab2 --- /dev/null +++ b/BasicSolver/2D-mpi/vector.plot @@ -0,0 +1,5 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'velocity.png' +set datafile separator whitespace + +plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/BasicSolver/2D-mpi/velocity.dat b/BasicSolver/2D-mpi/velocity.dat new file mode 100644 index 0000000..ef4d40f --- /dev/null +++ b/BasicSolver/2D-mpi/velocity.dat @@ -0,0 +1,12794 @@ +0.01 0.01 0.000000 0.000000 0.000000 +0.01 0.01 0.000000 0.000000 0.000000 +0.03 0.01 0.000000 0.000000 0.000000 +0.04 0.01 0.000000 0.000000 0.000000 +0.04 0.01 0.000000 0.000000 0.000000 +0.06 0.01 0.000000 0.000000 0.000000 +0.07 0.01 0.000000 0.000000 0.000000 +0.07 0.01 0.000000 0.000000 0.000000 +0.09 0.01 0.000000 0.000000 0.000000 +0.10 0.01 0.000000 0.000000 0.000000 +0.10 0.01 0.000000 0.000000 0.000000 +0.12 0.01 0.000000 0.000000 0.000000 +0.12 0.01 0.000000 0.000000 0.000000 +0.14 0.01 0.000000 0.000000 0.000000 +0.14 0.01 0.000000 0.000000 0.000000 +0.15 0.01 0.000000 0.000000 0.000000 +0.17 0.01 0.000000 0.000000 0.000000 +0.18 0.01 0.000000 0.000000 0.000000 +0.18 0.01 0.000000 0.000000 0.000000 +0.20 0.01 0.000000 0.000000 0.000000 +0.21 0.01 0.000000 0.000000 0.000000 +0.21 0.01 0.000000 0.000000 0.000000 +0.23 0.01 0.000000 0.000000 0.000000 +0.24 0.01 0.000000 0.000000 0.000000 +0.24 0.01 0.000000 0.000000 0.000000 +0.26 0.01 0.000000 0.000000 0.000000 +0.27 0.01 0.000000 0.000000 0.000000 +0.28 0.01 0.000000 0.000000 0.000000 +0.29 0.01 0.000000 0.000000 0.000000 +0.29 0.01 0.000000 0.000000 0.000000 +0.30 0.01 0.000000 0.000000 0.000000 +0.32 0.01 0.000000 0.000000 0.000000 +0.33 0.01 0.000000 0.000000 0.000000 +0.34 0.01 0.000000 0.000000 0.000000 +0.35 0.01 0.000000 0.000000 0.000000 +0.35 0.01 0.000000 0.000000 0.000000 +0.36 0.01 0.000000 0.000000 0.000000 +0.38 0.01 0.000000 0.000000 0.000000 +0.39 0.01 0.000000 0.000000 0.000000 +0.40 0.01 0.000000 0.000000 0.000000 +0.41 0.01 0.000000 0.000000 0.000000 +0.42 0.01 0.000000 0.000000 0.000000 +0.42 0.01 0.000000 0.000000 0.000000 +0.43 0.01 0.000000 0.000000 0.000000 +0.45 0.01 0.000000 0.000000 0.000000 +0.46 0.01 0.000000 0.000000 0.000000 +0.47 0.01 0.000000 0.000000 0.000000 +0.48 0.01 0.000000 0.000000 0.000000 +0.48 0.01 0.000000 0.000000 0.000000 +0.49 0.01 0.000000 0.000000 0.000000 +0.51 0.01 0.000000 0.000000 0.000000 +0.52 0.01 0.000000 0.000000 0.000000 +0.53 0.01 0.000000 0.000000 0.000000 +0.54 0.01 0.000000 0.000000 0.000000 +0.55 0.01 0.000000 0.000000 0.000000 +0.56 0.01 0.000000 0.000000 0.000000 +0.57 0.01 0.000000 0.000000 0.000000 +0.58 0.01 0.000000 0.000000 0.000000 +0.58 0.01 0.000000 0.000000 0.000000 +0.59 0.01 0.000000 0.000000 0.000000 +0.60 0.01 0.000000 0.000000 0.000000 +0.61 0.01 0.000000 0.000000 0.000000 +0.62 0.01 0.000000 0.000000 0.000000 +0.64 0.01 0.000000 0.000000 0.000000 +0.65 0.01 0.000000 0.000000 0.000000 +0.66 0.01 0.000000 0.000000 0.000000 +0.67 0.01 0.000000 0.000000 0.000000 +0.68 0.01 0.000000 0.000000 0.000000 +0.69 0.01 0.000000 0.000000 0.000000 +0.70 0.01 0.000000 0.000000 0.000000 +0.70 0.01 0.000000 0.000000 0.000000 +0.71 0.01 0.000000 0.000000 0.000000 +0.72 0.01 0.000000 0.000000 0.000000 +0.73 0.01 0.000000 0.000000 0.000000 +0.74 0.01 0.000000 0.000000 0.000000 +0.76 0.01 0.000000 0.000000 0.000000 +0.77 0.01 0.000000 0.000000 0.000000 +0.78 0.01 0.000000 0.000000 0.000000 +0.79 0.01 0.000000 0.000000 0.000000 +0.80 0.01 0.000000 0.000000 0.000000 +0.81 0.01 0.000000 0.000000 0.000000 +0.82 0.01 0.000000 0.000000 0.000000 +0.83 0.01 0.000000 0.000000 0.000000 +0.83 0.01 0.000000 0.000000 0.000000 +0.84 0.01 0.000000 0.000000 0.000000 +0.85 0.01 0.000000 0.000000 0.000000 +0.86 0.01 0.000000 0.000000 0.000000 +0.88 0.01 0.000000 0.000000 0.000000 +0.89 0.01 0.000000 0.000000 0.000000 +0.90 0.01 0.000000 0.000000 0.000000 +0.91 0.01 0.000000 0.000000 0.000000 +0.92 0.01 0.000000 0.000000 0.000000 +0.93 0.01 0.000000 0.000000 0.000000 +0.94 0.01 0.000000 0.000000 0.000000 +0.95 0.01 0.000000 0.000000 0.000000 +0.96 0.01 0.000000 0.000000 0.000000 +0.96 0.01 0.000000 0.000000 0.000000 +0.97 0.01 0.000000 0.000000 0.000000 +0.98 0.01 0.000000 0.000000 0.000000 +0.01 0.01 0.000000 0.000000 0.000000 +0.01 0.01 0.000000 0.000000 0.000000 +0.03 0.01 0.000000 0.000000 0.000000 +0.04 0.01 0.000000 0.000000 0.000000 +0.04 0.01 0.000000 0.000000 0.000000 +0.06 0.01 0.000000 0.000000 0.000000 +0.07 0.01 0.000000 0.000000 0.000000 +0.07 0.01 0.000000 0.000000 0.000000 +0.09 0.01 0.000000 0.000000 0.000000 +0.10 0.01 0.000000 0.000000 0.000000 +0.10 0.01 0.000000 0.000000 0.000000 +0.12 0.01 0.000000 0.000000 0.000000 +0.12 0.01 0.000000 0.000000 0.000000 +0.14 0.01 0.000000 0.000000 0.000000 +0.14 0.01 0.000000 0.000000 0.000000 +0.15 0.01 0.000000 0.000000 0.000000 +0.17 0.01 0.000000 0.000000 0.000000 +0.18 0.01 0.000000 0.000000 0.000000 +0.18 0.01 0.000000 0.000000 0.000000 +0.20 0.01 0.000000 0.000000 0.000000 +0.21 0.01 0.000000 0.000000 0.000000 +0.21 0.01 0.000000 0.000000 0.000000 +0.23 0.01 0.000000 0.000000 0.000000 +0.24 0.01 0.000000 0.000000 0.000000 +0.24 0.01 0.000000 0.000000 0.000000 +0.26 0.01 0.000000 0.000000 0.000000 +0.27 0.01 0.000000 0.000000 0.000000 +0.28 0.01 0.000000 0.000000 0.000000 +0.29 0.01 0.000000 0.000000 0.000000 +0.29 0.01 0.000000 0.000000 0.000000 +0.30 0.01 0.000000 0.000000 0.000000 +0.32 0.01 0.000000 0.000000 0.000000 +0.33 0.01 0.000000 0.000000 0.000000 +0.34 0.01 0.000000 0.000000 0.000000 +0.35 0.01 0.000000 0.000000 0.000000 +0.35 0.01 0.000000 0.000000 0.000000 +0.36 0.01 0.000000 0.000000 0.000000 +0.38 0.01 0.000000 0.000000 0.000000 +0.39 0.01 0.000000 0.000000 0.000000 +0.40 0.01 0.000000 0.000000 0.000000 +0.41 0.01 0.000000 0.000000 0.000000 +0.42 0.01 0.000000 0.000000 0.000000 +0.42 0.01 0.000000 0.000000 0.000000 +0.43 0.01 0.000000 0.000000 0.000000 +0.45 0.01 0.000000 0.000000 0.000000 +0.46 0.01 0.000000 0.000000 0.000000 +0.47 0.01 0.000000 0.000000 0.000000 +0.48 0.01 0.000000 0.000000 0.000000 +0.48 0.01 0.000000 0.000000 0.000000 +0.49 0.01 0.000000 0.000000 0.000000 +0.51 0.01 0.000000 0.000000 0.000000 +0.52 0.01 0.000000 0.000000 0.000000 +0.53 0.01 0.000000 0.000000 0.000000 +0.54 0.01 0.000000 0.000000 0.000000 +0.55 0.01 0.000000 0.000000 0.000000 +0.56 0.01 0.000000 0.000000 0.000000 +0.57 0.01 0.000000 0.000000 0.000000 +0.58 0.01 0.000000 0.000000 0.000000 +0.58 0.01 0.000000 0.000000 0.000000 +0.59 0.01 0.000000 0.000000 0.000000 +0.60 0.01 0.000000 0.000000 0.000000 +0.61 0.01 0.000000 0.000000 0.000000 +0.62 0.01 0.000000 0.000000 0.000000 +0.64 0.01 0.000000 0.000000 0.000000 +0.65 0.01 0.000000 0.000000 0.000000 +0.66 0.01 0.000000 0.000000 0.000000 +0.67 0.01 0.000000 0.000000 0.000000 +0.68 0.01 0.000000 0.000000 0.000000 +0.69 0.01 0.000000 0.000000 0.000000 +0.70 0.01 0.000000 0.000000 0.000000 +0.70 0.01 0.000000 0.000000 0.000000 +0.71 0.01 0.000000 0.000000 0.000000 +0.72 0.01 0.000000 0.000000 0.000000 +0.73 0.01 0.000000 0.000000 0.000000 +0.74 0.01 0.000000 0.000000 0.000000 +0.76 0.01 0.000000 0.000000 0.000000 +0.77 0.01 0.000000 0.000000 0.000000 +0.78 0.01 0.000000 0.000000 0.000000 +0.79 0.01 0.000000 0.000000 0.000000 +0.80 0.01 0.000000 0.000000 0.000000 +0.81 0.01 0.000000 0.000000 0.000000 +0.82 0.01 0.000000 0.000000 0.000000 +0.83 0.01 0.000000 0.000000 0.000000 +0.83 0.01 0.000000 0.000000 0.000000 +0.84 0.01 0.000000 0.000000 0.000000 +0.85 0.01 0.000000 0.000000 0.000000 +0.86 0.01 0.000000 0.000000 0.000000 +0.88 0.01 0.000000 0.000000 0.000000 +0.89 0.01 0.000000 0.000000 0.000000 +0.90 0.01 0.000000 0.000000 0.000000 +0.91 0.01 0.000000 0.000000 0.000000 +0.92 0.01 0.000000 0.000000 0.000000 +0.93 0.01 0.000000 0.000000 0.000000 +0.94 0.01 0.000000 0.000000 0.000000 +0.95 0.01 0.000000 0.000000 0.000000 +0.96 0.01 0.000000 0.000000 0.000000 +0.96 0.01 0.000000 0.000000 0.000000 +0.97 0.01 0.000000 0.000000 0.000000 +0.98 0.01 0.000000 0.000000 0.000000 +0.01 0.03 0.000000 0.000000 0.000000 +0.01 0.03 0.000000 0.000000 0.000000 +0.03 0.03 0.000000 0.000000 0.000000 +0.04 0.03 0.000000 0.000000 0.000000 +0.04 0.03 0.000000 0.000000 0.000000 +0.06 0.03 0.000000 0.000000 0.000000 +0.07 0.03 0.000000 0.000000 0.000000 +0.07 0.03 0.000000 0.000000 0.000000 +0.09 0.03 0.000000 0.000000 0.000000 +0.10 0.03 0.000000 0.000000 0.000000 +0.10 0.03 0.000000 0.000000 0.000000 +0.12 0.03 0.000000 0.000000 0.000000 +0.12 0.03 0.000000 0.000000 0.000000 +0.14 0.03 0.000000 0.000000 0.000000 +0.14 0.03 0.000000 0.000000 0.000000 +0.15 0.03 0.000000 0.000000 0.000000 +0.17 0.03 0.000000 0.000000 0.000000 +0.18 0.03 0.000000 0.000000 0.000000 +0.18 0.03 0.000000 0.000000 0.000000 +0.20 0.03 0.000000 0.000000 0.000000 +0.21 0.03 0.000000 0.000000 0.000000 +0.21 0.03 0.000000 0.000000 0.000000 +0.23 0.03 0.000000 0.000000 0.000000 +0.24 0.03 0.000000 0.000000 0.000000 +0.24 0.03 0.000000 0.000000 0.000000 +0.26 0.03 0.000000 0.000000 0.000000 +0.27 0.03 0.000000 0.000000 0.000000 +0.28 0.03 0.000000 0.000000 0.000000 +0.29 0.03 0.000000 0.000000 0.000000 +0.29 0.03 0.000000 0.000000 0.000000 +0.30 0.03 0.000000 0.000000 0.000000 +0.32 0.03 0.000000 0.000000 0.000000 +0.33 0.03 0.000000 0.000000 0.000000 +0.34 0.03 0.000000 0.000000 0.000000 +0.35 0.03 0.000000 0.000000 0.000000 +0.35 0.03 0.000000 0.000000 0.000000 +0.36 0.03 0.000000 0.000000 0.000000 +0.38 0.03 0.000000 0.000000 0.000000 +0.39 0.03 0.000000 0.000000 0.000000 +0.40 0.03 0.000000 0.000000 0.000000 +0.41 0.03 0.000000 0.000000 0.000000 +0.42 0.03 0.000000 0.000000 0.000000 +0.42 0.03 0.000000 0.000000 0.000000 +0.43 0.03 0.000000 0.000000 0.000000 +0.45 0.03 0.000000 0.000000 0.000000 +0.46 0.03 0.000000 0.000000 0.000000 +0.47 0.03 0.000000 0.000000 0.000000 +0.48 0.03 0.000000 0.000000 0.000000 +0.48 0.03 0.000000 0.000000 0.000000 +0.49 0.03 0.000000 0.000000 0.000000 +0.51 0.03 0.000000 0.000000 0.000000 +0.52 0.03 0.000000 0.000000 0.000000 +0.53 0.03 0.000000 0.000000 0.000000 +0.54 0.03 0.000000 0.000000 0.000000 +0.55 0.03 0.000000 0.000000 0.000000 +0.56 0.03 0.000000 0.000000 0.000000 +0.57 0.03 0.000000 0.000000 0.000000 +0.58 0.03 0.000000 0.000000 0.000000 +0.58 0.03 0.000000 0.000000 0.000000 +0.59 0.03 0.000000 0.000000 0.000000 +0.60 0.03 0.000000 0.000000 0.000000 +0.61 0.03 0.000000 0.000000 0.000000 +0.62 0.03 0.000000 0.000000 0.000000 +0.64 0.03 0.000000 0.000000 0.000000 +0.65 0.03 0.000000 0.000000 0.000000 +0.66 0.03 0.000000 0.000000 0.000000 +0.67 0.03 0.000000 0.000000 0.000000 +0.68 0.03 0.000000 0.000000 0.000000 +0.69 0.03 0.000000 0.000000 0.000000 +0.70 0.03 0.000000 0.000000 0.000000 +0.70 0.03 0.000000 0.000000 0.000000 +0.71 0.03 0.000000 0.000000 0.000000 +0.72 0.03 0.000000 0.000000 0.000000 +0.73 0.03 0.000000 0.000000 0.000000 +0.74 0.03 0.000000 0.000000 0.000000 +0.76 0.03 0.000000 0.000000 0.000000 +0.77 0.03 0.000000 0.000000 0.000000 +0.78 0.03 0.000000 0.000000 0.000000 +0.79 0.03 0.000000 0.000000 0.000000 +0.80 0.03 0.000000 0.000000 0.000000 +0.81 0.03 0.000000 0.000000 0.000000 +0.82 0.03 0.000000 0.000000 0.000000 +0.83 0.03 0.000000 0.000000 0.000000 +0.83 0.03 0.000000 0.000000 0.000000 +0.84 0.03 0.000000 0.000000 0.000000 +0.85 0.03 0.000000 0.000000 0.000000 +0.86 0.03 0.000000 0.000000 0.000000 +0.88 0.03 0.000000 0.000000 0.000000 +0.89 0.03 0.000000 0.000000 0.000000 +0.90 0.03 0.000000 0.000000 0.000000 +0.91 0.03 0.000000 0.000000 0.000000 +0.92 0.03 0.000000 0.000000 0.000000 +0.93 0.03 0.000000 0.000000 0.000000 +0.94 0.03 0.000000 0.000000 0.000000 +0.95 0.03 0.000000 0.000000 0.000000 +0.96 0.03 0.000000 0.000000 0.000000 +0.96 0.03 0.000000 0.000000 0.000000 +0.97 0.03 0.000000 0.000000 0.000000 +0.98 0.03 0.000000 0.000000 0.000000 +0.01 0.04 0.000000 0.000000 0.000000 +0.01 0.04 0.000000 0.000000 0.000000 +0.03 0.04 0.000000 0.000000 0.000000 +0.04 0.04 0.000000 0.000000 0.000000 +0.04 0.04 0.000000 0.000000 0.000000 +0.06 0.04 0.000000 0.000000 0.000000 +0.07 0.04 0.000000 0.000000 0.000000 +0.07 0.04 0.000000 0.000000 0.000000 +0.09 0.04 0.000000 0.000000 0.000000 +0.10 0.04 0.000000 0.000000 0.000000 +0.10 0.04 0.000000 0.000000 0.000000 +0.12 0.04 0.000000 0.000000 0.000000 +0.12 0.04 0.000000 0.000000 0.000000 +0.14 0.04 0.000000 0.000000 0.000000 +0.14 0.04 0.000000 0.000000 0.000000 +0.15 0.04 0.000000 0.000000 0.000000 +0.17 0.04 0.000000 0.000000 0.000000 +0.18 0.04 0.000000 0.000000 0.000000 +0.18 0.04 0.000000 0.000000 0.000000 +0.20 0.04 0.000000 0.000000 0.000000 +0.21 0.04 0.000000 0.000000 0.000000 +0.21 0.04 0.000000 0.000000 0.000000 +0.23 0.04 0.000000 0.000000 0.000000 +0.24 0.04 0.000000 0.000000 0.000000 +0.24 0.04 0.000000 0.000000 0.000000 +0.26 0.04 0.000000 0.000000 0.000000 +0.27 0.04 0.000000 0.000000 0.000000 +0.28 0.04 0.000000 0.000000 0.000000 +0.29 0.04 0.000000 0.000000 0.000000 +0.29 0.04 0.000000 0.000000 0.000000 +0.30 0.04 0.000000 0.000000 0.000000 +0.32 0.04 0.000000 0.000000 0.000000 +0.33 0.04 0.000000 0.000000 0.000000 +0.34 0.04 0.000000 0.000000 0.000000 +0.35 0.04 0.000000 0.000000 0.000000 +0.35 0.04 0.000000 0.000000 0.000000 +0.36 0.04 0.000000 0.000000 0.000000 +0.38 0.04 0.000000 0.000000 0.000000 +0.39 0.04 0.000000 0.000000 0.000000 +0.40 0.04 0.000000 0.000000 0.000000 +0.41 0.04 0.000000 0.000000 0.000000 +0.42 0.04 0.000000 0.000000 0.000000 +0.42 0.04 0.000000 0.000000 0.000000 +0.43 0.04 0.000000 0.000000 0.000000 +0.45 0.04 0.000000 0.000000 0.000000 +0.46 0.04 0.000000 0.000000 0.000000 +0.47 0.04 0.000000 0.000000 0.000000 +0.48 0.04 0.000000 0.000000 0.000000 +0.48 0.04 0.000000 0.000000 0.000000 +0.49 0.04 0.000000 0.000000 0.000000 +0.51 0.04 0.000000 0.000000 0.000000 +0.52 0.04 0.000000 0.000000 0.000000 +0.53 0.04 0.000000 0.000000 0.000000 +0.54 0.04 0.000000 0.000000 0.000000 +0.55 0.04 0.000000 0.000000 0.000000 +0.56 0.04 0.000000 0.000000 0.000000 +0.57 0.04 0.000000 0.000000 0.000000 +0.58 0.04 0.000000 0.000000 0.000000 +0.58 0.04 0.000000 0.000000 0.000000 +0.59 0.04 0.000000 0.000000 0.000000 +0.60 0.04 0.000000 0.000000 0.000000 +0.61 0.04 0.000000 0.000000 0.000000 +0.62 0.04 0.000000 0.000000 0.000000 +0.64 0.04 0.000000 0.000000 0.000000 +0.65 0.04 0.000000 0.000000 0.000000 +0.66 0.04 0.000000 0.000000 0.000000 +0.67 0.04 0.000000 0.000000 0.000000 +0.68 0.04 0.000000 0.000000 0.000000 +0.69 0.04 0.000000 0.000000 0.000000 +0.70 0.04 0.000000 0.000000 0.000000 +0.70 0.04 0.000000 0.000000 0.000000 +0.71 0.04 0.000000 0.000000 0.000000 +0.72 0.04 0.000000 0.000000 0.000000 +0.73 0.04 0.000000 0.000000 0.000000 +0.74 0.04 0.000000 0.000000 0.000000 +0.76 0.04 0.000000 0.000000 0.000000 +0.77 0.04 0.000000 0.000000 0.000000 +0.78 0.04 0.000000 0.000000 0.000000 +0.79 0.04 0.000000 0.000000 0.000000 +0.80 0.04 0.000000 0.000000 0.000000 +0.81 0.04 0.000000 0.000000 0.000000 +0.82 0.04 0.000000 0.000000 0.000000 +0.83 0.04 0.000000 0.000000 0.000000 +0.83 0.04 0.000000 0.000000 0.000000 +0.84 0.04 0.000000 0.000000 0.000000 +0.85 0.04 0.000000 0.000000 0.000000 +0.86 0.04 0.000000 0.000000 0.000000 +0.88 0.04 0.000000 0.000000 0.000000 +0.89 0.04 0.000000 0.000000 0.000000 +0.90 0.04 0.000000 0.000000 0.000000 +0.91 0.04 0.000000 0.000000 0.000000 +0.92 0.04 0.000000 0.000000 0.000000 +0.93 0.04 0.000000 0.000000 0.000000 +0.94 0.04 0.000000 0.000000 0.000000 +0.95 0.04 0.000000 0.000000 0.000000 +0.96 0.04 0.000000 0.000000 0.000000 +0.96 0.04 0.000000 0.000000 0.000000 +0.97 0.04 0.000000 0.000000 0.000000 +0.98 0.04 0.000000 0.000000 0.000000 +0.01 0.04 0.000000 0.000000 0.000000 +0.01 0.04 0.000000 0.000000 0.000000 +0.03 0.04 0.000000 0.000000 0.000000 +0.04 0.04 0.000000 0.000000 0.000000 +0.04 0.04 0.000000 0.000000 0.000000 +0.06 0.04 0.000000 0.000000 0.000000 +0.07 0.04 0.000000 0.000000 0.000000 +0.07 0.04 0.000000 0.000000 0.000000 +0.09 0.04 0.000000 0.000000 0.000000 +0.10 0.04 0.000000 0.000000 0.000000 +0.10 0.04 0.000000 0.000000 0.000000 +0.12 0.04 0.000000 0.000000 0.000000 +0.12 0.04 0.000000 0.000000 0.000000 +0.14 0.04 0.000000 0.000000 0.000000 +0.14 0.04 0.000000 0.000000 0.000000 +0.15 0.04 0.000000 0.000000 0.000000 +0.17 0.04 0.000000 0.000000 0.000000 +0.18 0.04 0.000000 0.000000 0.000000 +0.18 0.04 0.000000 0.000000 0.000000 +0.20 0.04 0.000000 0.000000 0.000000 +0.21 0.04 0.000000 0.000000 0.000000 +0.21 0.04 0.000000 0.000000 0.000000 +0.23 0.04 0.000000 0.000000 0.000000 +0.24 0.04 0.000000 0.000000 0.000000 +0.24 0.04 0.000000 0.000000 0.000000 +0.26 0.04 0.000000 0.000000 0.000000 +0.27 0.04 0.000000 0.000000 0.000000 +0.28 0.04 0.000000 0.000000 0.000000 +0.29 0.04 0.000000 0.000000 0.000000 +0.29 0.04 0.000000 0.000000 0.000000 +0.30 0.04 0.000000 0.000000 0.000000 +0.32 0.04 0.000000 0.000000 0.000000 +0.33 0.04 0.000000 0.000000 0.000000 +0.34 0.04 0.000000 0.000000 0.000000 +0.35 0.04 0.000000 0.000000 0.000000 +0.35 0.04 0.000000 0.000000 0.000000 +0.36 0.04 0.000000 0.000000 0.000000 +0.38 0.04 0.000000 0.000000 0.000000 +0.39 0.04 0.000000 0.000000 0.000000 +0.40 0.04 0.000000 0.000000 0.000000 +0.41 0.04 0.000000 0.000000 0.000000 +0.42 0.04 0.000000 0.000000 0.000000 +0.42 0.04 0.000000 0.000000 0.000000 +0.43 0.04 0.000000 0.000000 0.000000 +0.45 0.04 0.000000 0.000000 0.000000 +0.46 0.04 0.000000 0.000000 0.000000 +0.47 0.04 0.000000 0.000000 0.000000 +0.48 0.04 0.000000 0.000000 0.000000 +0.48 0.04 0.000000 0.000000 0.000000 +0.49 0.04 0.000000 0.000000 0.000000 +0.51 0.04 0.000000 0.000000 0.000000 +0.52 0.04 0.000000 0.000000 0.000000 +0.53 0.04 0.000000 0.000000 0.000000 +0.54 0.04 0.000000 0.000000 0.000000 +0.55 0.04 0.000000 0.000000 0.000000 +0.56 0.04 0.000000 0.000000 0.000000 +0.57 0.04 0.000000 0.000000 0.000000 +0.58 0.04 0.000000 0.000000 0.000000 +0.58 0.04 0.000000 0.000000 0.000000 +0.59 0.04 0.000000 0.000000 0.000000 +0.60 0.04 0.000000 0.000000 0.000000 +0.61 0.04 0.000000 0.000000 0.000000 +0.62 0.04 0.000000 0.000000 0.000000 +0.64 0.04 0.000000 0.000000 0.000000 +0.65 0.04 0.000000 0.000000 0.000000 +0.66 0.04 0.000000 0.000000 0.000000 +0.67 0.04 0.000000 0.000000 0.000000 +0.68 0.04 0.000000 0.000000 0.000000 +0.69 0.04 0.000000 0.000000 0.000000 +0.70 0.04 0.000000 0.000000 0.000000 +0.70 0.04 0.000000 0.000000 0.000000 +0.71 0.04 0.000000 0.000000 0.000000 +0.72 0.04 0.000000 0.000000 0.000000 +0.73 0.04 0.000000 0.000000 0.000000 +0.74 0.04 0.000000 0.000000 0.000000 +0.76 0.04 0.000000 0.000000 0.000000 +0.77 0.04 0.000000 0.000000 0.000000 +0.78 0.04 0.000000 0.000000 0.000000 +0.79 0.04 0.000000 0.000000 0.000000 +0.80 0.04 0.000000 0.000000 0.000000 +0.81 0.04 0.000000 0.000000 0.000000 +0.82 0.04 0.000000 0.000000 0.000000 +0.83 0.04 0.000000 0.000000 0.000000 +0.83 0.04 0.000000 0.000000 0.000000 +0.84 0.04 0.000000 0.000000 0.000000 +0.85 0.04 0.000000 0.000000 0.000000 +0.86 0.04 0.000000 0.000000 0.000000 +0.88 0.04 0.000000 0.000000 0.000000 +0.89 0.04 0.000000 0.000000 0.000000 +0.90 0.04 0.000000 0.000000 0.000000 +0.91 0.04 0.000000 0.000000 0.000000 +0.92 0.04 0.000000 0.000000 0.000000 +0.93 0.04 0.000000 0.000000 0.000000 +0.94 0.04 0.000000 0.000000 0.000000 +0.95 0.04 0.000000 0.000000 0.000000 +0.96 0.04 0.000000 0.000000 0.000000 +0.96 0.04 0.000000 0.000000 0.000000 +0.97 0.04 0.000000 0.000000 0.000000 +0.98 0.04 0.000000 0.000000 0.000000 +0.01 0.06 0.000000 0.000000 0.000000 +0.01 0.06 0.000000 0.000000 0.000000 +0.03 0.06 0.000000 0.000000 0.000000 +0.04 0.06 0.000000 0.000000 0.000000 +0.04 0.06 0.000000 0.000000 0.000000 +0.06 0.06 0.000000 0.000000 0.000000 +0.07 0.06 0.000000 0.000000 0.000000 +0.07 0.06 0.000000 0.000000 0.000000 +0.09 0.06 0.000000 0.000000 0.000000 +0.10 0.06 0.000000 0.000000 0.000000 +0.10 0.06 0.000000 0.000000 0.000000 +0.12 0.06 0.000000 0.000000 0.000000 +0.12 0.06 0.000000 0.000000 0.000000 +0.14 0.06 0.000000 0.000000 0.000000 +0.14 0.06 0.000000 0.000000 0.000000 +0.15 0.06 0.000000 0.000000 0.000000 +0.17 0.06 0.000000 0.000000 0.000000 +0.18 0.06 0.000000 0.000000 0.000000 +0.18 0.06 0.000000 0.000000 0.000000 +0.20 0.06 0.000000 0.000000 0.000000 +0.21 0.06 0.000000 0.000000 0.000000 +0.21 0.06 0.000000 0.000000 0.000000 +0.23 0.06 0.000000 0.000000 0.000000 +0.24 0.06 0.000000 0.000000 0.000000 +0.24 0.06 0.000000 0.000000 0.000000 +0.26 0.06 0.000000 0.000000 0.000000 +0.27 0.06 0.000000 0.000000 0.000000 +0.28 0.06 0.000000 0.000000 0.000000 +0.29 0.06 0.000000 0.000000 0.000000 +0.29 0.06 0.000000 0.000000 0.000000 +0.30 0.06 0.000000 0.000000 0.000000 +0.32 0.06 0.000000 0.000000 0.000000 +0.33 0.06 0.000000 0.000000 0.000000 +0.34 0.06 0.000000 0.000000 0.000000 +0.35 0.06 0.000000 0.000000 0.000000 +0.35 0.06 0.000000 0.000000 0.000000 +0.36 0.06 0.000000 0.000000 0.000000 +0.38 0.06 0.000000 0.000000 0.000000 +0.39 0.06 0.000000 0.000000 0.000000 +0.40 0.06 0.000000 0.000000 0.000000 +0.41 0.06 0.000000 0.000000 0.000000 +0.42 0.06 0.000000 0.000000 0.000000 +0.42 0.06 0.000000 0.000000 0.000000 +0.43 0.06 0.000000 0.000000 0.000000 +0.45 0.06 0.000000 0.000000 0.000000 +0.46 0.06 0.000000 0.000000 0.000000 +0.47 0.06 0.000000 0.000000 0.000000 +0.48 0.06 0.000000 0.000000 0.000000 +0.48 0.06 0.000000 0.000000 0.000000 +0.49 0.06 0.000000 0.000000 0.000000 +0.51 0.06 0.000000 0.000000 0.000000 +0.52 0.06 0.000000 0.000000 0.000000 +0.53 0.06 0.000000 0.000000 0.000000 +0.54 0.06 0.000000 0.000000 0.000000 +0.55 0.06 0.000000 0.000000 0.000000 +0.56 0.06 0.000000 0.000000 0.000000 +0.57 0.06 0.000000 0.000000 0.000000 +0.58 0.06 0.000000 0.000000 0.000000 +0.58 0.06 0.000000 0.000000 0.000000 +0.59 0.06 0.000000 0.000000 0.000000 +0.60 0.06 0.000000 0.000000 0.000000 +0.61 0.06 0.000000 0.000000 0.000000 +0.62 0.06 0.000000 0.000000 0.000000 +0.64 0.06 0.000000 0.000000 0.000000 +0.65 0.06 0.000000 0.000000 0.000000 +0.66 0.06 0.000000 0.000000 0.000000 +0.67 0.06 0.000000 0.000000 0.000000 +0.68 0.06 0.000000 0.000000 0.000000 +0.69 0.06 0.000000 0.000000 0.000000 +0.70 0.06 0.000000 0.000000 0.000000 +0.70 0.06 0.000000 0.000000 0.000000 +0.71 0.06 0.000000 0.000000 0.000000 +0.72 0.06 0.000000 0.000000 0.000000 +0.73 0.06 0.000000 0.000000 0.000000 +0.74 0.06 0.000000 0.000000 0.000000 +0.76 0.06 0.000000 0.000000 0.000000 +0.77 0.06 0.000000 0.000000 0.000000 +0.78 0.06 0.000000 0.000000 0.000000 +0.79 0.06 0.000000 0.000000 0.000000 +0.80 0.06 0.000000 0.000000 0.000000 +0.81 0.06 0.000000 0.000000 0.000000 +0.82 0.06 0.000000 0.000000 0.000000 +0.83 0.06 0.000000 0.000000 0.000000 +0.83 0.06 0.000000 0.000000 0.000000 +0.84 0.06 0.000000 0.000000 0.000000 +0.85 0.06 0.000000 0.000000 0.000000 +0.86 0.06 0.000000 0.000000 0.000000 +0.88 0.06 0.000000 0.000000 0.000000 +0.89 0.06 0.000000 0.000000 0.000000 +0.90 0.06 0.000000 0.000000 0.000000 +0.91 0.06 0.000000 0.000000 0.000000 +0.92 0.06 0.000000 0.000000 0.000000 +0.93 0.06 0.000000 0.000000 0.000000 +0.94 0.06 0.000000 0.000000 0.000000 +0.95 0.06 0.000000 0.000000 0.000000 +0.96 0.06 0.000000 0.000000 0.000000 +0.96 0.06 0.000000 0.000000 0.000000 +0.97 0.06 0.000000 0.000000 0.000000 +0.98 0.06 0.000000 0.000000 0.000000 +0.01 0.07 0.000000 0.000000 0.000000 +0.01 0.07 0.000000 0.000000 0.000000 +0.03 0.07 0.000000 0.000000 0.000000 +0.04 0.07 0.000000 0.000000 0.000000 +0.04 0.07 0.000000 0.000000 0.000000 +0.06 0.07 0.000000 0.000000 0.000000 +0.07 0.07 0.000000 0.000000 0.000000 +0.07 0.07 0.000000 0.000000 0.000000 +0.09 0.07 0.000000 0.000000 0.000000 +0.10 0.07 0.000000 0.000000 0.000000 +0.10 0.07 0.000000 0.000000 0.000000 +0.12 0.07 0.000000 0.000000 0.000000 +0.12 0.07 0.000000 0.000000 0.000000 +0.14 0.07 0.000000 0.000000 0.000000 +0.14 0.07 0.000000 0.000000 0.000000 +0.15 0.07 0.000000 0.000000 0.000000 +0.17 0.07 0.000000 0.000000 0.000000 +0.18 0.07 0.000000 0.000000 0.000000 +0.18 0.07 0.000000 0.000000 0.000000 +0.20 0.07 0.000000 0.000000 0.000000 +0.21 0.07 0.000000 0.000000 0.000000 +0.21 0.07 0.000000 0.000000 0.000000 +0.23 0.07 0.000000 0.000000 0.000000 +0.24 0.07 0.000000 0.000000 0.000000 +0.24 0.07 0.000000 0.000000 0.000000 +0.26 0.07 0.000000 0.000000 0.000000 +0.27 0.07 0.000000 0.000000 0.000000 +0.28 0.07 0.000000 0.000000 0.000000 +0.29 0.07 0.000000 0.000000 0.000000 +0.29 0.07 0.000000 0.000000 0.000000 +0.30 0.07 0.000000 0.000000 0.000000 +0.32 0.07 0.000000 0.000000 0.000000 +0.33 0.07 0.000000 0.000000 0.000000 +0.34 0.07 0.000000 0.000000 0.000000 +0.35 0.07 0.000000 0.000000 0.000000 +0.35 0.07 0.000000 0.000000 0.000000 +0.36 0.07 0.000000 0.000000 0.000000 +0.38 0.07 0.000000 0.000000 0.000000 +0.39 0.07 0.000000 0.000000 0.000000 +0.40 0.07 0.000000 0.000000 0.000000 +0.41 0.07 0.000000 0.000000 0.000000 +0.42 0.07 0.000000 0.000000 0.000000 +0.42 0.07 0.000000 0.000000 0.000000 +0.43 0.07 0.000000 0.000000 0.000000 +0.45 0.07 0.000000 0.000000 0.000000 +0.46 0.07 0.000000 0.000000 0.000000 +0.47 0.07 0.000000 0.000000 0.000000 +0.48 0.07 0.000000 0.000000 0.000000 +0.48 0.07 0.000000 0.000000 0.000000 +0.49 0.07 0.000000 0.000000 0.000000 +0.51 0.07 0.000000 0.000000 0.000000 +0.52 0.07 0.000000 0.000000 0.000000 +0.53 0.07 0.000000 0.000000 0.000000 +0.54 0.07 0.000000 0.000000 0.000000 +0.55 0.07 0.000000 0.000000 0.000000 +0.56 0.07 0.000000 0.000000 0.000000 +0.57 0.07 0.000000 0.000000 0.000000 +0.58 0.07 0.000000 0.000000 0.000000 +0.58 0.07 0.000000 0.000000 0.000000 +0.59 0.07 0.000000 0.000000 0.000000 +0.60 0.07 0.000000 0.000000 0.000000 +0.61 0.07 0.000000 0.000000 0.000000 +0.62 0.07 0.000000 0.000000 0.000000 +0.64 0.07 0.000000 0.000000 0.000000 +0.65 0.07 0.000000 0.000000 0.000000 +0.66 0.07 0.000000 0.000000 0.000000 +0.67 0.07 0.000000 0.000000 0.000000 +0.68 0.07 0.000000 0.000000 0.000000 +0.69 0.07 0.000000 0.000000 0.000000 +0.70 0.07 0.000000 0.000000 0.000000 +0.70 0.07 0.000000 0.000000 0.000000 +0.71 0.07 0.000000 0.000000 0.000000 +0.72 0.07 0.000000 0.000000 0.000000 +0.73 0.07 0.000000 0.000000 0.000000 +0.74 0.07 0.000000 0.000000 0.000000 +0.76 0.07 0.000000 0.000000 0.000000 +0.77 0.07 0.000000 0.000000 0.000000 +0.78 0.07 0.000000 0.000000 0.000000 +0.79 0.07 0.000000 0.000000 0.000000 +0.80 0.07 0.000000 0.000000 0.000000 +0.81 0.07 0.000000 0.000000 0.000000 +0.82 0.07 0.000000 0.000000 0.000000 +0.83 0.07 0.000000 0.000000 0.000000 +0.83 0.07 0.000000 0.000000 0.000000 +0.84 0.07 0.000000 0.000000 0.000000 +0.85 0.07 0.000000 0.000000 0.000000 +0.86 0.07 0.000000 0.000000 0.000000 +0.88 0.07 0.000000 0.000000 0.000000 +0.89 0.07 0.000000 0.000000 0.000000 +0.90 0.07 0.000000 0.000000 0.000000 +0.91 0.07 0.000000 0.000000 0.000000 +0.92 0.07 0.000000 0.000000 0.000000 +0.93 0.07 0.000000 0.000000 0.000000 +0.94 0.07 0.000000 0.000000 0.000000 +0.95 0.07 0.000000 0.000000 0.000000 +0.96 0.07 0.000000 0.000000 0.000000 +0.96 0.07 0.000000 0.000000 0.000000 +0.97 0.07 0.000000 0.000000 0.000000 +0.98 0.07 0.000000 0.000000 0.000000 +0.01 0.07 0.000000 0.000000 0.000000 +0.01 0.07 0.000000 0.000000 0.000000 +0.03 0.07 0.000000 0.000000 0.000000 +0.04 0.07 0.000000 0.000000 0.000000 +0.04 0.07 0.000000 0.000000 0.000000 +0.06 0.07 0.000000 0.000000 0.000000 +0.07 0.07 0.000000 0.000000 0.000000 +0.07 0.07 0.000000 0.000000 0.000000 +0.09 0.07 0.000000 0.000000 0.000000 +0.10 0.07 0.000000 0.000000 0.000000 +0.10 0.07 0.000000 0.000000 0.000000 +0.12 0.07 0.000000 0.000000 0.000000 +0.12 0.07 0.000000 0.000000 0.000000 +0.14 0.07 0.000000 0.000000 0.000000 +0.14 0.07 0.000000 0.000000 0.000000 +0.15 0.07 0.000000 0.000000 0.000000 +0.17 0.07 0.000000 0.000000 0.000000 +0.18 0.07 0.000000 0.000000 0.000000 +0.18 0.07 0.000000 0.000000 0.000000 +0.20 0.07 0.000000 0.000000 0.000000 +0.21 0.07 0.000000 0.000000 0.000000 +0.21 0.07 0.000000 0.000000 0.000000 +0.23 0.07 0.000000 0.000000 0.000000 +0.24 0.07 0.000000 0.000000 0.000000 +0.24 0.07 0.000000 0.000000 0.000000 +0.26 0.07 0.000000 0.000000 0.000000 +0.27 0.07 0.000000 0.000000 0.000000 +0.28 0.07 0.000000 0.000000 0.000000 +0.29 0.07 0.000000 0.000000 0.000000 +0.29 0.07 0.000000 0.000000 0.000000 +0.30 0.07 0.000000 0.000000 0.000000 +0.32 0.07 0.000000 0.000000 0.000000 +0.33 0.07 0.000000 0.000000 0.000000 +0.34 0.07 0.000000 0.000000 0.000000 +0.35 0.07 0.000000 0.000000 0.000000 +0.35 0.07 0.000000 0.000000 0.000000 +0.36 0.07 0.000000 0.000000 0.000000 +0.38 0.07 0.000000 0.000000 0.000000 +0.39 0.07 0.000000 0.000000 0.000000 +0.40 0.07 0.000000 0.000000 0.000000 +0.41 0.07 0.000000 0.000000 0.000000 +0.42 0.07 0.000000 0.000000 0.000000 +0.42 0.07 0.000000 0.000000 0.000000 +0.43 0.07 0.000000 0.000000 0.000000 +0.45 0.07 0.000000 0.000000 0.000000 +0.46 0.07 0.000000 0.000000 0.000000 +0.47 0.07 0.000000 0.000000 0.000000 +0.48 0.07 0.000000 0.000000 0.000000 +0.48 0.07 0.000000 0.000000 0.000000 +0.49 0.07 0.000000 0.000000 0.000000 +0.51 0.07 0.000000 0.000000 0.000000 +0.52 0.07 0.000000 0.000000 0.000000 +0.53 0.07 0.000000 0.000000 0.000000 +0.54 0.07 0.000000 0.000000 0.000000 +0.55 0.07 0.000000 0.000000 0.000000 +0.56 0.07 0.000000 0.000000 0.000000 +0.57 0.07 0.000000 0.000000 0.000000 +0.58 0.07 0.000000 0.000000 0.000000 +0.58 0.07 0.000000 0.000000 0.000000 +0.59 0.07 0.000000 0.000000 0.000000 +0.60 0.07 0.000000 0.000000 0.000000 +0.61 0.07 0.000000 0.000000 0.000000 +0.62 0.07 0.000000 0.000000 0.000000 +0.64 0.07 0.000000 0.000000 0.000000 +0.65 0.07 0.000000 0.000000 0.000000 +0.66 0.07 0.000000 0.000000 0.000000 +0.67 0.07 0.000000 0.000000 0.000000 +0.68 0.07 0.000000 0.000000 0.000000 +0.69 0.07 0.000000 0.000000 0.000000 +0.70 0.07 0.000000 0.000000 0.000000 +0.70 0.07 0.000000 0.000000 0.000000 +0.71 0.07 0.000000 0.000000 0.000000 +0.72 0.07 0.000000 0.000000 0.000000 +0.73 0.07 0.000000 0.000000 0.000000 +0.74 0.07 0.000000 0.000000 0.000000 +0.76 0.07 0.000000 0.000000 0.000000 +0.77 0.07 0.000000 0.000000 0.000000 +0.78 0.07 0.000000 0.000000 0.000000 +0.79 0.07 0.000000 0.000000 0.000000 +0.80 0.07 0.000000 0.000000 0.000000 +0.81 0.07 0.000000 0.000000 0.000000 +0.82 0.07 0.000000 0.000000 0.000000 +0.83 0.07 0.000000 0.000000 0.000000 +0.83 0.07 0.000000 0.000000 0.000000 +0.84 0.07 0.000000 0.000000 0.000000 +0.85 0.07 0.000000 0.000000 0.000000 +0.86 0.07 0.000000 0.000000 0.000000 +0.88 0.07 0.000000 0.000000 0.000000 +0.89 0.07 0.000000 0.000000 0.000000 +0.90 0.07 0.000000 0.000000 0.000000 +0.91 0.07 0.000000 0.000000 0.000000 +0.92 0.07 0.000000 0.000000 0.000000 +0.93 0.07 0.000000 0.000000 0.000000 +0.94 0.07 0.000000 0.000000 0.000000 +0.95 0.07 0.000000 0.000000 0.000000 +0.96 0.07 0.000000 0.000000 0.000000 +0.96 0.07 0.000000 0.000000 0.000000 +0.97 0.07 0.000000 0.000000 0.000000 +0.98 0.07 0.000000 0.000000 0.000000 +0.01 0.09 0.000000 0.000000 0.000000 +0.01 0.09 0.000000 0.000000 0.000000 +0.03 0.09 0.000000 0.000000 0.000000 +0.04 0.09 0.000000 0.000000 0.000000 +0.04 0.09 0.000000 0.000000 0.000000 +0.06 0.09 0.000000 0.000000 0.000000 +0.07 0.09 0.000000 0.000000 0.000000 +0.07 0.09 0.000000 0.000000 0.000000 +0.09 0.09 0.000000 0.000000 0.000000 +0.10 0.09 0.000000 0.000000 0.000000 +0.10 0.09 0.000000 0.000000 0.000000 +0.12 0.09 0.000000 0.000000 0.000000 +0.12 0.09 0.000000 0.000000 0.000000 +0.14 0.09 0.000000 0.000000 0.000000 +0.14 0.09 0.000000 0.000000 0.000000 +0.15 0.09 0.000000 0.000000 0.000000 +0.17 0.09 0.000000 0.000000 0.000000 +0.18 0.09 0.000000 0.000000 0.000000 +0.18 0.09 0.000000 0.000000 0.000000 +0.20 0.09 0.000000 0.000000 0.000000 +0.21 0.09 0.000000 0.000000 0.000000 +0.21 0.09 0.000000 0.000000 0.000000 +0.23 0.09 0.000000 0.000000 0.000000 +0.24 0.09 0.000000 0.000000 0.000000 +0.24 0.09 0.000000 0.000000 0.000000 +0.26 0.09 0.000000 0.000000 0.000000 +0.27 0.09 0.000000 0.000000 0.000000 +0.28 0.09 0.000000 0.000000 0.000000 +0.29 0.09 0.000000 0.000000 0.000000 +0.29 0.09 0.000000 0.000000 0.000000 +0.30 0.09 0.000000 0.000000 0.000000 +0.32 0.09 0.000000 0.000000 0.000000 +0.33 0.09 0.000000 0.000000 0.000000 +0.34 0.09 0.000000 0.000000 0.000000 +0.35 0.09 0.000000 0.000000 0.000000 +0.35 0.09 0.000000 0.000000 0.000000 +0.36 0.09 0.000000 0.000000 0.000000 +0.38 0.09 0.000000 0.000000 0.000000 +0.39 0.09 0.000000 0.000000 0.000000 +0.40 0.09 0.000000 0.000000 0.000000 +0.41 0.09 0.000000 0.000000 0.000000 +0.42 0.09 0.000000 0.000000 0.000000 +0.42 0.09 0.000000 0.000000 0.000000 +0.43 0.09 0.000000 0.000000 0.000000 +0.45 0.09 0.000000 0.000000 0.000000 +0.46 0.09 0.000000 0.000000 0.000000 +0.47 0.09 0.000000 0.000000 0.000000 +0.48 0.09 0.000000 0.000000 0.000000 +0.48 0.09 0.000000 0.000000 0.000000 +0.49 0.09 0.000000 0.000000 0.000000 +0.51 0.09 0.000000 0.000000 0.000000 +0.52 0.09 0.000000 0.000000 0.000000 +0.53 0.09 0.000000 0.000000 0.000000 +0.54 0.09 0.000000 0.000000 0.000000 +0.55 0.09 0.000000 0.000000 0.000000 +0.56 0.09 0.000000 0.000000 0.000000 +0.57 0.09 0.000000 0.000000 0.000000 +0.58 0.09 0.000000 0.000000 0.000000 +0.58 0.09 0.000000 0.000000 0.000000 +0.59 0.09 0.000000 0.000000 0.000000 +0.60 0.09 0.000000 0.000000 0.000000 +0.61 0.09 0.000000 0.000000 0.000000 +0.62 0.09 0.000000 0.000000 0.000000 +0.64 0.09 0.000000 0.000000 0.000000 +0.65 0.09 0.000000 0.000000 0.000000 +0.66 0.09 0.000000 0.000000 0.000000 +0.67 0.09 0.000000 0.000000 0.000000 +0.68 0.09 0.000000 0.000000 0.000000 +0.69 0.09 0.000000 0.000000 0.000000 +0.70 0.09 0.000000 0.000000 0.000000 +0.70 0.09 0.000000 0.000000 0.000000 +0.71 0.09 0.000000 0.000000 0.000000 +0.72 0.09 0.000000 0.000000 0.000000 +0.73 0.09 0.000000 0.000000 0.000000 +0.74 0.09 0.000000 0.000000 0.000000 +0.76 0.09 0.000000 0.000000 0.000000 +0.77 0.09 0.000000 0.000000 0.000000 +0.78 0.09 0.000000 0.000000 0.000000 +0.79 0.09 0.000000 0.000000 0.000000 +0.80 0.09 0.000000 0.000000 0.000000 +0.81 0.09 0.000000 0.000000 0.000000 +0.82 0.09 0.000000 0.000000 0.000000 +0.83 0.09 0.000000 0.000000 0.000000 +0.83 0.09 0.000000 0.000000 0.000000 +0.84 0.09 0.000000 0.000000 0.000000 +0.85 0.09 0.000000 0.000000 0.000000 +0.86 0.09 0.000000 0.000000 0.000000 +0.88 0.09 0.000000 0.000000 0.000000 +0.89 0.09 0.000000 0.000000 0.000000 +0.90 0.09 0.000000 0.000000 0.000000 +0.91 0.09 0.000000 0.000000 0.000000 +0.92 0.09 0.000000 0.000000 0.000000 +0.93 0.09 0.000000 0.000000 0.000000 +0.94 0.09 0.000000 0.000000 0.000000 +0.95 0.09 0.000000 0.000000 0.000000 +0.96 0.09 0.000000 0.000000 0.000000 +0.96 0.09 0.000000 0.000000 0.000000 +0.97 0.09 0.000000 0.000000 0.000000 +0.98 0.09 0.000000 0.000000 0.000000 +0.01 0.10 0.000000 0.000000 0.000000 +0.01 0.10 0.000000 0.000000 0.000000 +0.03 0.10 0.000000 0.000000 0.000000 +0.04 0.10 0.000000 0.000000 0.000000 +0.04 0.10 0.000000 0.000000 0.000000 +0.06 0.10 0.000000 0.000000 0.000000 +0.07 0.10 0.000000 0.000000 0.000000 +0.07 0.10 0.000000 0.000000 0.000000 +0.09 0.10 0.000000 0.000000 0.000000 +0.10 0.10 0.000000 0.000000 0.000000 +0.10 0.10 0.000000 0.000000 0.000000 +0.12 0.10 0.000000 0.000000 0.000000 +0.12 0.10 0.000000 0.000000 0.000000 +0.14 0.10 0.000000 0.000000 0.000000 +0.14 0.10 0.000000 0.000000 0.000000 +0.15 0.10 0.000000 0.000000 0.000000 +0.17 0.10 0.000000 0.000000 0.000000 +0.18 0.10 0.000000 0.000000 0.000000 +0.18 0.10 0.000000 0.000000 0.000000 +0.20 0.10 0.000000 0.000000 0.000000 +0.21 0.10 0.000000 0.000000 0.000000 +0.21 0.10 0.000000 0.000000 0.000000 +0.23 0.10 0.000000 0.000000 0.000000 +0.24 0.10 0.000000 0.000000 0.000000 +0.24 0.10 0.000000 0.000000 0.000000 +0.26 0.10 0.000000 0.000000 0.000000 +0.27 0.10 0.000000 0.000000 0.000000 +0.28 0.10 0.000000 0.000000 0.000000 +0.29 0.10 0.000000 0.000000 0.000000 +0.29 0.10 0.000000 0.000000 0.000000 +0.30 0.10 0.000000 0.000000 0.000000 +0.32 0.10 0.000000 0.000000 0.000000 +0.33 0.10 0.000000 0.000000 0.000000 +0.34 0.10 0.000000 0.000000 0.000000 +0.35 0.10 0.000000 0.000000 0.000000 +0.35 0.10 0.000000 0.000000 0.000000 +0.36 0.10 0.000000 0.000000 0.000000 +0.38 0.10 0.000000 0.000000 0.000000 +0.39 0.10 0.000000 0.000000 0.000000 +0.40 0.10 0.000000 0.000000 0.000000 +0.41 0.10 0.000000 0.000000 0.000000 +0.42 0.10 0.000000 0.000000 0.000000 +0.42 0.10 0.000000 0.000000 0.000000 +0.43 0.10 0.000000 0.000000 0.000000 +0.45 0.10 0.000000 0.000000 0.000000 +0.46 0.10 0.000000 0.000000 0.000000 +0.47 0.10 0.000000 0.000000 0.000000 +0.48 0.10 0.000000 0.000000 0.000000 +0.48 0.10 0.000000 0.000000 0.000000 +0.49 0.10 0.000000 0.000000 0.000000 +0.51 0.10 0.000000 0.000000 0.000000 +0.52 0.10 0.000000 0.000000 0.000000 +0.53 0.10 0.000000 0.000000 0.000000 +0.54 0.10 0.000000 0.000000 0.000000 +0.55 0.10 0.000000 0.000000 0.000000 +0.56 0.10 0.000000 0.000000 0.000000 +0.57 0.10 0.000000 0.000000 0.000000 +0.58 0.10 0.000000 0.000000 0.000000 +0.58 0.10 0.000000 0.000000 0.000000 +0.59 0.10 0.000000 0.000000 0.000000 +0.60 0.10 0.000000 0.000000 0.000000 +0.61 0.10 0.000000 0.000000 0.000000 +0.62 0.10 0.000000 0.000000 0.000000 +0.64 0.10 0.000000 0.000000 0.000000 +0.65 0.10 0.000000 0.000000 0.000000 +0.66 0.10 0.000000 0.000000 0.000000 +0.67 0.10 0.000000 0.000000 0.000000 +0.68 0.10 0.000000 0.000000 0.000000 +0.69 0.10 0.000000 0.000000 0.000000 +0.70 0.10 0.000000 0.000000 0.000000 +0.70 0.10 0.000000 0.000000 0.000000 +0.71 0.10 0.000000 0.000000 0.000000 +0.72 0.10 0.000000 0.000000 0.000000 +0.73 0.10 0.000000 0.000000 0.000000 +0.74 0.10 0.000000 0.000000 0.000000 +0.76 0.10 0.000000 0.000000 0.000000 +0.77 0.10 0.000000 0.000000 0.000000 +0.78 0.10 0.000000 0.000000 0.000000 +0.79 0.10 0.000000 0.000000 0.000000 +0.80 0.10 0.000000 0.000000 0.000000 +0.81 0.10 0.000000 0.000000 0.000000 +0.82 0.10 0.000000 0.000000 0.000000 +0.83 0.10 0.000000 0.000000 0.000000 +0.83 0.10 0.000000 0.000000 0.000000 +0.84 0.10 0.000000 0.000000 0.000000 +0.85 0.10 0.000000 0.000000 0.000000 +0.86 0.10 0.000000 0.000000 0.000000 +0.88 0.10 0.000000 0.000000 0.000000 +0.89 0.10 0.000000 0.000000 0.000000 +0.90 0.10 0.000000 0.000000 0.000000 +0.91 0.10 0.000000 0.000000 0.000000 +0.92 0.10 0.000000 0.000000 0.000000 +0.93 0.10 0.000000 0.000000 0.000000 +0.94 0.10 0.000000 0.000000 0.000000 +0.95 0.10 0.000000 0.000000 0.000000 +0.96 0.10 0.000000 0.000000 0.000000 +0.96 0.10 0.000000 0.000000 0.000000 +0.97 0.10 0.000000 0.000000 0.000000 +0.98 0.10 0.000000 0.000000 0.000000 +0.01 0.10 0.000000 0.000000 0.000000 +0.01 0.10 0.000000 0.000000 0.000000 +0.03 0.10 0.000000 0.000000 0.000000 +0.04 0.10 0.000000 0.000000 0.000000 +0.04 0.10 0.000000 0.000000 0.000000 +0.06 0.10 0.000000 0.000000 0.000000 +0.07 0.10 0.000000 0.000000 0.000000 +0.07 0.10 0.000000 0.000000 0.000000 +0.09 0.10 0.000000 0.000000 0.000000 +0.10 0.10 0.000000 0.000000 0.000000 +0.10 0.10 0.000000 0.000000 0.000000 +0.12 0.10 0.000000 0.000000 0.000000 +0.12 0.10 0.000000 0.000000 0.000000 +0.14 0.10 0.000000 0.000000 0.000000 +0.14 0.10 0.000000 0.000000 0.000000 +0.15 0.10 0.000000 0.000000 0.000000 +0.17 0.10 0.000000 0.000000 0.000000 +0.18 0.10 0.000000 0.000000 0.000000 +0.18 0.10 0.000000 0.000000 0.000000 +0.20 0.10 0.000000 0.000000 0.000000 +0.21 0.10 0.000000 0.000000 0.000000 +0.21 0.10 0.000000 0.000000 0.000000 +0.23 0.10 0.000000 0.000000 0.000000 +0.24 0.10 0.000000 0.000000 0.000000 +0.24 0.10 0.000000 0.000000 0.000000 +0.26 0.10 0.000000 0.000000 0.000000 +0.27 0.10 0.000000 0.000000 0.000000 +0.28 0.10 0.000000 0.000000 0.000000 +0.29 0.10 0.000000 0.000000 0.000000 +0.29 0.10 0.000000 0.000000 0.000000 +0.30 0.10 0.000000 0.000000 0.000000 +0.32 0.10 0.000000 0.000000 0.000000 +0.33 0.10 0.000000 0.000000 0.000000 +0.34 0.10 0.000000 0.000000 0.000000 +0.35 0.10 0.000000 0.000000 0.000000 +0.35 0.10 0.000000 0.000000 0.000000 +0.36 0.10 0.000000 0.000000 0.000000 +0.38 0.10 0.000000 0.000000 0.000000 +0.39 0.10 0.000000 0.000000 0.000000 +0.40 0.10 0.000000 0.000000 0.000000 +0.41 0.10 0.000000 0.000000 0.000000 +0.42 0.10 0.000000 0.000000 0.000000 +0.42 0.10 0.000000 0.000000 0.000000 +0.43 0.10 0.000000 0.000000 0.000000 +0.45 0.10 0.000000 0.000000 0.000000 +0.46 0.10 0.000000 0.000000 0.000000 +0.47 0.10 0.000000 0.000000 0.000000 +0.48 0.10 0.000000 0.000000 0.000000 +0.48 0.10 0.000000 0.000000 0.000000 +0.49 0.10 0.000000 0.000000 0.000000 +0.51 0.10 0.000000 0.000000 0.000000 +0.52 0.10 0.000000 0.000000 0.000000 +0.53 0.10 0.000000 0.000000 0.000000 +0.54 0.10 0.000000 0.000000 0.000000 +0.55 0.10 0.000000 0.000000 0.000000 +0.56 0.10 0.000000 0.000000 0.000000 +0.57 0.10 0.000000 0.000000 0.000000 +0.58 0.10 0.000000 0.000000 0.000000 +0.58 0.10 0.000000 0.000000 0.000000 +0.59 0.10 0.000000 0.000000 0.000000 +0.60 0.10 0.000000 0.000000 0.000000 +0.61 0.10 0.000000 0.000000 0.000000 +0.62 0.10 0.000000 0.000000 0.000000 +0.64 0.10 0.000000 0.000000 0.000000 +0.65 0.10 0.000000 0.000000 0.000000 +0.66 0.10 0.000000 0.000000 0.000000 +0.67 0.10 0.000000 0.000000 0.000000 +0.68 0.10 0.000000 0.000000 0.000000 +0.69 0.10 0.000000 0.000000 0.000000 +0.70 0.10 0.000000 0.000000 0.000000 +0.70 0.10 0.000000 0.000000 0.000000 +0.71 0.10 0.000000 0.000000 0.000000 +0.72 0.10 0.000000 0.000000 0.000000 +0.73 0.10 0.000000 0.000000 0.000000 +0.74 0.10 0.000000 0.000000 0.000000 +0.76 0.10 0.000000 0.000000 0.000000 +0.77 0.10 0.000000 0.000000 0.000000 +0.78 0.10 0.000000 0.000000 0.000000 +0.79 0.10 0.000000 0.000000 0.000000 +0.80 0.10 0.000000 0.000000 0.000000 +0.81 0.10 0.000000 0.000000 0.000000 +0.82 0.10 0.000000 0.000000 0.000000 +0.83 0.10 0.000000 0.000000 0.000000 +0.83 0.10 0.000000 0.000000 0.000000 +0.84 0.10 0.000000 0.000000 0.000000 +0.85 0.10 0.000000 0.000000 0.000000 +0.86 0.10 0.000000 0.000000 0.000000 +0.88 0.10 0.000000 0.000000 0.000000 +0.89 0.10 0.000000 0.000000 0.000000 +0.90 0.10 0.000000 0.000000 0.000000 +0.91 0.10 0.000000 0.000000 0.000000 +0.92 0.10 0.000000 0.000000 0.000000 +0.93 0.10 0.000000 0.000000 0.000000 +0.94 0.10 0.000000 0.000000 0.000000 +0.95 0.10 0.000000 0.000000 0.000000 +0.96 0.10 0.000000 0.000000 0.000000 +0.96 0.10 0.000000 0.000000 0.000000 +0.97 0.10 0.000000 0.000000 0.000000 +0.98 0.10 0.000000 0.000000 0.000000 +0.01 0.12 0.000000 0.000000 0.000000 +0.01 0.12 0.000000 0.000000 0.000000 +0.03 0.12 0.000000 0.000000 0.000000 +0.04 0.12 0.000000 0.000000 0.000000 +0.04 0.12 0.000000 0.000000 0.000000 +0.06 0.12 0.000000 0.000000 0.000000 +0.07 0.12 0.000000 0.000000 0.000000 +0.07 0.12 0.000000 0.000000 0.000000 +0.09 0.12 0.000000 0.000000 0.000000 +0.10 0.12 0.000000 0.000000 0.000000 +0.10 0.12 0.000000 0.000000 0.000000 +0.12 0.12 0.000000 0.000000 0.000000 +0.12 0.12 0.000000 0.000000 0.000000 +0.14 0.12 0.000000 0.000000 0.000000 +0.14 0.12 0.000000 0.000000 0.000000 +0.15 0.12 0.000000 0.000000 0.000000 +0.17 0.12 0.000000 0.000000 0.000000 +0.18 0.12 0.000000 0.000000 0.000000 +0.18 0.12 0.000000 0.000000 0.000000 +0.20 0.12 0.000000 0.000000 0.000000 +0.21 0.12 0.000000 0.000000 0.000000 +0.21 0.12 0.000000 0.000000 0.000000 +0.23 0.12 0.000000 0.000000 0.000000 +0.24 0.12 0.000000 0.000000 0.000000 +0.24 0.12 0.000000 0.000000 0.000000 +0.26 0.12 0.000000 0.000000 0.000000 +0.27 0.12 0.000000 0.000000 0.000000 +0.28 0.12 0.000000 0.000000 0.000000 +0.29 0.12 0.000000 0.000000 0.000000 +0.29 0.12 0.000000 0.000000 0.000000 +0.30 0.12 0.000000 0.000000 0.000000 +0.32 0.12 0.000000 0.000000 0.000000 +0.33 0.12 0.000000 0.000000 0.000000 +0.34 0.12 0.000000 0.000000 0.000000 +0.35 0.12 0.000000 0.000000 0.000000 +0.35 0.12 0.000000 0.000000 0.000000 +0.36 0.12 0.000000 0.000000 0.000000 +0.38 0.12 0.000000 0.000000 0.000000 +0.39 0.12 0.000000 0.000000 0.000000 +0.40 0.12 0.000000 0.000000 0.000000 +0.41 0.12 0.000000 0.000000 0.000000 +0.42 0.12 0.000000 0.000000 0.000000 +0.42 0.12 0.000000 0.000000 0.000000 +0.43 0.12 0.000000 0.000000 0.000000 +0.45 0.12 0.000000 0.000000 0.000000 +0.46 0.12 0.000000 0.000000 0.000000 +0.47 0.12 0.000000 0.000000 0.000000 +0.48 0.12 0.000000 0.000000 0.000000 +0.48 0.12 0.000000 0.000000 0.000000 +0.49 0.12 0.000000 0.000000 0.000000 +0.51 0.12 0.000000 0.000000 0.000000 +0.52 0.12 0.000000 0.000000 0.000000 +0.53 0.12 0.000000 0.000000 0.000000 +0.54 0.12 0.000000 0.000000 0.000000 +0.55 0.12 0.000000 0.000000 0.000000 +0.56 0.12 0.000000 0.000000 0.000000 +0.57 0.12 0.000000 0.000000 0.000000 +0.58 0.12 0.000000 0.000000 0.000000 +0.58 0.12 0.000000 0.000000 0.000000 +0.59 0.12 0.000000 0.000000 0.000000 +0.60 0.12 0.000000 0.000000 0.000000 +0.61 0.12 0.000000 0.000000 0.000000 +0.62 0.12 0.000000 0.000000 0.000000 +0.64 0.12 0.000000 0.000000 0.000000 +0.65 0.12 0.000000 0.000000 0.000000 +0.66 0.12 0.000000 0.000000 0.000000 +0.67 0.12 0.000000 0.000000 0.000000 +0.68 0.12 0.000000 0.000000 0.000000 +0.69 0.12 0.000000 0.000000 0.000000 +0.70 0.12 0.000000 0.000000 0.000000 +0.70 0.12 0.000000 0.000000 0.000000 +0.71 0.12 0.000000 0.000000 0.000000 +0.72 0.12 0.000000 0.000000 0.000000 +0.73 0.12 0.000000 0.000000 0.000000 +0.74 0.12 0.000000 0.000000 0.000000 +0.76 0.12 0.000000 0.000000 0.000000 +0.77 0.12 0.000000 0.000000 0.000000 +0.78 0.12 0.000000 0.000000 0.000000 +0.79 0.12 0.000000 0.000000 0.000000 +0.80 0.12 0.000000 0.000000 0.000000 +0.81 0.12 0.000000 0.000000 0.000000 +0.82 0.12 0.000000 0.000000 0.000000 +0.83 0.12 0.000000 0.000000 0.000000 +0.83 0.12 0.000000 0.000000 0.000000 +0.84 0.12 0.000000 0.000000 0.000000 +0.85 0.12 0.000000 0.000000 0.000000 +0.86 0.12 0.000000 0.000000 0.000000 +0.88 0.12 0.000000 0.000000 0.000000 +0.89 0.12 0.000000 0.000000 0.000000 +0.90 0.12 0.000000 0.000000 0.000000 +0.91 0.12 0.000000 0.000000 0.000000 +0.92 0.12 0.000000 0.000000 0.000000 +0.93 0.12 0.000000 0.000000 0.000000 +0.94 0.12 0.000000 0.000000 0.000000 +0.95 0.12 0.000000 0.000000 0.000000 +0.96 0.12 0.000000 0.000000 0.000000 +0.96 0.12 0.000000 0.000000 0.000000 +0.97 0.12 0.000000 0.000000 0.000000 +0.98 0.12 0.000000 0.000000 0.000000 +0.01 0.12 0.000000 0.000000 0.000000 +0.01 0.12 0.000000 0.000000 0.000000 +0.03 0.12 0.000000 0.000000 0.000000 +0.04 0.12 0.000000 0.000000 0.000000 +0.04 0.12 0.000000 0.000000 0.000000 +0.06 0.12 0.000000 0.000000 0.000000 +0.07 0.12 0.000000 0.000000 0.000000 +0.07 0.12 0.000000 0.000000 0.000000 +0.09 0.12 0.000000 0.000000 0.000000 +0.10 0.12 0.000000 0.000000 0.000000 +0.10 0.12 0.000000 0.000000 0.000000 +0.12 0.12 0.000000 0.000000 0.000000 +0.12 0.12 0.000000 0.000000 0.000000 +0.14 0.12 0.000000 0.000000 0.000000 +0.14 0.12 0.000000 0.000000 0.000000 +0.15 0.12 0.000000 0.000000 0.000000 +0.17 0.12 0.000000 0.000000 0.000000 +0.18 0.12 0.000000 0.000000 0.000000 +0.18 0.12 0.000000 0.000000 0.000000 +0.20 0.12 0.000000 0.000000 0.000000 +0.21 0.12 0.000000 0.000000 0.000000 +0.21 0.12 0.000000 0.000000 0.000000 +0.23 0.12 0.000000 0.000000 0.000000 +0.24 0.12 0.000000 0.000000 0.000000 +0.24 0.12 0.000000 0.000000 0.000000 +0.26 0.12 0.000000 0.000000 0.000000 +0.27 0.12 0.000000 0.000000 0.000000 +0.28 0.12 0.000000 0.000000 0.000000 +0.29 0.12 0.000000 0.000000 0.000000 +0.29 0.12 0.000000 0.000000 0.000000 +0.30 0.12 0.000000 0.000000 0.000000 +0.32 0.12 0.000000 0.000000 0.000000 +0.33 0.12 0.000000 0.000000 0.000000 +0.34 0.12 0.000000 0.000000 0.000000 +0.35 0.12 0.000000 0.000000 0.000000 +0.35 0.12 0.000000 0.000000 0.000000 +0.36 0.12 0.000000 0.000000 0.000000 +0.38 0.12 0.000000 0.000000 0.000000 +0.39 0.12 0.000000 0.000000 0.000000 +0.40 0.12 0.000000 0.000000 0.000000 +0.41 0.12 0.000000 0.000000 0.000000 +0.42 0.12 0.000000 0.000000 0.000000 +0.42 0.12 0.000000 0.000000 0.000000 +0.43 0.12 0.000000 0.000000 0.000000 +0.45 0.12 0.000000 0.000000 0.000000 +0.46 0.12 0.000000 0.000000 0.000000 +0.47 0.12 0.000000 0.000000 0.000000 +0.48 0.12 0.000000 0.000000 0.000000 +0.48 0.12 0.000000 0.000000 0.000000 +0.49 0.12 0.000000 0.000000 0.000000 +0.51 0.12 0.000000 0.000000 0.000000 +0.52 0.12 0.000000 0.000000 0.000000 +0.53 0.12 0.000000 0.000000 0.000000 +0.54 0.12 0.000000 0.000000 0.000000 +0.55 0.12 0.000000 0.000000 0.000000 +0.56 0.12 0.000000 0.000000 0.000000 +0.57 0.12 0.000000 0.000000 0.000000 +0.58 0.12 0.000000 0.000000 0.000000 +0.58 0.12 0.000000 0.000000 0.000000 +0.59 0.12 0.000000 0.000000 0.000000 +0.60 0.12 0.000000 0.000000 0.000000 +0.61 0.12 0.000000 0.000000 0.000000 +0.62 0.12 0.000000 0.000000 0.000000 +0.64 0.12 0.000000 0.000000 0.000000 +0.65 0.12 0.000000 0.000000 0.000000 +0.66 0.12 0.000000 0.000000 0.000000 +0.67 0.12 0.000000 0.000000 0.000000 +0.68 0.12 0.000000 0.000000 0.000000 +0.69 0.12 0.000000 0.000000 0.000000 +0.70 0.12 0.000000 0.000000 0.000000 +0.70 0.12 0.000000 0.000000 0.000000 +0.71 0.12 0.000000 0.000000 0.000000 +0.72 0.12 0.000000 0.000000 0.000000 +0.73 0.12 0.000000 0.000000 0.000000 +0.74 0.12 0.000000 0.000000 0.000000 +0.76 0.12 0.000000 0.000000 0.000000 +0.77 0.12 0.000000 0.000000 0.000000 +0.78 0.12 0.000000 0.000000 0.000000 +0.79 0.12 0.000000 0.000000 0.000000 +0.80 0.12 0.000000 0.000000 0.000000 +0.81 0.12 0.000000 0.000000 0.000000 +0.82 0.12 0.000000 0.000000 0.000000 +0.83 0.12 0.000000 0.000000 0.000000 +0.83 0.12 0.000000 0.000000 0.000000 +0.84 0.12 0.000000 0.000000 0.000000 +0.85 0.12 0.000000 0.000000 0.000000 +0.86 0.12 0.000000 0.000000 0.000000 +0.88 0.12 0.000000 0.000000 0.000000 +0.89 0.12 0.000000 0.000000 0.000000 +0.90 0.12 0.000000 0.000000 0.000000 +0.91 0.12 0.000000 0.000000 0.000000 +0.92 0.12 0.000000 0.000000 0.000000 +0.93 0.12 0.000000 0.000000 0.000000 +0.94 0.12 0.000000 0.000000 0.000000 +0.95 0.12 0.000000 0.000000 0.000000 +0.96 0.12 0.000000 0.000000 0.000000 +0.96 0.12 0.000000 0.000000 0.000000 +0.97 0.12 0.000000 0.000000 0.000000 +0.98 0.12 0.000000 0.000000 0.000000 +0.01 0.14 0.000000 0.000000 0.000000 +0.01 0.14 0.000000 0.000000 0.000000 +0.03 0.14 0.000000 0.000000 0.000000 +0.04 0.14 0.000000 0.000000 0.000000 +0.04 0.14 0.000000 0.000000 0.000000 +0.06 0.14 0.000000 0.000000 0.000000 +0.07 0.14 0.000000 0.000000 0.000000 +0.07 0.14 0.000000 0.000000 0.000000 +0.09 0.14 0.000000 0.000000 0.000000 +0.10 0.14 0.000000 0.000000 0.000000 +0.10 0.14 0.000000 0.000000 0.000000 +0.12 0.14 0.000000 0.000000 0.000000 +0.12 0.14 0.000000 0.000000 0.000000 +0.14 0.14 0.000000 0.000000 0.000000 +0.14 0.14 0.000000 0.000000 0.000000 +0.15 0.14 0.000000 0.000000 0.000000 +0.17 0.14 0.000000 0.000000 0.000000 +0.18 0.14 0.000000 0.000000 0.000000 +0.18 0.14 0.000000 0.000000 0.000000 +0.20 0.14 0.000000 0.000000 0.000000 +0.21 0.14 0.000000 0.000000 0.000000 +0.21 0.14 0.000000 0.000000 0.000000 +0.23 0.14 0.000000 0.000000 0.000000 +0.24 0.14 0.000000 0.000000 0.000000 +0.24 0.14 0.000000 0.000000 0.000000 +0.26 0.14 0.000000 0.000000 0.000000 +0.27 0.14 0.000000 0.000000 0.000000 +0.28 0.14 0.000000 0.000000 0.000000 +0.29 0.14 0.000000 0.000000 0.000000 +0.29 0.14 0.000000 0.000000 0.000000 +0.30 0.14 0.000000 0.000000 0.000000 +0.32 0.14 0.000000 0.000000 0.000000 +0.33 0.14 0.000000 0.000000 0.000000 +0.34 0.14 0.000000 0.000000 0.000000 +0.35 0.14 0.000000 0.000000 0.000000 +0.35 0.14 0.000000 0.000000 0.000000 +0.36 0.14 0.000000 0.000000 0.000000 +0.38 0.14 0.000000 0.000000 0.000000 +0.39 0.14 0.000000 0.000000 0.000000 +0.40 0.14 0.000000 0.000000 0.000000 +0.41 0.14 0.000000 0.000000 0.000000 +0.42 0.14 0.000000 0.000000 0.000000 +0.42 0.14 0.000000 0.000000 0.000000 +0.43 0.14 0.000000 0.000000 0.000000 +0.45 0.14 0.000000 0.000000 0.000000 +0.46 0.14 0.000000 0.000000 0.000000 +0.47 0.14 0.000000 0.000000 0.000000 +0.48 0.14 0.000000 0.000000 0.000000 +0.48 0.14 0.000000 0.000000 0.000000 +0.49 0.14 0.000000 0.000000 0.000000 +0.51 0.14 0.000000 0.000000 0.000000 +0.52 0.14 0.000000 0.000000 0.000000 +0.53 0.14 0.000000 0.000000 0.000000 +0.54 0.14 0.000000 0.000000 0.000000 +0.55 0.14 0.000000 0.000000 0.000000 +0.56 0.14 0.000000 0.000000 0.000000 +0.57 0.14 0.000000 0.000000 0.000000 +0.58 0.14 0.000000 0.000000 0.000000 +0.58 0.14 0.000000 0.000000 0.000000 +0.59 0.14 0.000000 0.000000 0.000000 +0.60 0.14 0.000000 0.000000 0.000000 +0.61 0.14 0.000000 0.000000 0.000000 +0.62 0.14 0.000000 0.000000 0.000000 +0.64 0.14 0.000000 0.000000 0.000000 +0.65 0.14 0.000000 0.000000 0.000000 +0.66 0.14 0.000000 0.000000 0.000000 +0.67 0.14 0.000000 0.000000 0.000000 +0.68 0.14 0.000000 0.000000 0.000000 +0.69 0.14 0.000000 0.000000 0.000000 +0.70 0.14 0.000000 0.000000 0.000000 +0.70 0.14 0.000000 0.000000 0.000000 +0.71 0.14 0.000000 0.000000 0.000000 +0.72 0.14 0.000000 0.000000 0.000000 +0.73 0.14 0.000000 0.000000 0.000000 +0.74 0.14 0.000000 0.000000 0.000000 +0.76 0.14 0.000000 0.000000 0.000000 +0.77 0.14 0.000000 0.000000 0.000000 +0.78 0.14 0.000000 0.000000 0.000000 +0.79 0.14 0.000000 0.000000 0.000000 +0.80 0.14 0.000000 0.000000 0.000000 +0.81 0.14 0.000000 0.000000 0.000000 +0.82 0.14 0.000000 0.000000 0.000000 +0.83 0.14 0.000000 0.000000 0.000000 +0.83 0.14 0.000000 0.000000 0.000000 +0.84 0.14 0.000000 0.000000 0.000000 +0.85 0.14 0.000000 0.000000 0.000000 +0.86 0.14 0.000000 0.000000 0.000000 +0.88 0.14 0.000000 0.000000 0.000000 +0.89 0.14 0.000000 0.000000 0.000000 +0.90 0.14 0.000000 0.000000 0.000000 +0.91 0.14 0.000000 0.000000 0.000000 +0.92 0.14 0.000000 0.000000 0.000000 +0.93 0.14 0.000000 0.000000 0.000000 +0.94 0.14 0.000000 0.000000 0.000000 +0.95 0.14 0.000000 0.000000 0.000000 +0.96 0.14 0.000000 0.000000 0.000000 +0.96 0.14 0.000000 0.000000 0.000000 +0.97 0.14 0.000000 0.000000 0.000000 +0.98 0.14 0.000000 0.000000 0.000000 +0.01 0.14 0.000000 0.000000 0.000000 +0.01 0.14 0.000000 0.000000 0.000000 +0.03 0.14 0.000000 0.000000 0.000000 +0.04 0.14 0.000000 0.000000 0.000000 +0.04 0.14 0.000000 0.000000 0.000000 +0.06 0.14 0.000000 0.000000 0.000000 +0.07 0.14 0.000000 0.000000 0.000000 +0.07 0.14 0.000000 0.000000 0.000000 +0.09 0.14 0.000000 0.000000 0.000000 +0.10 0.14 0.000000 0.000000 0.000000 +0.10 0.14 0.000000 0.000000 0.000000 +0.12 0.14 0.000000 0.000000 0.000000 +0.12 0.14 0.000000 0.000000 0.000000 +0.14 0.14 0.000000 0.000000 0.000000 +0.14 0.14 0.000000 0.000000 0.000000 +0.15 0.14 0.000000 0.000000 0.000000 +0.17 0.14 0.000000 0.000000 0.000000 +0.18 0.14 0.000000 0.000000 0.000000 +0.18 0.14 0.000000 0.000000 0.000000 +0.20 0.14 0.000000 0.000000 0.000000 +0.21 0.14 0.000000 0.000000 0.000000 +0.21 0.14 0.000000 0.000000 0.000000 +0.23 0.14 0.000000 0.000000 0.000000 +0.24 0.14 0.000000 0.000000 0.000000 +0.24 0.14 0.000000 0.000000 0.000000 +0.26 0.14 0.000000 0.000000 0.000000 +0.27 0.14 0.000000 0.000000 0.000000 +0.28 0.14 0.000000 0.000000 0.000000 +0.29 0.14 0.000000 0.000000 0.000000 +0.29 0.14 0.000000 0.000000 0.000000 +0.30 0.14 0.000000 0.000000 0.000000 +0.32 0.14 0.000000 0.000000 0.000000 +0.33 0.14 0.000000 0.000000 0.000000 +0.34 0.14 0.000000 0.000000 0.000000 +0.35 0.14 0.000000 0.000000 0.000000 +0.35 0.14 0.000000 0.000000 0.000000 +0.36 0.14 0.000000 0.000000 0.000000 +0.38 0.14 0.000000 0.000000 0.000000 +0.39 0.14 0.000000 0.000000 0.000000 +0.40 0.14 0.000000 0.000000 0.000000 +0.41 0.14 0.000000 0.000000 0.000000 +0.42 0.14 0.000000 0.000000 0.000000 +0.42 0.14 0.000000 0.000000 0.000000 +0.43 0.14 0.000000 0.000000 0.000000 +0.45 0.14 0.000000 0.000000 0.000000 +0.46 0.14 0.000000 0.000000 0.000000 +0.47 0.14 0.000000 0.000000 0.000000 +0.48 0.14 0.000000 0.000000 0.000000 +0.48 0.14 0.000000 0.000000 0.000000 +0.49 0.14 0.000000 0.000000 0.000000 +0.51 0.14 0.000000 0.000000 0.000000 +0.52 0.14 0.000000 0.000000 0.000000 +0.53 0.14 0.000000 0.000000 0.000000 +0.54 4 nan nan nan +0.46 0.24 nan nan nan +0.47 0.24 nan nan nan +0.48 0.24 nan nan nan +0.48 0.24 nan nan nan +0.49 0.24 nan nan nan +0.51 0.24 nan nan nan +0.52 0.24 nan nan nan +0.53 0.24 nan nan nan +0.54 0.24 nan nan nan +0.55 0.24 nan nan nan +0.56 0.24 nan nan nan +0.57 0.24 nan nan nan +0.58 0.24 nan nan nan +0.58 0.24 nan nan nan +0.59 0.24 nan nan nan +0.60 0.24 nan nan nan +0.61 0.24 nan nan nan +0.62 0.24 nan nan nan +0.64 0.24 nan nan nan +0.65 0.24 nan nan nan +0.66 0.24 nan nan nan +0.67 0.24 nan nan nan +0.68 0.24 nan nan nan +0.69 0.24 nan nan nan +0.70 0.24 nan nan nan +0.70 0.24 nan nan nan +0.71 0.24 nan nan nan +0.72 0.24 nan nan nan +0.73 0.24 nan nan nan +0.74 0.24 nan nan nan +0.76 0.24 nan nan nan +0.77 0.24 nan nan nan +0.78 0.24 nan nan nan +0.79 0.24 nan nan nan +0.80 0.24 nan nan nan +0.81 0.24 nan nan nan +0.82 0.24 nan nan nan +0.83 0.24 nan nan nan +0.83 0.24 nan nan nan +0.84 0.24 nan nan nan +0.85 0.24 nan nan nan +0.86 0.24 nan nan nan +0.88 0.24 nan nan nan +0.89 0.24 nan nan nan +0.90 0.24 nan nan nan +0.91 0.24 nan nan nan +0.92 0.24 nan nan nan +0.93 0.24 nan nan nan +0.94 0.24 nan nan nan +0.95 0.24 nan nan nan +0.96 0.24 nan nan nan +0.96 0.24 nan nan nan +0.97 0.24 nan nan nan +0.98 0.24 nan nan nan +0.01 0.26 nan nan nan +0.01 0.26 nan nan nan +0.03 0.26 nan nan nan +0.04 0.26 nan nan nan +0.04 0.26 nan nan nan +0.06 0.26 nan nan nan +0.07 0.26 nan nan nan +0.07 0.26 nan nan nan +0.09 0.26 nan nan nan +0.10 0.26 nan nan nan +0.10 0.26 nan nan nan +0.12 0.26 nan nan nan +0.12 0.26 nan nan nan +0.14 0.26 nan nan nan +0.14 0.26 nan nan nan +0.15 0.26 nan nan nan +0.17 0.26 nan nan nan +0.18 0.26 nan nan nan +0.18 0.26 nan nan nan +0.20 0.26 nan nan nan +0.21 0.26 nan nan nan +0.21 0.26 nan nan nan +0.23 0.26 nan nan nan +0.24 0.26 nan nan nan +0.24 0.26 nan nan nan +0.26 0.26 nan nan nan +0.27 0.26 nan nan nan +0.28 0.26 nan nan nan +0.29 0.26 nan nan nan +0.29 0.26 nan nan nan +0.30 0.26 nan nan nan +0.32 0.26 nan nan nan +0.33 0.26 nan nan nan +0.34 0.26 nan nan nan +0.35 0.26 nan nan nan +0.35 0.26 nan nan nan +0.36 0.26 nan nan nan +0.38 0.26 nan nan nan +0.39 0.26 nan nan nan +0.40 0.26 nan nan nan +0.41 0.26 nan nan nan +0.42 0.26 nan nan nan +0.42 0.26 nan nan nan +0.43 0.26 nan nan nan +0.45 0.26 nan nan nan +0.46 0.26 nan nan nan +0.47 0.26 nan nan nan +0.48 0.26 nan nan nan +0.48 0.26 nan nan nan +0.49 0.26 nan nan nan +0.51 0.26 nan nan nan +0.52 0.26 nan nan nan +0.53 0.26 nan nan nan +0.54 0.26 nan nan nan +0.55 0.26 nan nan nan +0.56 0.26 nan nan nan +0.57 0.26 nan nan nan +0.58 0.26 nan nan nan +0.58 0.26 nan nan nan +0.59 0.26 nan nan nan +0.60 0.26 nan nan nan +0.61 0.26 nan nan nan +0.62 0.26 nan nan nan +0.64 0.26 nan nan nan +0.65 0.26 nan nan nan +0.66 0.26 nan nan nan +0.67 0.26 nan nan nan +0.68 0.26 nan nan nan +0.69 0.26 nan nan nan +0.70 0.26 nan nan nan +0.70 0.26 nan nan nan +0.71 0.26 nan nan nan +0.72 0.26 nan nan nan +0.73 0.26 nan nan nan +0.74 0.26 nan nan nan +0.76 0.26 nan nan nan +0.77 0.26 nan nan nan +0.78 0.26 nan nan nan +0.79 0.26 nan nan nan +0.80 0.26 nan nan nan +0.81 0.26 nan nan nan +0.82 0.26 nan nan nan +0.83 0.26 nan nan nan +0.83 0.26 nan nan nan +0.84 0.26 nan nan nan +0.85 0.26 nan nan nan +0.86 0.26 nan nan nan +0.88 0.26 nan nan nan +0.89 0.26 nan nan nan +0.90 0.26 nan nan nan +0.91 0.26 nan nan nan +0.92 0.26 nan nan nan +0.93 0.26 nan nan nan +0.94 0.26 nan nan nan +0.95 0.26 nan nan nan +0.96 0.26 nan nan nan +0.96 0.26 nan nan nan +0.97 0.26 nan nan nan +0.98 0.26 nan nan nan +0.01 0.27 nan nan nan +0.01 0.27 nan nan nan +0.03 0.27 nan nan nan +0.04 0.27 nan nan nan +0.04 0.27 nan nan nan +0.06 0.27 nan nan nan +0.07 0.27 nan nan nan +0.07 0.27 nan nan nan +0.09 0.27 nan nan nan +0.10 0.27 nan nan nan +0.10 0.27 nan nan nan +0.12 0.27 nan nan nan +0.12 0.27 nan nan nan +0.14 0.27 nan nan nan +0.14 0.27 nan nan nan +0.15 0.27 nan nan nan +0.17 0.27 nan nan nan +0.18 0.27 nan nan nan +0.18 0.27 nan nan nan +0.20 0.27 nan nan nan +0.21 0.27 nan nan nan +0.21 0.27 nan nan nan +0.23 0.27 nan nan nan +0.24 0.27 nan nan nan +0.24 0.27 nan nan nan +0.26 0.27 nan nan nan +0.27 0.27 nan nan nan +0.28 0.27 nan nan nan +0.29 0.27 nan nan nan +0.29 0.27 nan nan nan +0.30 0.27 nan nan nan +0.32 0.27 nan nan nan +0.33 0.27 nan nan nan +0.34 0.27 nan nan nan +0.35 0.27 nan nan nan +0.35 0.27 nan nan nan +0.36 0.27 nan nan nan +0.38 0.27 nan nan nan +0.39 0.27 nan nan nan +0.40 0.27 nan nan nan +0.41 0.27 nan nan nan +0.42 0.27 nan nan nan +0.42 0.27 nan nan nan +0.43 0.27 nan nan nan +0.45 0.27 nan nan nan +0.46 0.27 nan nan nan +0.47 0.27 nan nan nan +0.48 0.27 nan nan nan +0.48 0.27 nan nan nan +0.49 0.27 nan nan nan +0.51 0.27 nan nan nan +0.52 0.27 nan nan nan +0.53 0.27 nan nan nan +0.54 0.27 nan nan nan +0.55 0.27 nan nan nan +0.56 0.27 nan nan nan +0.57 0.27 nan nan nan +0.58 0.27 nan nan nan +0.58 0.27 nan nan nan +0.59 0.27 nan nan nan +0.60 0.27 nan nan nan +0.61 0.27 nan nan nan +0.62 0.27 nan nan nan +0.64 0.27 nan nan nan +0.65 0.27 nan nan nan +0.66 0.27 nan nan nan +0.67 0.27 nan nan nan +0.68 0.27 nan nan nan +0.69 0.27 nan nan nan +0.70 0.27 nan nan nan +0.70 0.27 nan nan nan +0.71 0.27 nan nan nan +0.72 0.27 nan nan nan +0.73 0.27 nan nan nan +0.74 0.27 nan nan nan +0.76 0.27 nan nan nan +0.77 0.27 nan nan nan +0.78 0.27 nan nan nan +0.79 0.27 nan nan nan +0.80 0.27 nan nan nan +0.81 0.27 nan nan nan +0.82 0.27 nan nan nan +0.83 0.27 nan nan nan +0.83 0.27 nan nan nan +0.84 0.27 nan nan nan +0.85 0.27 nan nan nan +0.86 0.27 nan nan nan +0.88 0.27 nan nan nan +0.89 0.27 nan nan nan +0.90 0.27 nan nan nan +0.91 0.27 nan nan nan +0.92 0.27 nan nan nan +0.93 0.27 nan nan nan +0.94 0.27 nan nan nan +0.95 0.27 nan nan nan +0.96 0.27 nan nan nan +0.96 0.27 nan nan nan +0.97 0.27 nan nan nan +0.98 0.27 nan nan nan +0.01 0.28 nan nan nan +0.01 0.28 nan nan nan +0.03 0.28 nan nan nan +0.04 0.28 nan nan nan +0.04 0.28 nan nan nan +0.06 0.28 nan nan nan +0.07 0.28 nan nan nan +0.07 0.28 nan nan nan +0.09 0.28 nan nan nan +0.10 0.28 nan nan nan +0.10 0.28 nan nan nan +0.12 0.28 nan nan nan +0.12 0.28 nan nan nan +0.14 0.28 nan nan nan +0.14 0.28 nan nan nan +0.15 0.28 nan nan nan +0.17 0.28 nan nan nan +0.18 0.28 nan nan nan +0.18 0.28 nan nan nan +0.20 0.28 nan nan nan +0.21 0.28 nan nan nan +0.21 0.28 nan nan nan +0.23 0.28 nan nan nan +0.24 0.28 nan nan nan +0.24 0.28 nan nan nan +0.26 0.28 nan nan nan +0.27 0.28 nan nan nan +0.28 0.28 nan nan nan +0.29 0.28 nan nan nan +0.29 0.28 nan nan nan +0.30 0.28 nan nan nan +0.32 0.28 nan nan nan +0.33 0.28 nan nan nan +0.34 0.28 nan nan nan +0.35 0.28 nan nan nan +0.35 0.28 nan nan nan +0.36 0.28 nan nan nan +0.38 0.28 nan nan nan +0.39 0.28 nan nan nan +0.40 0.28 nan nan nan +0.41 0.28 nan nan nan +0.42 0.28 nan nan nan +0.42 0.28 nan nan nan +0.43 0.28 nan nan nan +0.45 0.28 nan nan nan +0.46 0.28 nan nan nan +0.47 0.28 nan nan nan +0.48 0.28 nan nan nan +0.48 0.28 nan nan nan +0.49 0.28 nan nan nan +0.51 0.28 nan nan nan +0.52 0.28 nan nan nan +0.53 0.28 nan nan nan +0.54 0.28 nan nan nan +0.55 0.28 nan nan nan +0.56 0.28 nan nan nan +0.57 0.28 nan nan nan +0.58 0.28 nan nan nan +0.58 0.28 nan nan nan +0.59 0.28 nan nan nan +0.60 0.28 nan nan nan +0.61 0.28 nan nan nan +0.62 0.28 nan nan nan +0.64 0.28 nan nan nan +0.65 0.28 nan nan nan +0.66 0.28 nan nan nan +0.67 0.28 nan nan nan +0.68 0.28 nan nan nan +0.69 0.28 nan nan nan +0.70 0.28 nan nan nan +0.70 0.28 nan nan nan +0.71 0.28 nan nan nan +0.72 0.28 nan nan nan +0.73 0.28 nan nan nan +0.74 0.28 nan nan nan +0.76 0.28 nan nan nan +0.77 0.28 nan nan nan +0.78 0.28 nan nan nan +0.79 0.28 nan nan nan +0.80 0.28 nan nan nan +0.81 0.28 nan nan nan +0.82 0.28 nan nan nan +0.83 0.28 nan nan nan +0.83 0.28 nan nan nan +0.84 0.28 nan nan nan +0.85 0.28 nan nan nan +0.86 0.28 nan nan nan +0.88 0.28 nan nan nan +0.89 0.28 nan nan nan +0.90 0.28 nan nan nan +0.91 0.28 nan nan nan +0.92 0.28 nan nan nan +0.93 0.28 nan nan nan +0.94 0.28 nan nan nan +0.95 0.28 nan nan nan +0.96 0.28 nan nan nan +0.96 0.28 nan nan nan +0.97 0.28 nan nan nan +0.98 0.28 nan nan nan +0.01 0.29 nan nan nan +0.01 0.29 nan nan nan +0.03 0.29 nan nan nan +0.04 0.29 nan nan nan +0.04 0.29 nan nan nan +0.06 0.29 nan nan nan +0.07 0.29 nan nan nan +0.07 0.29 nan nan nan +0.09 0.29 nan nan nan +0.10 0.29 nan nan nan +0.10 0.29 nan nan nan +0.12 0.29 nan nan nan +0.12 0.29 nan nan nan +0.14 0.29 nan nan nan +0.14 0.29 nan nan nan +0.15 0.29 nan nan nan +0.17 0.29 nan nan nan +0.18 0.29 nan nan nan +0.18 0.29 nan nan nan +0.20 0.29 nan nan nan +0.21 0.29 nan nan nan +0.21 0.29 nan nan nan +0.23 0.29 nan nan nan +0.24 0.29 nan nan nan +0.24 0.29 nan nan nan +0.26 0.29 nan nan nan +0.27 0.29 nan nan nan +0.28 0.29 nan nan nan +0.29 0.29 nan nan nan +0.29 0.29 nan nan nan +0.30 0.29 nan nan nan +0.32 0.29 nan nan nan +0.33 0.29 nan nan nan +0.34 0.29 nan nan nan +0.35 0.29 nan nan nan +0.35 0.29 nan nan nan +0.36 0.29 nan nan nan +0.38 0.29 nan nan nan +0.39 0.29 nan nan nan +0.40 0.29 nan nan nan +0.41 0.29 nan nan nan +0.42 0.29 nan nan nan +0.42 0.29 nan nan nan +0.43 0.29 nan nan nan +0.45 0.29 nan nan nan +0.46 0.29 nan nan nan +0.47 0.29 nan nan nan +0.48 0.29 nan nan nan +0.48 0.29 nan nan nan +0.49 0.29 nan nan nan +0.51 0.29 nan nan nan +0.52 0.29 nan nan nan +0.53 0.29 nan nan nan +0.54 0.29 nan nan nan +0.55 0.29 nan nan nan +0.56 0.29 nan nan nan +0.57 0.29 nan nan nan +0.58 0.29 nan nan nan +0.58 0.29 nan nan nan +0.59 0.29 nan nan nan +0.60 0.29 nan nan nan +0.61 0.29 nan nan nan +0.62 0.29 nan nan nan +0.64 0.29 nan nan nan +0.65 0.29 nan nan nan +0.66 0.29 nan nan nan +0.67 0.29 nan nan nan +0.68 0.29 nan nan nan +0.69 0.29 nan nan nan +0.70 0.29 nan nan nan +0.70 0.29 nan nan nan +0.71 0.29 nan nan nan +0.72 0.29 nan nan nan +0.73 0.29 nan nan nan +0.74 0.29 nan nan nan +0.76 0.29 nan nan nan +0.77 0.29 nan nan nan +0.78 0.29 nan nan nan +0.79 0.29 nan nan nan +0.80 0.29 nan nan nan +0.81 0.29 nan nan nan +0.82 0.29 nan nan nan +0.83 0.29 nan nan nan +0.83 0.29 nan nan nan +0.84 0.29 nan nan nan +0.85 0.29 nan nan nan +0.86 0.29 nan nan nan +0.88 0.29 nan nan nan +0.89 0.29 nan nan nan +0.90 0.29 nan nan nan +0.91 0.29 nan nan nan +0.92 0.29 nan nan nan +0.93 0.29 nan nan nan +0.94 0.29 nan nan nan +0.95 0.29 nan nan nan +0.96 0.29 nan nan nan +0.96 0.29 nan nan nan +0.97 0.29 nan nan nan +0.98 0.29 nan nan nan +0.01 0.29 nan nan nan +0.01 0.29 nan nan nan +0.03 0.29 nan nan nan +0.04 0.29 nan nan nan +0.04 0.29 nan nan nan +0.06 0.29 nan nan nan +0.07 0.29 nan nan nan +0.07 0.29 nan nan nan +0.09 0.29 nan nan nan +0.10 0.29 nan nan nan +0.10 0.29 nan nan nan +0.12 0.29 nan nan nan +0.12 0.29 nan nan nan +0.14 0.29 nan nan nan +0.14 0.29 nan nan nan +0.15 0.29 nan nan nan +0.17 0.29 nan nan nan +0.18 0.29 nan nan nan +0.18 0.29 nan nan nan +0.20 0.29 nan nan nan +0.21 0.29 nan nan nan +0.21 0.29 nan nan nan +0.23 0.29 nan nan nan +0.24 0.29 nan nan nan +0.24 0.29 nan nan nan +0.26 0.29 nan nan nan +0.27 0.29 nan nan nan +0.28 0.29 nan nan nan +0.29 0.29 nan nan nan +0.29 0.29 nan nan nan +0.30 0.29 nan nan nan +0.32 0.29 nan nan nan +0.33 0.29 nan nan nan +0.34 0.29 nan nan nan +0.35 0.29 nan nan nan +0.35 0.29 nan nan nan +0.36 0.29 nan nan nan +0.38 0.29 nan nan nan +0.39 0.29 nan nan nan +0.40 0.29 nan nan nan +0.41 0.29 nan nan nan +0.42 0.29 nan nan nan +0.42 0.29 nan nan nan +0.43 0.29 nan nan nan +0.45 0.29 nan nan nan +0.46 0.29 nan nan nan +0.47 0.29 nan nan nan +0.48 0.29 nan nan nan +0.48 0.29 nan nan nan +0.49 0.29 nan nan nan +0.51 0.29 nan nan nan +0.52 0.29 nan nan nan +0.53 0.29 nan nan nan +0.54 0.29 nan nan nan +0.55 0.29 nan nan nan +0.56 0.29 nan nan nan +0.57 0.29 nan nan nan +0.58 0.29 nan nan nan +0.58 0.29 nan nan nan +0.59 0.29 nan nan nan +0.60 0.29 nan nan nan +0.61 0.29 nan nan nan +0.62 0.29 nan nan nan +0.64 0.29 nan nan nan +0.65 0.29 nan nan nan +0.66 0.29 nan nan nan +0.67 0.29 nan nan nan +0.68 0.29 nan nan nan +0.69 0.29 nan nan nan +0.70 0.29 nan nan nan +0.70 0.29 nan nan nan +0.71 0.29 nan nan nan +0.72 0.29 nan nan nan +0.73 0.29 nan nan nan +0.74 0.29 nan nan nan +0.76 0.29 nan nan nan +0.77 0.29 nan nan nan +0.78 0.29 nan nan nan +0.79 0.29 nan nan nan +0.80 0.29 nan nan nan +0.81 0.29 nan nan nan +0.82 0.29 nan nan nan +0.83 0.29 nan nan nan +0.83 0.29 nan nan nan +0.84 0.29 nan nan nan +0.85 0.29 nan nan nan +0.86 0.29 nan nan nan +0.88 0.29 nan nan nan +0.89 0.29 nan nan nan +0.90 0.29 nan nan nan +0.91 0.29 nan nan nan +0.92 0.29 nan nan nan +0.93 0.29 nan nan nan +0.94 0.29 nan nan nan +0.95 0.29 nan nan nan +0.96 0.29 nan nan nan +0.96 0.29 nan nan nan +0.97 0.29 nan nan nan +0.98 0.29 nan nan nan +0.01 0.30 nan nan nan +0.01 0.30 nan nan nan +0.03 0.30 nan nan nan +0.04 0.30 nan nan nan +0.04 0.30 nan nan nan +0.06 0.30 nan nan nan +0.07 0.30 nan nan nan +0.07 0.30 nan nan nan +0.09 0.30 nan nan nan +0.10 0.30 nan nan nan +0.10 0.30 nan nan nan +0.12 0.30 nan nan nan +0.12 0.30 nan nan nan +0.14 0.30 nan nan nan +0.14 0.30 nan nan nan +0.15 0.30 nan nan nan +0.17 0.30 nan nan nan +0.18 0.30 nan nan nan +0.18 0.30 nan nan nan +0.20 0.30 nan nan nan +0.21 0.30 nan nan nan +0.21 0.30 nan nan nan +0.23 0.30 nan nan nan +0.24 0.30 nan nan nan +0.24 0.30 nan nan nan +0.26 0.30 nan nan nan +0.27 0.30 nan nan nan +0.28 0.30 nan nan nan +0.29 0.30 nan nan nan +0.29 0.30 nan nan nan +0.30 0.30 nan nan nan +0.32 0.30 nan nan nan +0.33 0.30 nan nan nan +0.34 0.30 nan nan nan +0.35 0.30 nan nan nan +0.35 0.30 nan nan nan +0.36 0.30 nan nan nan +0.38 0.30 nan nan nan +0.39 0.30 nan nan nan +0.40 0.30 nan nan nan +0.41 0.30 nan nan nan +0.42 0.30 nan nan nan +0.42 0.30 nan nan nan +0.43 0.30 nan nan nan +0.45 0.30 nan nan nan +0.46 0.30 nan nan nan +0.47 0.30 nan nan nan +0.48 0.30 nan nan nan +0.48 0.30 nan nan nan +0.49 0.30 nan nan nan +0.51 0.30 nan nan nan +0.52 0.30 nan nan nan +0.53 0.30 nan nan nan +0.54 0.30 nan nan nan +0.55 0.30 nan nan nan +0.56 0.30 nan nan nan +0.57 0.30 nan nan nan +0.58 0.30 nan nan nan +0.58 0.30 nan nan nan +0.59 0.30 nan nan nan +0.60 0.30 nan nan nan +0.61 0.30 nan nan nan +0.62 0.30 nan nan nan +0.64 0.30 nan nan nan +0.65 0.30 nan nan nan +0.66 0.30 nan nan nan +0.67 0.30 nan nan nan +0.68 0.30 nan nan nan +0.69 0.30 nan nan nan +0.70 0.30 nan nan nan +0.70 0.30 nan nan nan +0.71 0.30 nan nan nan +0.72 0.30 nan nan nan +0.73 0.30 nan nan nan +0.74 0.30 nan nan nan +0.76 0.30 nan nan nan +0.77 0.30 nan nan nan +0.78 0.30 nan nan nan +0.79 0.30 nan nan nan +0.80 0.30 nan nan nan +0.81 0.30 nan nan nan +0.82 0.30 nan nan nan +0.83 0.30 nan nan nan +0.83 0.30 nan nan nan +0.84 0.30 nan nan nan +0.85 0.30 nan nan nan +0.86 0.30 nan nan nan +0.88 0.30 nan nan nan +0.89 0.30 nan nan nan +0.90 0.30 nan nan nan +0.91 0.30 nan nan nan +0.92 0.30 nan nan nan +0.93 0.30 nan nan nan +0.94 0.30 nan nan nan +0.95 0.30 nan nan nan +0.96 0.30 nan nan nan +0.96 0.30 nan nan nan +0.97 0.30 nan nan nan +0.98 0.30 nan nan nan +0.01 0.32 nan nan nan +0.01 0.32 nan nan nan +0.03 0.32 nan nan nan +0.04 0.32 nan nan nan +0.04 0.32 nan nan nan +0.06 0.32 nan nan nan +0.07 0.32 nan nan nan +0.07 0.32 nan nan nan +0.09 0.32 nan nan nan +0.10 0.32 nan nan nan +0.10 0.32 nan nan nan +0.12 0.32 nan nan nan +0.12 0.32 nan nan nan +0.14 0.32 nan nan nan +0.14 0.32 nan nan nan +0.15 0.32 nan nan nan +0.17 0.32 nan nan nan +0.18 0.32 nan nan nan +0.18 0.32 nan nan nan +0.20 0.32 nan nan nan +0.21 0.32 nan nan nan +0.21 0.32 nan nan nan +0.23 0.32 nan nan nan +0.24 0.32 nan nan nan +0.24 0.32 nan nan nan +0.26 0.32 nan nan nan +0.27 0.32 nan nan nan +0.28 0.32 nan nan nan +0.29 0.32 nan nan nan +0.29 0.32 nan nan nan +0.30 0.32 nan nan nan +0.32 0.32 nan nan nan +0.33 0.32 nan nan nan +0.34 0.32 nan nan nan +0.35 0.32 nan nan nan +0.35 0.32 nan nan nan +0.36 0.32 nan nan nan +0.38 0.32 nan nan nan +0.39 0.32 nan nan nan +0.40 0.32 nan nan nan +0.41 0.32 nan nan nan +0.42 0.32 nan nan nan +0.42 0.32 nan nan nan +0.43 0.32 nan nan nan +0.45 0.32 nan nan nan +0.46 0.32 nan nan nan +0.47 0.32 nan nan nan +0.48 0.32 nan nan nan +0.48 0.32 nan nan nan +0.49 0.32 nan nan nan +0.51 0.32 nan nan nan +0.52 0.32 nan nan nan +0.53 0.32 nan nan nan +0.54 0.32 nan nan nan +0.55 0.32 nan nan nan +0.56 0.32 nan nan nan +0.57 0.32 nan nan nan +0.58 0.32 nan nan nan +0.58 0.32 nan nan nan +0.59 0.32 nan nan nan +0.60 0.32 nan nan nan +0.61 0.32 nan nan nan +0.62 0.32 nan nan nan +0.64 0.32 nan nan nan +0.65 0.32 nan nan nan +0.66 0.32 nan nan nan +0.67 0.32 nan nan nan +0.68 0.32 nan nan nan +0.69 0.32 nan nan nan +0.70 0.32 nan nan nan +0.70 0.32 nan nan nan +0.71 0.32 nan nan nan +0.72 0.32 nan nan nan +0.73 0.32 nan nan nan +0.74 0.32 nan nan nan +0.76 0.32 nan nan nan +0.77 0.32 nan nan nan +0.78 0.32 nan nan nan +0.79 0.32 nan nan nan +0.80 0.32 nan nan nan +0.81 0.32 nan nan nan +0.82 0.32 nan nan nan +0.83 0.32 nan nan nan +0.83 0.32 nan nan nan +0.84 0.32 nan nan nan +0.85 0.32 nan nan nan +0.86 0.32 nan nan nan +0.88 0.32 nan nan nan +0.89 0.32 nan nan nan +0.90 0.32 nan nan nan +0.91 0.32 nan nan nan +0.92 0.32 nan nan nan +0.93 0.32 nan nan nan +0.94 0.32 nan nan nan +0.95 0.32 nan nan nan +0.96 0.32 nan nan nan +0.96 0.32 nan nan nan +0.97 0.32 nan nan nan +0.98 0.32 nan nan nan +0.01 0.33 nan nan nan +0.01 0.33 nan nan nan +0.03 0.33 nan nan nan +0.04 0.33 nan nan nan +0.04 0.33 nan nan nan +0.06 0.33 nan nan nan +0.07 0.33 nan nan nan +0.07 0.33 nan nan nan +0.09 0.33 nan nan nan +0.10 0.33 nan nan nan +0.10 0.33 nan nan nan +0.12 0.33 nan nan nan +0.12 0.33 nan nan nan +0.14 0.33 nan nan nan +0.14 0.33 nan nan nan +0.15 0.33 nan nan nan +0.17 0.33 nan nan nan +0.18 0.33 nan nan nan +0.18 0.33 nan nan nan +0.20 0.33 nan nan nan +0.21 0.33 nan nan nan +0.21 0.33 nan nan nan +0.23 0.33 nan nan nan +0.24 0.33 nan nan nan +0.24 0.33 nan nan nan +0.26 0.33 nan nan nan +0.27 0.33 nan nan nan +0.28 0.33 nan nan nan +0.29 0.33 nan nan nan +0.29 0.33 nan nan nan +0.30 0.33 nan nan nan +0.32 0.33 nan nan nan +0.33 0.33 nan nan nan +0.34 0.33 nan nan nan +0.35 0.33 nan nan nan +0.35 0.33 nan nan nan +0.36 0.33 nan nan nan +0.38 0.33 nan nan nan +0.39 0.33 nan nan nan +0.40 0.33 nan nan nan +0.41 0.33 nan nan nan +0.42 0.33 nan nan nan +0.42 0.33 nan nan nan +0.43 0.33 nan nan nan +0.45 0.33 nan nan nan +0.46 0.33 nan nan nan +0.47 0.33 nan nan nan +0.48 0.33 nan nan nan +0.48 0.33 nan nan nan +0.49 0.33 nan nan nan +0.51 0.33 nan nan nan +0.52 0.33 nan nan nan +0.53 0.33 nan nan nan +0.54 0.33 nan nan nan +0.55 0.33 nan nan nan +0.56 0.33 nan nan nan +0.57 0.33 nan nan nan +0.58 0.33 nan nan nan +0.58 0.33 nan nan nan +0.59 0.33 nan nan nan +0.60 0.33 nan nan nan +0.61 0.33 nan nan nan +0.62 0.33 nan nan nan +0.64 0.33 nan nan nan +0.65 0.33 nan nan nan +0.66 0.33 nan nan nan +0.67 0.33 nan nan nan +0.68 0.33 nan nan nan +0.69 0.33 nan nan nan +0.70 0.33 nan nan nan +0.70 0.33 nan nan nan +0.71 0.33 nan nan nan +0.72 0.33 nan nan nan +0.73 0.33 nan nan nan +0.74 0.33 nan nan nan +0.76 0.33 nan nan nan +0.77 0.33 nan nan nan +0.78 0.33 nan nan nan +0.79 0.33 nan nan nan +0.80 0.33 nan nan nan +0.81 0.33 nan nan nan +0.82 0.33 nan nan nan +0.83 0.33 nan nan nan +0.83 0.33 nan nan nan +0.84 0.33 nan nan nan +0.85 0.33 nan nan nan +0.86 0.33 nan nan nan +0.88 0.33 nan nan nan +0.89 0.33 nan nan nan +0.90 0.33 nan nan nan +0.91 0.33 nan nan nan +0.92 0.33 nan nan nan +0.93 0.33 nan nan nan +0.94 0.33 nan nan nan +0.95 0.33 nan nan nan +0.96 0.33 nan nan nan +0.96 0.33 nan nan nan +0.97 0.33 nan nan nan +0.98 0.33 nan nan nan +0.01 0.34 nan nan nan +0.01 0.34 nan nan nan +0.03 0.34 nan nan nan +0.04 0.34 nan nan nan +0.04 0.34 nan nan nan +0.06 0.34 nan nan nan +0.07 0.34 nan nan nan +0.07 0.34 nan nan nan +0.09 0.34 nan nan nan +0.10 0.34 nan nan nan +0.10 0.34 nan nan nan +0.12 0.34 nan nan nan +0.12 0.34 nan nan nan +0.14 0.34 nan nan nan +0.14 0.34 nan nan nan +0.15 0.34 nan nan nan +0.17 0.34 nan nan nan +0.18 0.34 nan nan nan +0.18 0.34 nan nan nan +0.20 0.34 nan nan nan +0.21 0.34 nan nan nan +0.21 0.34 nan nan nan +0.23 0.34 nan nan nan +0.24 0.34 nan nan nan +0.24 0.34 nan nan nan +0.26 0.34 nan nan nan +0.27 0.34 nan nan nan +0.28 0.34 nan nan nan +0.29 0.34 nan nan nan +0.29 0.34 nan nan nan +0.30 0.34 nan nan nan +0.32 0.34 nan nan nan +0.33 0.34 nan nan nan +0.34 0.34 nan nan nan +0.35 0.34 nan nan nan +0.35 0.34 nan nan nan +0.36 0.34 nan nan nan +0.38 0.34 nan nan nan +0.39 0.34 nan nan nan +0.40 0.34 nan nan nan +0.41 0.34 nan nan nan +0.42 0.34 nan nan nan +0.42 0.34 nan nan nan +0.43 0.34 nan nan nan +0.45 0.34 nan nan nan +0.46 0.34 nan nan nan +0.47 0.34 nan nan nan +0.48 0.34 nan nan nan +0.48 0.34 nan nan nan +0.49 0.34 nan nan nan +0.51 0.34 nan nan nan +0.52 0.34 nan nan nan +0.53 0.34 nan nan nan +0.54 0.34 nan nan nan +0.55 0.34 nan nan nan +0.56 0.34 nan nan nan +0.57 0.34 nan nan nan +0.58 0.34 nan nan nan +0.58 0.34 nan nan nan +0.59 0.34 nan nan nan +0.60 0.34 nan nan nan +0.61 0.34 nan nan nan +0.62 0.34 nan nan nan +0.64 0.34 nan nan nan +0.65 0.34 nan nan nan +0.66 0.34 nan nan nan +0.67 0.34 nan nan nan +0.68 0.34 nan nan nan +0.69 0.34 nan nan nan +0.70 0.34 nan nan nan +0.70 0.34 nan nan nan +0.71 0.34 nan nan nan +0.72 0.34 nan nan nan +0.73 0.34 nan nan nan +0.74 0.34 nan nan nan +0.76 0.34 nan nan nan +0.77 0.34 nan nan nan +0.78 0.34 nan nan nan +0.79 0.34 nan nan nan +0.80 0.34 nan nan nan +0.81 0.34 nan nan nan +0.82 0.34 nan nan nan +0.83 0.34 nan nan nan +0.83 0.34 nan nan nan +0.84 0.34 nan nan nan +0.85 0.34 nan nan nan +0.86 0.34 nan nan nan +0.88 0.34 nan nan nan +0.89 0.34 nan nan nan +0.90 0.34 nan nan nan +0.91 0.34 nan nan nan +0.92 0.34 nan nan nan +0.93 0.34 nan nan nan +0.94 0.34 nan nan nan +0.95 0.34 nan nan nan +0.96 0.34 nan nan nan +0.96 0.34 nan nan nan +0.97 0.34 nan nan nan +0.98 0.34 nan nan nan +0.01 0.35 nan nan nan +0.01 0.35 nan nan nan +0.03 0.35 nan nan nan +0.04 0.35 nan nan nan +0.04 0.35 nan nan nan +0.06 0.35 nan nan nan +0.07 0.35 nan nan nan +0.07 0.35 nan nan nan +0.09 0.35 nan nan nan +0.10 0.35 nan nan nan +0.10 0.35 nan nan nan +0.12 0.35 nan nan nan +0.12 0.35 nan nan nan +0.14 0.35 nan nan nan +0.14 0.35 nan nan nan +0.15 0.35 nan nan nan +0.17 0.35 nan nan nan +0.18 0.35 nan nan nan +0.18 0.35 nan nan nan +0.20 0.35 nan nan nan +0.21 0.35 nan nan nan +0.21 0.35 nan nan nan +0.23 0.35 nan nan nan +0.24 0.35 nan nan nan +0.24 0.35 nan nan nan +0.26 0.35 nan nan nan +0.27 0.35 nan nan nan +0.28 0.35 nan nan nan +0.29 0.35 nan nan nan +0.29 0.35 nan nan nan +0.30 0.35 nan nan nan +0.32 0.35 nan nan nan +0.33 0.35 nan nan nan +0.34 0.35 nan nan nan +0.35 0.35 nan nan nan +0.35 0.35 nan nan nan +0.36 0.35 nan nan nan +0.38 0.35 nan nan nan +0.39 0.35 nan nan nan +0.40 0.35 nan nan nan +0.41 0.35 nan nan nan +0.42 0.35 nan nan nan +0.42 0.35 nan nan nan +0.43 0.35 nan nan nan +0.45 0.35 nan nan nan +0.46 0.35 nan nan nan +0.47 0.35 nan nan nan +0.48 0.35 nan nan nan +0.48 0.35 nan nan nan +0.49 0.35 nan nan nan +0.51 0.35 nan nan nan +0.52 0.35 nan nan nan +0.53 0.35 nan nan nan +0.54 0.35 nan nan nan +0.55 0.35 nan nan nan +0.56 0.35 nan nan nan +0.57 0.35 nan nan nan +0.58 0.35 nan nan nan +0.58 0.35 nan nan nan +0.59 0.35 nan nan nan +0.60 0.35 nan nan nan +0.61 0.35 nan nan nan +0.62 0.35 nan nan nan +0.64 0.35 nan nan nan +0.65 0.35 nan nan nan +0.66 0.35 nan nan nan +0.67 0.35 nan nan nan +0.68 0.35 nan nan nan +0.69 0.35 nan nan nan +0.70 0.35 nan nan nan +0.70 0.35 nan nan nan +0.71 0.35 nan nan nan +0.72 0.35 nan nan nan +0.73 0.35 nan nan nan +0.74 0.35 nan nan nan +0.76 0.35 nan nan nan +0.77 0.35 nan nan nan +0.78 0.35 nan nan nan +0.79 0.35 nan nan nan +0.80 0.35 nan nan nan +0.81 0.35 nan nan nan +0.82 0.35 nan nan nan +0.83 0.35 nan nan nan +0.83 0.35 nan nan nan +0.84 0.35 nan nan nan +0.85 0.35 nan nan nan +0.86 0.35 nan nan nan +0.88 0.35 nan nan nan +0.89 0.35 nan nan nan +0.90 0.35 nan nan nan +0.91 0.35 nan nan nan +0.92 0.35 nan nan nan +0.93 0.35 nan nan nan +0.94 0.35 nan nan nan +0.95 0.35 nan nan nan +0.96 0.35 nan nan nan +0.96 0.35 nan nan nan +0.97 0.35 nan nan nan +0.98 0.35 nan nan nan +0.01 0.35 nan nan nan +0.01 0.35 nan nan nan +0.03 0.35 nan nan nan +0.04 0.35 nan nan nan +0.04 0.35 nan nan nan +0.06 0.35 nan nan nan +0.07 0.35 nan nan nan +0.07 0.35 nan nan nan +0.09 0.35 nan nan nan +0.10 0.35 nan nan nan +0.10 0.35 nan nan nan +0.12 0.35 nan nan nan +0.12 0.35 nan nan nan +0.14 0.35 nan nan nan +0.14 0.35 nan nan nan +0.15 0.35 nan nan nan +0.17 0.35 nan nan nan +0.18 0.35 nan nan nan +0.18 0.35 nan nan nan +0.20 0.35 nan nan nan +0.21 0.35 nan nan nan +0.21 0.35 nan nan nan +0.23 0.35 nan nan nan +0.24 0.35 nan nan nan +0.24 0.35 nan nan nan +0.26 0.35 nan nan nan +0.27 0.35 nan nan nan +0.28 0.35 nan nan nan +0.29 0.35 nan nan nan +0.29 0.35 nan nan nan +0.30 0.35 nan nan nan +0.32 0.35 nan nan nan +0.33 0.35 nan nan nan +0.34 0.35 nan nan nan +0.35 0.35 nan nan nan +0.35 0.35 nan nan nan +0.36 0.35 nan nan nan +0.38 0.35 nan nan nan +0.39 0.35 nan nan nan +0.40 0.35 nan nan nan +0.41 0.35 nan nan nan +0.42 0.35 nan nan nan +0.42 0.35 nan nan nan +0.43 0.35 nan nan nan +0.45 0.35 nan nan nan +0.46 0.35 nan nan nan +0.47 0.35 nan nan nan +0.48 0.35 nan nan nan +0.48 0.35 nan nan nan +0.49 0.35 nan nan nan +0.51 0.35 nan nan nan +0.52 0.35 nan nan nan +0.53 0.35 nan nan nan +0.54 0.35 nan nan nan +0.55 0.35 nan nan nan +0.56 0.35 nan nan nan +0.57 0.35 nan nan nan +0.58 0.35 nan nan nan +0.58 0.35 nan nan nan +0.59 0.35 nan nan nan +0.60 0.35 nan nan nan +0.61 0.35 nan nan nan +0.62 0.35 nan nan nan +0.64 0.35 nan nan nan +0.65 0.35 nan nan nan +0.66 0.35 nan nan nan +0.67 0.35 nan nan nan +0.68 0.35 nan nan nan +0.69 0.35 nan nan nan +0.70 0.35 nan nan nan +0.70 0.35 nan nan nan +0.71 0.35 nan nan nan +0.72 0.35 nan nan nan +0.73 0.35 nan nan nan +0.74 0.35 nan nan nan +0.76 0.35 nan nan nan +0.77 0.35 nan nan nan +0.78 0.35 nan nan nan +0.79 0.35 nan nan nan +0.80 0.35 nan nan nan +0.81 0.35 nan nan nan +0.82 0.35 nan nan nan +0.83 0.35 nan nan nan +0.83 0.35 nan nan nan +0.84 0.35 nan nan nan +0.85 0.35 nan nan nan +0.86 0.35 nan nan nan +0.88 0.35 nan nan nan +0.89 0.35 nan nan nan +0.90 0.35 nan nan nan +0.91 0.35 nan nan nan +0.92 0.35 nan nan nan +0.93 0.35 nan nan nan +0.94 0.35 nan nan nan +0.95 0.35 nan nan nan +0.96 0.35 nan nan nan +0.96 0.35 nan nan nan +0.97 0.35 nan nan nan +0.98 0.35 nan nan nan +0.01 0.36 nan nan nan +0.01 0.36 nan nan nan +0.03 0.36 nan nan nan +0.04 0.36 nan nan nan +0.04 0.36 nan nan nan +0.06 0.36 nan nan nan +0.07 0.36 nan nan nan +0.07 0.36 nan nan nan +0.09 0.36 nan nan nan +0.10 0.36 nan nan nan +0.10 0.36 nan nan nan +0.12 0.36 nan nan nan +0.12 0.36 nan nan nan +0.14 0.36 nan nan nan +0.14 0.36 nan nan nan +0.15 0.36 nan nan nan +0.17 0.36 nan nan nan +0.18 0.36 nan nan nan +0.18 0.36 nan nan nan +0.20 0.36 nan nan nan +0.21 0.36 nan nan nan +0.21 0.36 nan nan nan +0.23 0.36 nan nan nan +0.24 0.36 nan nan nan +0.24 0.36 nan nan nan +0.26 0.36 nan nan nan +0.27 0.36 nan nan nan +0.28 0.36 nan nan nan +0.29 0.36 nan nan nan +0.29 0.36 nan nan nan +0.30 0.36 nan nan nan +0.32 0.36 nan nan nan +0.33 0.36 nan nan nan +0.34 0.36 nan nan nan +0.35 0.36 nan nan nan +0.35 0.36 nan nan nan +0.36 0.36 nan nan nan +0.38 0.36 nan nan nan +0.39 0.36 nan nan nan +0.40 0.36 nan nan nan +0.41 0.36 nan nan nan +0.42 0.36 nan nan nan +0.42 0.36 nan nan nan +0.43 0.36 nan nan nan +0.45 0.36 nan nan nan +0.46 0.36 nan nan nan +0.47 0.36 nan nan nan +0.48 0.36 nan nan nan +0.48 0.36 nan nan nan +0.49 0.36 nan nan nan +0.51 0.36 nan nan nan +0.52 0.36 nan nan nan +0.53 0.36 nan nan nan +0.54 0.36 nan nan nan +0.55 0.36 nan nan nan +0.56 0.36 nan nan nan +0.57 0.36 nan nan nan +0.58 0.36 nan nan nan +0.58 0.36 nan nan nan +0.59 0.36 nan nan nan +0.60 0.36 nan nan nan +0.61 0.36 nan nan nan +0.62 0.36 nan nan nan +0.64 0.36 nan nan nan +0.65 0.36 nan nan nan +0.66 0.36 nan nan nan +0.67 0.36 nan nan nan +0.68 0.36 nan nan nan +0.69 0.36 nan nan nan +0.70 0.36 nan nan nan +0.70 0.36 nan nan nan +0.71 0.36 nan nan nan +0.72 0.36 nan nan nan +0.73 0.36 nan nan nan +0.74 0.36 nan nan nan +0.76 0.36 nan nan nan +0.77 0.36 nan nan nan +0.78 0.36 nan nan nan +0.79 0.36 nan nan nan +0.80 0.36 nan nan nan +0.81 0.36 nan nan nan +0.82 0.36 nan nan nan +0.83 0.36 nan nan nan +0.83 0.36 nan nan nan +0.84 0.36 nan nan nan +0.85 0.36 nan nan nan +0.86 0.36 nan nan nan +0.88 0.36 nan nan nan +0.89 0.36 nan nan nan +0.90 0.36 nan nan nan +0.91 0.36 nan nan nan +0.92 0.36 nan nan nan +0.93 0.36 nan nan nan +0.94 0.36 nan nan nan +0.95 0.36 nan nan nan +0.96 0.36 nan nan nan +0.96 0.36 nan nan nan +0.97 0.36 nan nan nan +0.98 0.36 nan nan nan +0.01 0.38 nan nan nan +0.01 0.38 nan nan nan +0.03 0.38 nan nan nan +0.04 0.38 nan nan nan +0.04 0.38 nan nan nan +0.06 0.38 nan nan nan +0.07 0.38 nan nan nan +0.07 0.38 nan nan nan +0.09 0.38 nan nan nan +0.10 0.38 nan nan nan +0.10 0.38 nan nan nan +0.12 0.38 nan nan nan +0.12 0.38 nan nan nan +0.14 0.38 nan nan nan +0.14 0.38 nan nan nan +0.15 0.38 nan nan nan +0.17 0.38 nan nan nan +0.18 0.38 nan nan nan +0.18 0.38 nan nan nan +0.20 0.38 nan nan nan +0.21 0.38 nan nan nan +0.21 0.38 nan nan nan +0.23 0.38 nan nan nan +0.24 0.38 nan nan nan +0.24 0.38 nan nan nan +0.26 0.38 nan nan nan +0.27 0.38 nan nan nan +0.28 0.38 nan nan nan +0.29 0.38 nan nan nan +0.29 0.38 nan nan nan +0.30 0.38 nan nan nan +0.32 0.38 nan nan nan +0.33 0.38 nan nan nan +0.34 0.38 nan nan nan +0.35 0.38 nan nan nan +0.35 0.38 nan nan nan +0.36 0.38 nan nan nan +0.38 0.38 nan nan nan +0.39 0.38 nan nan nan +0.40 0.38 nan nan nan +0.41 0.38 nan nan nan +0.42 0.38 nan nan nan +0.42 0.38 nan nan nan +0.43 0.38 nan nan nan +0.45 0.38 nan nan nan +0.46 0.38 nan nan nan +0.47 0.38 nan nan nan +0.48 0.38 nan nan nan +0.48 0.38 nan nan nan +0.49 0.38 nan nan nan +0.51 0.38 nan nan nan +0.52 0.38 nan nan nan +0.53 0.38 nan nan nan +0.54 0.38 nan nan nan +0.55 0.38 nan nan nan +0.56 0.38 nan nan nan +0.57 0.38 nan nan nan +0.58 0.38 nan nan nan +0.58 0.38 nan nan nan +0.59 0.38 nan nan nan +0.60 0.38 nan nan nan +0.61 0.38 nan nan nan +0.62 0.38 nan nan nan +0.64 0.38 nan nan nan +0.65 0.38 nan nan nan +0.66 0.38 nan nan nan +0.67 0.38 nan nan nan +0.68 0.38 nan nan nan +0.69 0.38 nan nan nan +0.70 0.38 nan nan nan +0.70 0.38 nan nan nan +0.71 0.38 nan nan nan +0.72 0.38 nan nan nan +0.73 0.38 nan nan nan +0.74 0.38 nan nan nan +0.76 0.38 nan nan nan +0.77 0.38 nan nan nan +0.78 0.38 nan nan nan +0.79 0.38 nan nan nan +0.80 0.38 nan nan nan +0.81 0.38 nan nan nan +0.82 0.38 nan nan nan +0.83 0.38 nan nan nan +0.83 0.38 nan nan nan +0.84 0.38 nan nan nan +0.85 0.38 nan nan nan +0.86 0.38 nan nan nan +0.88 0.38 nan nan nan +0.89 0.38 nan nan nan +0.90 0.38 nan nan nan +0.91 0.38 nan nan nan +0.92 0.38 nan nan nan +0.93 0.38 nan nan nan +0.94 0.38 nan nan nan +0.95 0.38 nan nan nan +0.96 0.38 nan nan nan +0.96 0.38 nan nan nan +0.97 0.38 nan nan nan +0.98 0.38 nan nan nan +0.01 0.39 nan nan nan +0.01 0.39 nan nan nan +0.03 0.39 nan nan nan +0.04 0.39 nan nan nan +0.04 0.39 nan nan nan +0.06 0.39 nan nan nan +0.07 0.39 nan nan nan +0.07 0.39 nan nan nan +0.09 0.39 nan nan nan +0.10 0.39 nan nan nan +0.10 0.39 nan nan nan +0.12 0.39 nan nan nan +0.12 0.39 nan nan nan +0.14 0.39 nan nan nan +0.14 0.39 nan nan nan +0.15 0.39 nan nan nan +0.17 0.39 nan nan nan +0.18 0.39 nan nan nan +0.18 0.39 nan nan nan +0.20 0.39 nan nan nan +0.21 0.39 nan nan nan +0.21 0.39 nan nan nan +0.23 0.39 nan nan nan +0.24 0.39 nan nan nan +0.24 0.39 nan nan nan +0.26 0.39 nan nan nan +0.27 0.39 nan nan nan +0.28 0.39 nan nan nan +0.29 0.39 nan nan nan +0.29 0.39 nan nan nan +0.30 0.39 nan nan nan +0.32 0.39 nan nan nan +0.33 0.39 nan nan nan +0.34 0.39 nan nan nan +0.35 0.39 nan nan nan +0.35 0.39 nan nan nan +0.36 0.39 nan nan nan +0.38 0.39 nan nan nan +0.39 0.39 nan nan nan +0.40 0.39 nan nan nan +0.41 0.39 nan nan nan +0.42 0.39 nan nan nan +0.42 0.39 nan nan nan +0.43 0.39 nan nan nan +0.45 0.39 nan nan nan +0.46 0.39 nan nan nan +0.47 0.39 nan nan nan +0.48 0.39 nan nan nan +0.48 0.39 nan nan nan +0.49 0.39 nan nan nan +0.51 0.39 nan nan nan +0.52 0.39 nan nan nan +0.53 0.39 nan nan nan +0.54 0.39 nan nan nan +0.55 0.39 nan nan nan +0.56 0.39 nan nan nan +0.57 0.39 nan nan nan +0.58 0.39 nan nan nan +0.58 0.39 nan nan nan +0.59 0.39 nan nan nan +0.60 0.39 nan nan nan +0.61 0.39 nan nan nan +0.62 0.39 nan nan nan +0.64 0.39 nan nan nan +0.65 0.39 nan nan nan +0.66 0.39 nan nan nan +0.67 0.39 nan nan nan +0.68 0.39 nan nan nan +0.69 0.39 nan nan nan +0.70 0.39 nan nan nan +0.70 0.39 nan nan nan +0.71 0.39 nan nan nan +0.72 0.39 nan nan nan +0.73 0.39 nan nan nan +0.74 0.39 nan nan nan +0.76 0.39 nan nan nan +0.77 0.39 nan nan nan +0.78 0.39 nan nan nan +0.79 0.39 nan nan nan +0.80 0.39 nan nan nan +0.81 0.39 nan nan nan +0.82 0.39 nan nan nan +0.83 0.39 nan nan nan +0.83 0.39 nan nan nan +0.84 0.39 nan nan nan +0.85 0.39 nan nan nan +0.86 0.39 nan nan nan +0.88 0.39 nan nan nan +0.89 0.39 nan nan nan +0.90 0.39 nan nan nan +0.91 0.39 nan nan nan +0.92 0.39 nan nan nan +0.93 0.39 nan nan nan +0.94 0.39 nan nan nan +0.95 0.39 nan nan nan +0.96 0.39 nan nan nan +0.96 0.39 nan nan nan +0.97 0.39 nan nan nan +0.98 0.39 nan nan nan +0.01 0.40 nan nan nan +0.01 0.40 nan nan nan +0.03 0.40 nan nan nan +0.04 0.40 nan nan nan +0.04 0.40 nan nan nan +0.06 0.40 nan nan nan +0.07 0.40 nan nan nan +0.07 0.40 nan nan nan +0.09 0.40 nan nan nan +0.10 0.40 nan nan nan +0.10 0.40 nan nan nan +0.12 0.40 nan nan nan +0.12 0.40 nan nan nan +0.14 0.40 nan nan nan +0.14 0.40 nan nan nan +0.15 0.40 nan nan nan +0.17 0.40 nan nan nan +0.18 0.40 nan nan nan +0.18 0.40 nan nan nan +0.20 0.40 nan nan nan +0.21 0.40 nan nan nan +0.21 0.40 nan nan nan +0.23 0.40 nan nan nan +0.24 0.40 nan nan nan +0.24 0.40 nan nan nan +0.26 0.40 nan nan nan +0.27 0.40 nan nan nan +0.28 0.40 nan nan nan +0.29 0.40 nan nan nan +0.29 0.40 nan nan nan +0.30 0.40 nan nan nan +0.32 0.40 nan nan nan +0.33 0.40 nan nan nan +0.34 0.40 nan nan nan +0.35 0.40 nan nan nan +0.35 0.40 nan nan nan +0.36 0.40 nan nan nan +0.38 0.40 nan nan nan +0.39 0.40 nan nan nan +0.40 0.40 nan nan nan +0.41 0.40 nan nan nan +0.42 0.40 nan nan nan +0.42 0.40 nan nan nan +0.43 0.40 nan nan nan +0.45 0.40 nan nan nan +0.46 0.40 nan nan nan +0.47 0.40 nan nan nan +0.48 0.40 nan nan nan +0.48 0.40 nan nan nan +0.49 0.40 nan nan nan +0.51 0.40 nan nan nan +0.52 0.40 nan nan nan +0.53 0.40 nan nan nan +0.54 0.40 nan nan nan +0.55 0.40 nan nan nan +0.56 0.40 nan nan nan +0.57 0.40 nan nan nan +0.58 0.40 nan nan nan +0.58 0.40 nan nan nan +0.59 0.40 nan nan nan +0.60 0.40 nan nan nan +0.61 0.40 nan nan nan +0.62 0.40 nan nan nan +0.64 0.40 nan nan nan +0.65 0.40 nan nan nan +0.66 0.40 nan nan nan +0.67 0.40 nan nan nan +0.68 0.40 nan nan nan +0.69 0.40 nan nan nan +0.70 0.40 nan nan nan +0.70 0.40 nan nan nan +0.71 0.40 nan nan nan +0.72 0.40 nan nan nan +0.73 0.40 nan nan nan +0.74 0.40 nan nan nan +0.76 0.40 nan nan nan +0.77 0.40 nan nan nan +0.78 0.40 nan nan nan +0.79 0.40 nan nan nan +0.80 0.40 nan nan nan +0.81 0.40 nan nan nan +0.82 0.40 nan nan nan +0.83 0.40 nan nan nan +0.83 0.40 nan nan nan +0.84 0.40 nan nan nan +0.85 0.40 nan nan nan +0.86 0.40 nan nan nan +0.88 0.40 nan nan nan +0.89 0.40 nan nan nan +0.90 0.40 nan nan nan +0.91 0.40 nan nan nan +0.92 0.40 nan nan nan +0.93 0.40 nan nan nan +0.94 0.40 nan nan nan +0.95 0.40 nan nan nan +0.96 0.40 nan nan nan +0.96 0.40 nan nan nan +0.97 0.40 nan nan nan +0.98 0.40 nan nan nan +0.01 0.41 nan nan nan +0.01 0.41 nan nan nan +0.03 0.41 nan nan nan +0.04 0.41 nan nan nan +0.04 0.41 nan nan nan +0.06 0.41 nan nan nan +0.07 0.41 nan nan nan +0.07 0.41 nan nan nan +0.09 0.41 nan nan nan +0.10 0.41 nan nan nan +0.10 0.41 nan nan nan +0.12 0.41 nan nan nan +0.12 0.41 nan nan nan +0.14 0.41 nan nan nan +0.14 0.41 nan nan nan +0.15 0.41 nan nan nan +0.17 0.41 nan nan nan +0.18 0.41 nan nan nan +0.18 0.41 nan nan nan +0.20 0.41 nan nan nan +0.21 0.41 nan nan nan +0.21 0.41 nan nan nan +0.23 0.41 nan nan nan +0.24 0.41 nan nan nan +0.24 0.41 nan nan nan +0.26 0.41 nan nan nan +0.27 0.41 nan nan nan +0.28 0.41 nan nan nan +0.29 0.41 nan nan nan +0.29 0.41 nan nan nan +0.30 0.41 nan nan nan +0.32 0.41 nan nan nan +0.33 0.41 nan nan nan +0.34 0.41 nan nan nan +0.35 0.41 nan nan nan +0.35 0.41 nan nan nan +0.36 0.41 nan nan nan +0.38 0.41 nan nan nan +0.39 0.41 nan nan nan +0.40 0.41 nan nan nan +0.41 0.41 nan nan nan +0.42 0.41 nan nan nan +0.42 0.41 nan nan nan +0.43 0.41 nan nan nan +0.45 0.41 nan nan nan +0.46 0.41 nan nan nan +0.47 0.41 nan nan nan +0.48 0.41 nan nan nan +0.48 0.41 nan nan nan +0.49 0.41 nan nan nan +0.51 0.41 nan nan nan +0.52 0.41 nan nan nan +0.53 0.41 nan nan nan +0.54 0.41 nan nan nan +0.55 0.41 nan nan nan +0.56 0.41 nan nan nan +0.57 0.41 nan nan nan +0.58 0.41 nan nan nan +0.58 0.41 nan nan nan +0.59 0.41 nan nan nan +0.60 0.41 nan nan nan +0.61 0.41 nan nan nan +0.62 0.41 nan nan nan +0.64 0.41 nan nan nan +0.65 0.41 nan nan nan +0.66 0.41 nan nan nan +0.67 0.41 nan nan nan +0.68 0.41 nan nan nan +0.69 0.41 nan nan nan +0.70 0.41 nan nan nan +0.70 0.41 nan nan nan +0.71 0.41 nan nan nan +0.72 0.41 nan nan nan +0.73 0.41 nan nan nan +0.74 0.41 nan nan nan +0.76 0.41 nan nan nan +0.77 0.41 nan nan nan +0.78 0.41 nan nan nan +0.79 0.41 nan nan nan +0.80 0.41 nan nan nan +0.81 0.41 nan nan nan +0.82 0.41 nan nan nan +0.83 0.41 nan nan nan +0.83 0.41 nan nan nan +0.84 0.41 nan nan nan +0.85 0.41 nan nan nan +0.86 0.41 nan nan nan +0.88 0.41 nan nan nan +0.89 0.41 nan nan nan +0.90 0.41 nan nan nan +0.91 0.41 nan nan nan +0.92 0.41 nan nan nan +0.93 0.41 nan nan nan +0.94 0.41 nan nan nan +0.95 0.41 nan nan nan +0.96 0.41 nan nan nan +0.96 0.41 nan nan nan +0.97 0.41 nan nan nan +0.98 0.41 nan nan nan +0.01 0.42 nan nan nan +0.01 0.42 nan nan nan +0.03 0.42 nan nan nan +0.04 0.42 nan nan nan +0.04 0.42 nan nan nan +0.06 0.42 nan nan nan +0.07 0.42 nan nan nan +0.07 0.42 nan nan nan +0.09 0.42 nan nan nan +0.10 0.42 nan nan nan +0.10 0.42 nan nan nan +0.12 0.42 nan nan nan +0.12 0.42 nan nan nan +0.14 0.42 nan nan nan +0.14 0.42 nan nan nan +0.15 0.42 nan nan nan +0.17 0.42 nan nan nan +0.18 0.42 nan nan nan +0.18 0.42 nan nan nan +0.20 0.42 nan nan nan +0.21 0.42 nan nan nan +0.21 0.42 nan nan nan +0.23 0.42 nan nan nan +0.24 0.42 nan nan nan +0.24 0.42 nan nan nan +0.26 0.42 nan nan nan +0.27 0.42 nan nan nan +0.28 0.42 nan nan nan +0.29 0.42 nan nan nan +0.29 0.42 nan nan nan +0.30 0.42 nan nan nan +0.32 0.42 nan nan nan +0.33 0.42 nan nan nan +0.34 0.42 nan nan nan +0.35 0.42 nan nan nan +0.35 0.42 nan nan nan +0.36 0.42 nan nan nan +0.38 0.42 nan nan nan +0.39 0.42 nan nan nan +0.40 0.42 nan nan nan +0.41 0.42 nan nan nan +0.42 0.42 nan nan nan +0.42 0.42 nan nan nan +0.43 0.42 nan nan nan +0.45 0.42 nan nan nan +0.46 0.42 nan nan nan +0.47 0.42 nan nan nan +0.48 0.42 nan nan nan +0.48 0.42 nan nan nan +0.49 0.42 nan nan nan +0.51 0.42 nan nan nan +0.52 0.42 nan nan nan +0.53 0.42 nan nan nan +0.54 0.42 nan nan nan +0.55 0.42 nan nan nan +0.56 0.42 nan nan nan +0.57 0.42 nan nan nan +0.58 0.42 nan nan nan +0.58 0.42 nan nan nan +0.59 0.42 nan nan nan +0.60 0.42 nan nan nan +0.61 0.42 nan nan nan +0.62 0.42 nan nan nan +0.64 0.42 nan nan nan +0.65 0.42 nan nan nan +0.66 0.42 nan nan nan +0.67 0.42 nan nan nan +0.68 0.42 nan nan nan +0.69 0.42 nan nan nan +0.70 0.42 nan nan nan +0.70 0.42 nan nan nan +0.71 0.42 nan nan nan +0.72 0.42 nan nan nan +0.73 0.42 nan nan nan +0.74 0.42 nan nan nan +0.76 0.42 nan nan nan +0.77 0.42 nan nan nan +0.78 0.42 nan nan nan +0.79 0.42 nan nan nan +0.80 0.42 nan nan nan +0.81 0.42 nan nan nan +0.82 0.42 nan nan nan +0.83 0.42 nan nan nan +0.83 0.42 nan nan nan +0.84 0.42 nan nan nan +0.85 0.42 nan nan nan +0.86 0.42 nan nan nan +0.88 0.42 nan nan nan +0.89 0.42 nan nan nan +0.90 0.42 nan nan nan +0.91 0.42 nan nan nan +0.92 0.42 nan nan nan +0.93 0.42 nan nan nan +0.94 0.42 nan nan nan +0.95 0.42 nan nan nan +0.96 0.42 nan nan nan +0.96 0.42 nan nan nan +0.97 0.42 nan nan nan +0.98 0.42 nan nan nan +0.01 0.42 nan nan nan +0.01 0.42 nan nan nan +0.03 0.42 nan nan nan +0.04 0.42 nan nan nan +0.04 0.42 nan nan nan +0.06 0.42 nan nan nan +0.07 0.42 nan nan nan +0.07 0.42 nan nan nan +0.09 0.42 nan nan nan +0.10 0.42 nan nan nan +0.10 0.42 nan nan nan +0.12 0.42 nan nan nan +0.12 0.42 nan nan nan +0.14 0.42 nan nan nan +0.14 0.42 nan nan nan +0.15 0.42 nan nan nan +0.17 0.42 nan nan nan +0.18 0.42 nan nan nan +0.18 0.42 nan nan nan +0.20 0.42 nan nan nan +0.21 0.42 nan nan nan +0.21 0.42 nan nan nan +0.23 0.42 nan nan nan +0.24 0.42 nan nan nan +0.24 0.42 nan nan nan +0.26 0.42 nan nan nan +0.27 0.42 nan nan nan +0.28 0.42 nan nan nan +0.29 0.42 nan nan nan +0.29 0.42 nan nan nan +0.30 0.42 nan nan nan +0.32 0.42 nan nan nan +0.33 0.42 nan nan nan +0.34 0.42 nan nan nan +0.35 0.42 nan nan nan +0.35 0.42 nan nan nan +0.36 0.42 nan nan nan +0.38 0.42 nan nan nan +0.39 0.42 nan nan nan +0.40 0.42 nan nan nan +0.41 0.42 nan nan nan +0.42 0.42 nan nan nan +0.42 0.42 nan nan nan +0.43 0.42 nan nan nan +0.45 0.42 nan nan nan +0.46 0.42 nan nan nan +0.47 0.42 nan nan nan +0.48 0.42 nan nan nan +0.48 0.42 nan nan nan +0.49 0.42 nan nan nan +0.51 0.42 nan nan nan +0.52 0.42 nan nan nan +0.53 0.42 nan nan nan +0.54 0.42 nan nan nan +0.55 0.42 nan nan nan +0.56 0.42 nan nan nan +0.57 0.42 nan nan nan +0.58 0.42 nan nan nan +0.58 0.42 nan nan nan +0.59 0.42 nan nan nan +0.60 0.42 nan nan nan +0.61 0.42 nan nan nan +0.62 0.42 nan nan nan +0.64 0.42 nan nan nan +0.65 0.42 nan nan nan +0.66 0.42 nan nan nan +0.67 0.42 nan nan nan +0.68 0.42 nan nan nan +0.69 0.42 nan nan nan +0.70 0.42 nan nan nan +0.70 0.42 nan nan nan +0.71 0.42 nan nan nan +0.72 0.42 nan nan nan +0.73 0.42 nan nan nan +0.74 0.42 nan nan nan +0.76 0.42 nan nan nan +0.77 0.42 nan nan nan +0.78 0.42 nan nan nan +0.79 0.42 nan nan nan +0.80 0.42 nan nan nan +0.81 0.42 nan nan nan +0.82 0.42 nan nan nan +0.83 0.42 nan nan nan +0.83 0.42 nan nan nan +0.84 0.42 nan nan nan +0.85 0.42 nan nan nan +0.86 0.42 nan nan nan +0.88 0.42 nan nan nan +0.89 0.42 nan nan nan +0.90 0.42 nan nan nan +0.91 0.42 nan nan nan +0.92 0.42 nan nan nan +0.93 0.42 nan nan nan +0.94 0.42 nan nan nan +0.95 0.42 nan nan nan +0.96 0.42 nan nan nan +0.96 0.42 nan nan nan +0.97 0.42 nan nan nan +0.98 0.42 nan nan nan +0.01 0.43 nan nan nan +0.01 0.43 nan nan nan +0.03 0.43 nan nan nan +0.04 0.43 nan nan nan +0.04 0.43 nan nan nan +0.06 0.43 nan nan nan +0.07 0.43 nan nan nan +0.07 0.43 nan nan nan +0.09 0.43 nan nan nan +0.10 0.43 nan nan nan +0.10 0.43 nan nan nan +0.12 0.43 nan nan nan +0.12 0.43 nan nan nan +0.14 0.43 nan nan nan +0.14 0.43 nan nan nan +0.15 0.43 nan nan nan +0.17 0.43 nan nan nan +0.18 0.43 nan nan nan +0.18 0.43 nan nan nan +0.20 0.43 nan nan nan +0.21 0.43 nan nan nan +0.21 0.43 nan nan nan +0.23 0.43 nan nan nan +0.24 0.43 nan nan nan +0.24 0.43 nan nan nan +0.26 0.43 nan nan nan +0.27 0.43 nan nan nan +0.28 0.43 nan nan nan +0.29 0.43 nan nan nan +0.29 0.43 nan nan nan +0.30 0.43 nan nan nan +0.32 0.43 nan nan nan +0.33 0.43 nan nan nan +0.34 0.43 nan nan nan +0.35 0.43 nan nan nan +0.35 0.43 nan nan nan +0.36 0.43 nan nan nan +0.38 0.43 nan nan nan +0.39 0.43 nan nan nan +0.40 0.43 nan nan nan +0.41 0.43 nan nan nan +0.42 0.43 nan nan nan +0.42 0.43 nan nan nan +0.43 0.43 nan nan nan +0.45 0.43 nan nan nan +0.46 0.43 nan nan nan +0.47 0.43 nan nan nan +0.48 0.43 nan nan nan +0.48 0.43 nan nan nan +0.49 0.43 nan nan nan +0.51 0.43 nan nan nan +0.52 0.43 nan nan nan +0.53 0.43 nan nan nan +0.54 0.43 nan nan nan +0.55 0.43 nan nan nan +0.56 0.43 nan nan nan +0.57 0.43 nan nan nan +0.58 0.43 nan nan nan +0.58 0.43 nan nan nan +0.59 0.43 nan nan nan +0.60 0.43 nan nan nan +0.61 0.43 nan nan nan +0.62 0.43 nan nan nan +0.64 0.43 nan nan nan +0.65 0.43 nan nan nan +0.66 0.43 nan nan nan +0.67 0.43 nan nan nan +0.68 0.43 nan nan nan +0.69 0.43 nan nan nan +0.70 0.43 nan nan nan +0.70 0.43 nan nan nan +0.71 0.43 nan nan nan +0.72 0.43 nan nan nan +0.73 0.43 nan nan nan +0.74 0.43 nan nan nan +0.76 0.43 nan nan nan +0.77 0.43 nan nan nan +0.78 0.43 nan nan nan +0.79 0.43 nan nan nan +0.80 0.43 nan nan nan +0.81 0.43 nan nan nan +0.82 0.43 nan nan nan +0.83 0.43 nan nan nan +0.83 0.43 nan nan nan +0.84 0.43 nan nan nan +0.85 0.43 nan nan nan +0.86 0.43 nan nan nan +0.88 0.43 nan nan nan +0.89 0.43 nan nan nan +0.90 0.43 nan nan nan +0.91 0.43 nan nan nan +0.92 0.43 nan nan nan +0.93 0.43 nan nan nan +0.94 0.43 nan nan nan +0.95 0.43 nan nan nan +0.96 0.43 nan nan nan +0.96 0.43 nan nan nan +0.97 0.43 nan nan nan +0.98 0.43 nan nan nan +0.01 0.45 nan nan nan +0.01 0.45 nan nan nan +0.03 0.45 nan nan nan +0.04 0.45 nan nan nan +0.04 0.45 nan nan nan +0.06 0.45 nan nan nan +0.07 0.45 nan nan nan +0.07 0.45 nan nan nan +0.09 0.45 nan nan nan +0.10 0.45 nan nan nan +0.10 0.45 nan nan nan +0.12 0.45 nan nan nan +0.12 0.45 nan nan nan +0.14 0.45 nan nan nan +0.14 0.45 nan nan nan +0.15 0.45 nan nan nan +0.17 0.45 nan nan nan +0.18 0.45 nan nan nan +0.18 0.45 nan nan nan +0.20 0.45 nan nan nan +0.21 0.45 nan nan nan +0.21 0.45 nan nan nan +0.23 0.45 nan nan nan +0.24 0.45 nan nan nan +0.24 0.45 nan nan nan +0.26 0.45 nan nan nan +0.27 0.45 nan nan nan +0.28 0.45 nan nan nan +0.29 0.45 nan nan nan +0.29 0.45 nan nan nan +0.30 0.45 nan nan nan +0.32 0.45 nan nan nan +0.33 0.45 nan nan nan +0.34 0.45 nan nan nan +0.35 0.45 nan nan nan +0.35 0.45 nan nan nan +0.36 0.45 nan nan nan +0.38 0.45 nan nan nan +0.39 0.45 nan nan nan +0.40 0.45 nan nan nan +0.41 0.45 nan nan nan +0.42 0.45 nan nan nan +0.42 0.45 nan nan nan +0.43 0.45 nan nan nan +0.45 0.45 nan nan nan +0.46 0.45 nan nan nan +0.47 0.45 nan nan nan +0.48 0.45 nan nan nan +0.48 0.45 nan nan nan +0.49 0.45 nan nan nan +0.51 0.45 nan nan nan +0.52 0.45 nan nan nan +0.53 0.45 nan nan nan +0.54 0.45 nan nan nan +0.55 0.45 nan nan nan +0.56 0.45 nan nan nan +0.57 0.45 nan nan nan +0.58 0.45 nan nan nan +0.58 0.45 nan nan nan +0.59 0.45 nan nan nan +0.60 0.45 nan nan nan +0.61 0.45 nan nan nan +0.62 0.45 nan nan nan +0.64 0.45 nan nan nan +0.65 0.45 nan nan nan +0.66 0.45 nan nan nan +0.67 0.45 nan nan nan +0.68 0.45 nan nan nan +0.69 0.45 nan nan nan +0.70 0.45 nan nan nan +0.70 0.45 nan nan nan +0.71 0.45 nan nan nan +0.72 0.45 nan nan nan +0.73 0.45 nan nan nan +0.74 0.45 nan nan nan +0.76 0.45 nan nan nan +0.77 0.45 nan nan nan +0.78 0.45 nan nan nan +0.79 0.45 nan nan nan +0.80 0.45 nan nan nan +0.81 0.45 nan nan nan +0.82 0.45 nan nan nan +0.83 0.45 nan nan nan +0.83 0.45 nan nan nan +0.84 0.45 nan nan nan +0.85 0.45 nan nan nan +0.86 0.45 nan nan nan +0.88 0.45 nan nan nan +0.89 0.45 nan nan nan +0.90 0.45 nan nan nan +0.91 0.45 nan nan nan +0.92 0.45 nan nan nan +0.93 0.45 nan nan nan +0.94 0.45 nan nan nan +0.95 0.45 nan nan nan +0.96 0.45 nan nan nan +0.96 0.45 nan nan nan +0.97 0.45 nan nan nan +0.98 0.45 nan nan nan +0.01 0.46 nan nan nan +0.01 0.46 nan nan nan +0.03 0.46 nan nan nan +0.04 0.46 nan nan nan +0.04 0.46 nan nan nan +0.06 0.46 nan nan nan +0.07 0.46 nan nan nan +0.07 0.46 nan nan nan +0.09 0.46 nan nan nan +0.10 0.46 nan nan nan +0.10 0.46 nan nan nan +0.12 0.46 nan nan nan +0.12 0.46 nan nan nan +0.14 0.46 nan nan nan +0.14 0.46 nan nan nan +0.15 0.46 nan nan nan +0.17 0.46 nan nan nan +0.18 0.46 nan nan nan +0.18 0.46 nan nan nan +0.20 0.46 nan nan nan +0.21 0.46 nan nan nan +0.21 0.46 nan nan nan +0.23 0.46 nan nan nan +0.24 0.46 nan nan nan +0.24 0.46 nan nan nan +0.26 0.46 nan nan nan +0.27 0.46 nan nan nan +0.28 0.46 nan nan nan +0.29 0.46 nan nan nan +0.29 0.46 nan nan nan +0.30 0.46 nan nan nan +0.32 0.46 nan nan nan +0.33 0.46 nan nan nan +0.34 0.46 nan nan nan +0.35 0.46 nan nan nan +0.35 0.46 nan nan nan +0.36 0.46 nan nan nan +0.38 0.46 nan nan nan +0.39 0.46 nan nan nan +0.40 0.46 nan nan nan +0.41 0.46 nan nan nan +0.42 0.46 nan nan nan +0.42 0.46 nan nan nan +0.43 0.46 nan nan nan +0.45 0.46 nan nan nan +0.46 0.46 nan nan nan +0.47 0.46 nan nan nan +0.48 0.46 nan nan nan +0.48 0.46 nan nan nan +0.49 0.46 nan nan nan +0.51 0.46 nan nan nan +0.52 0.46 nan nan nan +0.53 0.46 nan nan nan +0.54 0.46 nan nan nan +0.55 0.46 nan nan nan +0.56 0.46 nan nan nan +0.57 0.46 nan nan nan +0.58 0.46 nan nan nan +0.58 0.46 nan nan nan +0.59 0.46 nan nan nan +0.60 0.46 nan nan nan +0.61 0.46 nan nan nan +0.62 0.46 nan nan nan +0.64 0.46 nan nan nan +0.65 0.46 nan nan nan +0.66 0.46 nan nan nan +0.67 0.46 nan nan nan +0.68 0.46 nan nan nan +0.69 0.46 nan nan nan +0.70 0.46 nan nan nan +0.70 0.46 nan nan nan +0.71 0.46 nan nan nan +0.72 0.46 nan nan nan +0.73 0.46 nan nan nan +0.74 0.46 nan nan nan +0.76 0.46 nan nan nan +0.77 0.46 nan nan nan +0.78 0.46 nan nan nan +0.79 0.46 nan nan nan +0.80 0.46 nan nan nan +0.81 0.46 nan nan nan +0.82 0.46 nan nan nan +0.83 0.46 nan nan nan +0.83 0.46 nan nan nan +0.84 0.46 nan nan nan +0.85 0.46 nan nan nan +0.86 0.46 nan nan nan +0.88 0.46 nan nan nan +0.89 0.46 nan nan nan +0.90 0.46 nan nan nan +0.91 0.46 nan nan nan +0.92 0.46 nan nan nan +0.93 0.46 nan nan nan +0.94 0.46 nan nan nan +0.95 0.46 nan nan nan +0.96 0.46 nan nan nan +0.96 0.46 nan nan nan +0.97 0.46 nan nan nan +0.98 0.46 nan nan nan +0.01 0.47 nan nan nan +0.01 0.47 nan nan nan +0.03 0.47 nan nan nan +0.04 0.47 nan nan nan +0.04 0.47 nan nan nan +0.06 0.47 nan nan nan +0.07 0.47 nan nan nan +0.07 0.47 nan nan nan +0.09 0.47 nan nan nan +0.10 0.47 nan nan nan +0.10 0.47 nan nan nan +0.12 0.47 nan nan nan +0.12 0.47 nan nan nan +0.14 0.47 nan nan nan +0.14 0.47 nan nan nan +0.15 0.47 nan nan nan +0.17 0.47 nan nan nan +0.18 0.47 nan nan nan +0.18 0.47 nan nan nan +0.20 0.47 nan nan nan +0.21 0.47 nan nan nan +0.21 0.47 nan nan nan +0.23 0.47 nan nan nan +0.24 0.47 nan nan nan +0.24 0.47 nan nan nan +0.26 0.47 nan nan nan +0.27 0.47 nan nan nan +0.28 0.47 nan nan nan +0.29 0.47 nan nan nan +0.29 0.47 nan nan nan +0.30 0.47 nan nan nan +0.32 0.47 nan nan nan +0.33 0.47 nan nan nan +0.34 0.47 nan nan nan +0.35 0.47 nan nan nan +0.35 0.47 nan nan nan +0.36 0.47 nan nan nan +0.38 0.47 nan nan nan +0.39 0.47 nan nan nan +0.40 0.47 nan nan nan +0.41 0.47 nan nan nan +0.42 0.47 nan nan nan +0.42 0.47 nan nan nan +0.43 0.47 nan nan nan +0.45 0.47 nan nan nan +0.46 0.47 nan nan nan +0.47 0.47 nan nan nan +0.48 0.47 nan nan nan +0.48 0.47 nan nan nan +0.49 0.47 nan nan nan +0.51 0.47 nan nan nan +0.52 0.47 nan nan nan +0.53 0.47 nan nan nan +0.54 0.47 nan nan nan +0.55 0.47 nan nan nan +0.56 0.47 nan nan nan +0.57 0.47 nan nan nan +0.58 0.47 nan nan nan +0.58 0.47 nan nan nan +0.59 0.47 nan nan nan +0.60 0.47 nan nan nan +0.61 0.47 nan nan nan +0.62 0.47 nan nan nan +0.64 0.47 nan nan nan +0.65 0.47 nan nan nan +0.66 0.47 nan nan nan +0.67 0.47 nan nan nan +0.68 0.47 nan nan nan +0.69 0.47 nan nan nan +0.70 0.47 nan nan nan +0.70 0.47 nan nan nan +0.71 0.47 nan nan nan +0.72 0.47 nan nan nan +0.73 0.47 nan nan nan +0.74 0.47 nan nan nan +0.76 0.47 nan nan nan +0.77 0.47 nan nan nan +0.78 0.47 nan nan nan +0.79 0.47 nan nan nan +0.80 0.47 nan nan nan +0.81 0.47 nan nan nan +0.82 0.47 nan nan nan +0.83 0.47 nan nan nan +0.83 0.47 nan nan nan +0.84 0.47 nan nan nan +0.85 0.47 nan nan nan +0.86 0.47 nan nan nan +0.88 0.47 nan nan nan +0.89 0.47 nan nan nan +0.90 0.47 nan nan nan +0.91 0.47 nan nan nan +0.92 0.47 nan nan nan +0.93 0.47 nan nan nan +0.94 0.47 nan nan nan +0.95 0.47 nan nan nan +0.96 0.47 nan nan nan +0.96 0.47 nan nan nan +0.97 0.47 nan nan nan +0.98 0.47 nan nan nan +0.01 0.48 nan nan nan +0.01 0.48 nan nan nan +0.03 0.48 nan nan nan +0.04 0.48 nan nan nan +0.04 0.48 nan nan nan +0.06 0.48 nan nan nan +0.07 0.48 nan nan nan +0.07 0.48 nan nan nan +0.09 0.48 nan nan nan +0.10 0.48 nan nan nan +0.10 0.48 nan nan nan +0.12 0.48 nan nan nan +0.12 0.48 nan nan nan +0.14 0.48 nan nan nan +0.14 0.48 nan nan nan +0.15 0.48 nan nan nan +0.17 0.48 nan nan nan +0.18 0.48 nan nan nan +0.18 0.48 nan nan nan +0.20 0.48 nan nan nan +0.21 0.48 nan nan nan +0.21 0.48 nan nan nan +0.23 0.48 nan nan nan +0.24 0.48 nan nan nan +0.24 0.48 nan nan nan +0.26 0.48 nan nan nan +0.27 0.48 nan nan nan +0.28 0.48 nan nan nan +0.29 0.48 nan nan nan +0.29 0.48 nan nan nan +0.30 0.48 nan nan nan +0.32 0.48 nan nan nan +0.33 0.48 nan nan nan +0.34 0.48 nan nan nan +0.35 0.48 nan nan nan +0.35 0.48 nan nan nan +0.36 0.48 nan nan nan +0.38 0.48 nan nan nan +0.39 0.48 nan nan nan +0.40 0.48 nan nan nan +0.41 0.48 nan nan nan +0.42 0.48 nan nan nan +0.42 0.48 nan nan nan +0.43 0.48 nan nan nan +0.45 0.48 nan nan nan +0.46 0.48 nan nan nan +0.47 0.48 nan nan nan +0.48 0.48 nan nan nan +0.48 0.48 nan nan nan +0.49 0.48 nan nan nan +0.51 0.48 nan nan nan +0.52 0.48 nan nan nan +0.53 0.48 nan nan nan +0.54 0.48 nan nan nan +0.55 0.48 nan nan nan +0.56 0.48 nan nan nan +0.57 0.48 nan nan nan +0.58 0.48 nan nan nan +0.58 0.48 nan nan nan +0.59 0.48 nan nan nan +0.60 0.48 nan nan nan +0.61 0.48 nan nan nan +0.62 0.48 nan nan nan +0.64 0.48 nan nan nan +0.65 0.48 nan nan nan +0.66 0.48 nan nan nan +0.67 0.48 nan nan nan +0.68 0.48 nan nan nan +0.69 0.48 nan nan nan +0.70 0.48 nan nan nan +0.70 0.48 nan nan nan +0.71 0.48 nan nan nan +0.72 0.48 nan nan nan +0.73 0.48 nan nan nan +0.74 0.48 nan nan nan +0.76 0.48 nan nan nan +0.77 0.48 nan nan nan +0.78 0.48 nan nan nan +0.79 0.48 nan nan nan +0.80 0.48 nan nan nan +0.81 0.48 nan nan nan +0.82 0.48 nan nan nan +0.83 0.48 nan nan nan +0.83 0.48 nan nan nan +0.84 0.48 nan nan nan +0.85 0.48 nan nan nan +0.86 0.48 nan nan nan +0.88 0.48 nan nan nan +0.89 0.48 nan nan nan +0.90 0.48 nan nan nan +0.91 0.48 nan nan nan +0.92 0.48 nan nan nan +0.93 0.48 nan nan nan +0.94 0.48 nan nan nan +0.95 0.48 nan nan nan +0.96 0.48 nan nan nan +0.96 0.48 nan nan nan +0.97 0.48 nan nan nan +0.98 0.48 nan nan nan +0.01 0.48 nan nan nan +0.01 0.48 nan nan nan +0.03 0.48 nan nan nan +0.04 0.48 nan nan nan +0.04 0.48 nan nan nan +0.06 0.48 nan nan nan +0.07 0.48 nan nan nan +0.07 0.48 nan nan nan +0.09 0.48 nan nan nan +0.10 0.48 nan nan nan +0.10 0.48 nan nan nan +0.12 0.48 nan nan nan +0.12 0.48 nan nan nan +0.14 0.48 nan nan nan +0.14 0.48 nan nan nan +0.15 0.48 nan nan nan +0.17 0.48 nan nan nan +0.18 0.48 nan nan nan +0.18 0.48 nan nan nan +0.20 0.48 nan nan nan +0.21 0.48 nan nan nan +0.21 0.48 nan nan nan +0.23 0.48 nan nan nan +0.24 0.48 nan nan nan +0.24 0.48 nan nan nan +0.26 0.48 nan nan nan +0.27 0.48 nan nan nan +0.28 0.48 nan nan nan +0.29 0.48 nan nan nan +0.29 0.48 nan nan nan +0.30 0.48 nan nan nan +0.32 0.48 nan nan nan +0.33 0.48 nan nan nan +0.34 0.48 nan nan nan +0.35 0.48 nan nan nan +0.35 0.48 nan nan nan +0.36 0.48 nan nan nan +0.38 0.48 nan nan nan +0.39 0.48 nan nan nan +0.40 0.48 nan nan nan +0.41 0.48 nan nan nan +0.42 0.48 nan nan nan +0.42 0.48 nan nan nan +0.43 0.48 nan nan nan +0.45 0.48 nan nan nan +0.46 0.48 nan nan nan +0.47 0.48 nan nan nan +0.48 0.48 nan nan nan +0.48 0.48 nan nan nan +0.49 0.48 nan nan nan +0.51 0.48 nan nan nan +0.52 0.48 nan nan nan +0.53 0.48 nan nan nan +0.54 0.48 nan nan nan +0.55 0.48 nan nan nan +0.56 0.48 nan nan nan +0.57 0.48 nan nan nan +0.58 0.48 nan nan nan +0.58 0.48 nan nan nan +0.59 0.48 nan nan nan +0.60 0.48 nan nan nan +0.61 0.48 nan nan nan +0.62 0.48 nan nan nan +0.64 0.48 nan nan nan +0.65 0.48 nan nan nan +0.66 0.48 nan nan nan +0.67 0.48 nan nan nan +0.68 0.48 nan nan nan +0.69 0.48 nan nan nan +0.70 0.48 nan nan nan +0.70 0.48 nan nan nan +0.71 0.48 nan nan nan +0.72 0.48 nan nan nan +0.73 0.48 nan nan nan +0.74 0.48 nan nan nan +0.76 0.48 nan nan nan +0.77 0.48 nan nan nan +0.78 0.48 nan nan nan +0.79 0.48 nan nan nan +0.80 0.48 nan nan nan +0.81 0.48 nan nan nan +0.82 0.48 nan nan nan +0.83 0.48 nan nan nan +0.83 0.48 nan nan nan +0.84 0.48 nan nan nan +0.85 0.48 nan nan nan +0.86 0.48 nan nan nan +0.88 0.48 nan nan nan +0.89 0.48 nan nan nan +0.90 0.48 nan nan nan +0.91 0.48 nan nan nan +0.92 0.48 nan nan nan +0.93 0.48 nan nan nan +0.94 0.48 nan nan nan +0.95 0.48 nan nan nan +0.96 0.48 nan nan nan +0.96 0.48 nan nan nan +0.97 0.48 nan nan nan +0.98 0.48 nan nan nan +0.01 0.49 nan nan nan +0.01 0.49 nan nan nan +0.03 0.49 nan nan nan +0.04 0.49 nan nan nan +0.04 0.49 nan nan nan +0.06 0.49 nan nan nan +0.07 0.49 nan nan nan +0.07 0.49 nan nan nan +0.09 0.49 nan nan nan +0.10 0.49 nan nan nan +0.10 0.49 nan nan nan +0.12 0.49 nan nan nan +0.12 0.49 nan nan nan +0.14 0.49 nan nan nan +0.14 0.49 nan nan nan +0.15 0.49 nan nan nan +0.17 0.49 nan nan nan +0.18 0.49 nan nan nan +0.18 0.49 nan nan nan +0.20 0.49 nan nan nan +0.21 0.49 nan nan nan +0.21 0.49 nan nan nan +0.23 0.49 nan nan nan +0.24 0.49 nan nan nan +0.24 0.49 nan nan nan +0.26 0.49 nan nan nan +0.27 0.49 nan nan nan +0.28 0.49 nan nan nan +0.29 0.49 nan nan nan +0.29 0.49 nan nan nan +0.30 0.49 nan nan nan +0.32 0.49 nan nan nan +0.33 0.49 nan nan nan +0.34 0.49 nan nan nan +0.35 0.49 nan nan nan +0.35 0.49 nan nan nan +0.36 0.49 nan nan nan +0.38 0.49 nan nan nan +0.39 0.49 nan nan nan +0.40 0.49 nan nan nan +0.41 0.49 nan nan nan +0.42 0.49 nan nan nan +0.42 0.49 nan nan nan +0.43 0.49 nan nan nan +0.45 0.49 nan nan nan +0.46 0.49 nan nan nan +0.47 0.49 nan nan nan +0.48 0.49 nan nan nan +0.48 0.49 nan nan nan +0.49 0.49 nan nan nan +0.51 0.49 nan nan nan +0.52 0.49 nan nan nan +0.53 0.49 nan nan nan +0.54 0.49 nan nan nan +0.55 0.49 nan nan nan +0.56 0.49 nan nan nan +0.57 0.49 nan nan nan +0.58 0.49 nan nan nan +0.58 0.49 nan nan nan +0.59 0.49 nan nan nan +0.60 0.49 nan nan nan +0.61 0.49 nan nan nan +0.62 0.49 nan nan nan +0.64 0.49 nan nan nan +0.65 0.49 nan nan nan +0.66 0.49 nan nan nan +0.67 0.49 nan nan nan +0.68 0.49 nan nan nan +0.69 0.49 nan nan nan +0.70 0.49 nan nan nan +0.70 0.49 nan nan nan +0.71 0.49 nan nan nan +0.72 0.49 nan nan nan +0.73 0.49 nan nan nan +0.74 0.49 nan nan nan +0.76 0.49 nan nan nan +0.77 0.49 nan nan nan +0.78 0.49 nan nan nan +0.79 0.49 nan nan nan +0.80 0.49 nan nan nan +0.81 0.49 nan nan nan +0.82 0.49 nan nan nan +0.83 0.49 nan nan nan +0.83 0.49 nan nan nan +0.84 0.49 nan nan nan +0.85 0.49 nan nan nan +0.86 0.49 nan nan nan +0.88 0.49 nan nan nan +0.89 0.49 nan nan nan +0.90 0.49 nan nan nan +0.91 0.49 nan nan nan +0.92 0.49 nan nan nan +0.93 0.49 nan nan nan +0.94 0.49 nan nan nan +0.95 0.49 nan nan nan +0.96 0.49 nan nan nan +0.96 0.49 nan nan nan +0.97 0.49 nan nan nan +0.98 0.49 nan nan nan +0.01 0.51 nan nan nan +0.01 0.51 nan nan nan +0.03 0.51 nan nan nan +0.04 0.51 nan nan nan +0.04 0.51 nan nan nan +0.06 0.51 nan nan nan +0.07 0.51 nan nan nan +0.07 0.51 nan nan nan +0.09 0.51 nan nan nan +0.10 0.51 nan nan nan +0.10 0.51 nan nan nan +0.12 0.51 nan nan nan +0.12 0.51 nan nan nan +0.14 0.51 nan nan nan +0.14 0.51 nan nan nan +0.15 0.51 nan nan nan +0.17 0.51 nan nan nan +0.18 0.51 nan nan nan +0.18 0.51 nan nan nan +0.20 0.51 nan nan nan +0.21 0.51 nan nan nan +0.21 0.51 nan nan nan +0.23 0.51 nan nan nan +0.24 0.51 nan nan nan +0.24 0.51 nan nan nan +0.26 0.51 nan nan nan +0.27 0.51 nan nan nan +0.28 0.51 nan nan nan +0.29 0.51 nan nan nan +0.29 0.51 nan nan nan +0.30 0.51 nan nan nan +0.32 0.51 nan nan nan +0.33 0.51 nan nan nan +0.34 0.51 nan nan nan +0.35 0.51 nan nan nan +0.35 0.51 nan nan nan +0.36 0.51 nan nan nan +0.38 0.51 nan nan nan +0.39 0.51 nan nan nan +0.40 0.51 nan nan nan +0.41 0.51 nan nan nan +0.42 0.51 nan nan nan +0.42 0.51 nan nan nan +0.43 0.51 nan nan nan +0.45 0.51 nan nan nan +0.46 0.51 nan nan nan +0.47 0.51 nan nan nan +0.48 0.51 nan nan nan +0.48 0.51 nan nan nan +0.49 0.51 nan nan nan +0.51 0.51 nan nan nan +0.52 0.51 nan nan nan +0.53 0.51 nan nan nan +0.54 0.51 nan nan nan +0.55 0.51 nan nan nan +0.56 0.51 nan nan nan +0.57 0.51 nan nan nan +0.58 0.51 nan nan nan +0.58 0.51 nan nan nan +0.59 0.51 nan nan nan +0.60 0.51 nan nan nan +0.61 0.51 nan nan nan +0.62 0.51 nan nan nan +0.64 0.51 nan nan nan +0.65 0.51 nan nan nan +0.66 0.51 nan nan nan +0.67 0.51 nan nan nan +0.68 0.51 nan nan nan +0.69 0.51 nan nan nan +0.70 0.51 nan nan nan +0.70 0.51 nan nan nan +0.71 0.51 nan nan nan +0.72 0.51 nan nan nan +0.73 0.51 nan nan nan +0.74 0.51 nan nan nan +0.76 0.51 nan nan nan +0.77 0.51 nan nan nan +0.78 0.51 nan nan nan +0.79 0.51 nan nan nan +0.80 0.51 nan nan nan +0.81 0.51 nan nan nan +0.82 0.51 nan nan nan +0.83 0.51 nan nan nan +0.83 0.51 nan nan nan +0.84 0.51 nan nan nan +0.85 0.51 nan nan nan +0.86 0.51 nan nan nan +0.88 0.51 nan nan nan +0.89 0.51 nan nan nan +0.90 0.51 nan nan nan +0.91 0.51 nan nan nan +0.92 0.51 nan nan nan +0.93 0.51 nan nan nan +0.94 0.51 nan nan nan +0.95 0.51 nan nan nan +0.96 0.51 nan nan nan +0.96 0.51 nan nan nan +0.97 0.51 nan nan nan +0.98 0.51 nan nan nan +0.01 0.52 nan nan nan +0.01 0.52 nan nan nan +0.03 0.52 nan nan nan +0.04 0.52 nan nan nan +0.04 0.52 nan nan nan +0.06 0.52 nan nan nan +0.07 0.52 nan nan nan +0.07 0.52 nan nan nan +0.09 0.52 nan nan nan +0.10 0.52 nan nan nan +0.10 0.52 nan nan nan +0.12 0.52 nan nan nan +0.12 0.52 nan nan nan +0.14 0.52 nan nan nan +0.14 0.52 nan nan nan +0.15 0.52 nan nan nan +0.17 0.52 nan nan nan +0.18 0.52 nan nan nan +0.18 0.52 nan nan nan +0.20 0.52 nan nan nan +0.21 0.52 nan nan nan +0.21 0.52 nan nan nan +0.23 0.52 nan nan nan +0.24 0.52 nan nan nan +0.24 0.52 nan nan nan +0.26 0.52 nan nan nan +0.27 0.52 nan nan nan +0.28 0.52 nan nan nan +0.29 0.52 nan nan nan +0.29 0.52 nan nan nan +0.30 0.52 nan nan nan +0.32 0.52 nan nan nan +0.33 0.52 nan nan nan +0.34 0.52 nan nan nan +0.35 0.52 nan nan nan +0.35 0.52 nan nan nan +0.36 0.52 nan nan nan +0.38 0.52 nan nan nan +0.39 0.52 nan nan nan +0.40 0.52 nan nan nan +0.41 0.52 nan nan nan +0.42 0.52 nan nan nan +0.42 0.52 nan nan nan +0.43 0.52 nan nan nan +0.45 0.52 nan nan nan +0.46 0.52 nan nan nan +0.47 0.52 nan nan nan +0.48 0.52 nan nan nan +0.48 0.52 nan nan nan +0.49 0.52 nan nan nan +0.51 0.52 nan nan nan +0.52 0.52 nan nan nan +0.53 0.52 nan nan nan +0.54 0.52 nan nan nan +0.55 0.52 nan nan nan +0.56 0.52 nan nan nan +0.57 0.52 nan nan nan +0.58 0.52 nan nan nan +0.58 0.52 nan nan nan +0.59 0.52 nan nan nan +0.60 0.52 nan nan nan +0.61 0.52 nan nan nan +0.62 0.52 nan nan nan +0.64 0.52 nan nan nan +0.65 0.52 nan nan nan +0.66 0.52 nan nan nan +0.67 0.52 nan nan nan +0.68 0.52 nan nan nan +0.69 0.52 nan nan nan +0.70 0.52 nan nan nan +0.70 0.52 nan nan nan +0.71 0.52 nan nan nan +0.72 0.52 nan nan nan +0.73 0.52 nan nan nan +0.74 0.52 nan nan nan +0.76 0.52 nan nan nan +0.77 0.52 nan nan nan +0.78 0.52 nan nan nan +0.79 0.52 nan nan nan +0.80 0.52 nan nan nan +0.81 0.52 nan nan nan +0.82 0.52 nan nan nan +0.83 0.52 nan nan nan +0.83 0.52 nan nan nan +0.84 0.52 nan nan nan +0.85 0.52 nan nan nan +0.86 0.52 nan nan nan +0.88 0.52 nan nan nan +0.89 0.52 nan nan nan +0.90 0.52 nan nan nan +0.91 0.52 nan nan nan +0.92 0.52 nan nan nan +0.93 0.52 nan nan nan +0.94 0.52 nan nan nan +0.95 0.52 nan nan nan +0.96 0.52 nan nan nan +0.96 0.52 nan nan nan +0.97 0.52 nan nan nan +0.98 0.52 nan nan nan +0.01 0.53 nan nan nan +0.01 0.53 nan nan nan +0.03 0.53 nan nan nan +0.04 0.53 nan nan nan +0.04 0.53 nan nan nan +0.06 0.53 nan nan nan +0.07 0.53 nan nan nan +0.07 0.53 nan nan nan +0.09 0.53 nan nan nan +0.10 0.53 nan nan nan +0.10 0.53 nan nan nan +0.12 0.53 nan nan nan +0.12 0.53 nan nan nan +0.14 0.53 nan nan nan +0.14 0.53 nan nan nan +0.15 0.53 nan nan nan +0.17 0.53 nan nan nan +0.18 0.53 nan nan nan +0.18 0.53 nan nan nan +0.20 0.53 nan nan nan +0.21 0.53 nan nan nan +0.21 0.53 nan nan nan +0.23 0.53 nan nan nan +0.24 0.53 nan nan nan +0.24 0.53 nan nan nan +0.26 0.53 nan nan nan +0.27 0.53 nan nan nan +0.28 0.53 nan nan nan +0.29 0.53 nan nan nan +0.29 0.53 nan nan nan +0.30 0.53 nan nan nan +0.32 0.53 nan nan nan +0.33 0.53 nan nan nan +0.34 0.53 nan nan nan +0.35 0.53 nan nan nan +0.35 0.53 nan nan nan +0.36 0.53 nan nan nan +0.38 0.53 nan nan nan +0.39 0.53 nan nan nan +0.40 0.53 nan nan nan +0.41 0.53 nan nan nan +0.42 0.53 nan nan nan +0.42 0.53 nan nan nan +0.43 0.53 nan nan nan +0.45 0.53 nan nan nan +0.46 0.53 nan nan nan +0.47 0.53 nan nan nan +0.48 0.53 nan nan nan +0.48 0.53 nan nan nan +0.49 0.53 nan nan nan +0.51 0.53 nan nan nan +0.52 0.53 nan nan nan +0.53 0.53 nan nan nan +0.54 0.53 nan nan nan +0.55 0.53 nan nan nan +0.56 0.53 nan nan nan +0.57 0.53 nan nan nan +0.58 0.53 nan nan nan +0.58 0.53 nan nan nan +0.59 0.53 nan nan nan +0.60 0.53 nan nan nan +0.61 0.53 nan nan nan +0.62 0.53 nan nan nan +0.64 0.53 nan nan nan +0.65 0.53 nan nan nan +0.66 0.53 nan nan nan +0.67 0.53 nan nan nan +0.68 0.53 nan nan nan +0.69 0.53 nan nan nan +0.70 0.53 nan nan nan +0.70 0.53 nan nan nan +0.71 0.53 nan nan nan +0.72 0.53 nan nan nan +0.73 0.53 nan nan nan +0.74 0.53 nan nan nan +0.76 0.53 nan nan nan +0.77 0.53 nan nan nan +0.78 0.53 nan nan nan +0.79 0.53 nan nan nan +0.80 0.53 nan nan nan +0.81 0.53 nan nan nan +0.82 0.53 nan nan nan +0.83 0.53 nan nan nan +0.83 0.53 nan nan nan +0.84 0.53 nan nan nan +0.85 0.53 nan nan nan +0.86 0.53 nan nan nan +0.88 0.53 nan nan nan +0.89 0.53 nan nan nan +0.90 0.53 nan nan nan +0.91 0.53 nan nan nan +0.92 0.53 nan nan nan +0.93 0.53 nan nan nan +0.94 0.53 nan nan nan +0.95 0.53 nan nan nan +0.96 0.53 nan nan nan +0.96 0.53 nan nan nan +0.97 0.53 nan nan nan +0.98 0.53 nan nan nan +0.01 0.54 nan nan nan +0.01 0.54 nan nan nan +0.03 0.54 nan nan nan +0.04 0.54 nan nan nan +0.04 0.54 nan nan nan +0.06 0.54 nan nan nan +0.07 0.54 nan nan nan +0.07 0.54 nan nan nan +0.09 0.54 nan nan nan +0.10 0.54 nan nan nan +0.10 0.54 nan nan nan +0.12 0.54 nan nan nan +0.12 0.54 nan nan nan +0.14 0.54 nan nan nan +0.14 0.54 nan nan nan +0.15 0.54 nan nan nan +0.17 0.54 nan nan nan +0.18 0.54 nan nan nan +0.18 0.54 nan nan nan +0.20 0.54 nan nan nan +0.21 0.54 nan nan nan +0.21 0.54 nan nan nan +0.23 0.54 nan nan nan +0.24 0.54 nan nan nan +0.24 0.54 nan nan nan +0.26 0.54 nan nan nan +0.27 0.54 nan nan nan +0.28 0.54 nan nan nan +0.29 0.54 nan nan nan +0.29 0.54 nan nan nan +0.30 0.54 nan nan nan +0.32 0.54 nan nan nan +0.33 0.54 nan nan nan +0.34 0.54 nan nan nan +0.35 0.54 nan nan nan +0.35 0.54 nan nan nan +0.36 0.54 nan nan nan +0.38 0.54 nan nan nan +0.39 0.54 nan nan nan +0.40 0.54 nan nan nan +0.41 0.54 nan nan nan +0.42 0.54 nan nan nan +0.42 0.54 nan nan nan +0.43 0.54 nan nan nan +0.45 0.54 nan nan nan +0.46 0.54 nan nan nan +0.47 0.54 nan nan nan +0.48 0.54 nan nan nan +0.48 0.54 nan nan nan +0.49 0.54 nan nan nan +0.51 0.54 nan nan nan +0.52 0.54 nan nan nan +0.53 0.54 nan nan nan +0.54 0.54 nan nan nan +0.55 0.54 nan nan nan +0.56 0.54 nan nan nan +0.57 0.54 nan nan nan +0.58 0.54 nan nan nan +0.58 0.54 nan nan nan +0.59 0.54 nan nan nan +0.60 0.54 nan nan nan +0.61 0.54 nan nan nan +0.62 0.54 nan nan nan +0.64 0.54 nan nan nan +0.65 0.54 nan nan nan +0.66 0.54 nan nan nan +0.67 0.54 nan nan nan +0.68 0.54 nan nan nan +0.69 0.54 nan nan nan +0.70 0.54 nan nan nan +0.70 0.54 nan nan nan +0.71 0.54 nan nan nan +0.72 0.54 nan nan nan +0.73 0.54 nan nan nan +0.74 0.54 nan nan nan +0.76 0.54 nan nan nan +0.77 0.54 nan nan nan +0.78 0.54 nan nan nan +0.79 0.54 nan nan nan +0.80 0.54 nan nan nan +0.81 0.54 nan nan nan +0.82 0.54 nan nan nan +0.83 0.54 nan nan nan +0.83 0.54 nan nan nan +0.84 0.54 nan nan nan +0.85 0.54 nan nan nan +0.86 0.54 nan nan nan +0.88 0.54 nan nan nan +0.89 0.54 nan nan nan +0.90 0.54 nan nan nan +0.91 0.54 nan nan nan +0.92 0.54 nan nan nan +0.93 0.54 nan nan nan +0.94 0.54 nan nan nan +0.95 0.54 nan nan nan +0.96 0.54 nan nan nan +0.96 0.54 nan nan nan +0.97 0.54 nan nan nan +0.98 0.54 nan nan nan +0.01 0.55 nan nan nan +0.01 0.55 nan nan nan +0.03 0.55 nan nan nan +0.04 0.55 nan nan nan +0.04 0.55 nan nan nan +0.06 0.55 nan nan nan +0.07 0.55 nan nan nan +0.07 0.55 nan nan nan +0.09 0.55 nan nan nan +0.10 0.55 nan nan nan +0.10 0.55 nan nan nan +0.12 0.55 nan nan nan +0.12 0.55 nan nan nan +0.14 0.55 nan nan nan +0.14 0.55 nan nan nan +0.15 0.55 nan nan nan +0.17 0.55 nan nan nan +0.18 0.55 nan nan nan +0.18 0.55 nan nan nan +0.20 0.55 nan nan nan +0.21 0.55 nan nan nan +0.21 0.55 nan nan nan +0.23 0.55 nan nan nan +0.24 0.55 nan nan nan +0.24 0.55 nan nan nan +0.26 0.55 nan nan nan +0.27 0.55 nan nan nan +0.28 0.55 nan nan nan +0.29 0.55 nan nan nan +0.29 0.55 nan nan nan +0.30 0.55 nan nan nan +0.32 0.55 nan nan nan +0.33 0.55 nan nan nan +0.34 0.55 nan nan nan +0.35 0.55 nan nan nan +0.35 0.55 nan nan nan +0.36 0.55 nan nan nan +0.38 0.55 nan nan nan +0.39 0.55 nan nan nan +0.40 0.55 nan nan nan +0.41 0.55 nan nan nan +0.42 0.55 nan nan nan +0.42 0.55 nan nan nan +0.43 0.55 nan nan nan +0.45 0.55 nan nan nan +0.46 0.55 nan nan nan +0.47 0.55 nan nan nan +0.48 0.55 nan nan nan +0.48 0.55 nan nan nan +0.49 0.55 nan nan nan +0.51 0.55 nan nan nan +0.52 0.55 nan nan nan +0.53 0.55 nan nan nan +0.54 0.55 nan nan nan +0.55 0.55 nan nan nan +0.56 0.55 nan nan nan +0.57 0.55 nan nan nan +0.58 0.55 nan nan nan +0.58 0.55 nan nan nan +0.59 0.55 nan nan nan +0.60 0.55 nan nan nan +0.61 0.55 nan nan nan +0.62 0.55 nan nan nan +0.64 0.55 nan nan nan +0.65 0.55 nan nan nan +0.66 0.55 nan nan nan +0.67 0.55 nan nan nan +0.68 0.55 nan nan nan +0.69 0.55 nan nan nan +0.70 0.55 nan nan nan +0.70 0.55 nan nan nan +0.71 0.55 nan nan nan +0.72 0.55 nan nan nan +0.73 0.55 nan nan nan +0.74 0.55 nan nan nan +0.76 0.55 nan nan nan +0.77 0.55 nan nan nan +0.78 0.55 nan nan nan +0.79 0.55 nan nan nan +0.80 0.55 nan nan nan +0.81 0.55 nan nan nan +0.82 0.55 nan nan nan +0.83 0.55 nan nan nan +0.83 0.55 nan nan nan +0.84 0.55 nan nan nan +0.85 0.55 nan nan nan +0.86 0.55 nan nan nan +0.88 0.55 nan nan nan +0.89 0.55 nan nan nan +0.90 0.55 nan nan nan +0.91 0.55 nan nan nan +0.92 0.55 nan nan nan +0.93 0.55 nan nan nan +0.94 0.55 nan nan nan +0.95 0.55 nan nan nan +0.96 0.55 nan nan nan +0.96 0.55 nan nan nan +0.97 0.55 nan nan nan +0.98 0.55 nan nan nan +0.01 0.56 nan nan nan +0.01 0.56 nan nan nan +0.03 0.56 nan nan nan +0.04 0.56 nan nan nan +0.04 0.56 nan nan nan +0.06 0.56 nan nan nan +0.07 0.56 nan nan nan +0.07 0.56 nan nan nan +0.09 0.56 nan nan nan +0.10 0.56 nan nan nan +0.10 0.56 nan nan nan +0.12 0.56 nan nan nan +0.12 0.56 nan nan nan +0.14 0.56 nan nan nan +0.14 0.56 nan nan nan +0.15 0.56 nan nan nan +0.17 0.56 nan nan nan +0.18 0.56 nan nan nan +0.18 0.56 nan nan nan +0.20 0.56 nan nan nan +0.21 0.56 nan nan nan +0.21 0.56 nan nan nan +0.23 0.56 nan nan nan +0.24 0.56 nan nan nan +0.24 0.56 nan nan nan +0.26 0.56 nan nan nan +0.27 0.56 nan nan nan +0.28 0.56 nan nan nan +0.29 0.56 nan nan nan +0.29 0.56 nan nan nan +0.30 0.56 nan nan nan +0.32 0.56 nan nan nan +0.33 0.56 nan nan nan +0.34 0.56 nan nan nan +0.35 0.56 nan nan nan +0.35 0.56 nan nan nan +0.36 0.56 nan nan nan +0.38 0.56 nan nan nan +0.39 0.56 nan nan nan +0.40 0.56 nan nan nan +0.41 0.56 nan nan nan +0.42 0.56 nan nan nan +0.42 0.56 nan nan nan +0.43 0.56 nan nan nan +0.45 0.56 nan nan nan +0.46 0.56 nan nan nan +0.47 0.56 nan nan nan +0.48 0.56 nan nan nan +0.48 0.56 nan nan nan +0.49 0.56 nan nan nan +0.51 0.56 nan nan nan +0.52 0.56 nan nan nan +0.53 0.56 nan nan nan +0.54 0.56 nan nan nan +0.55 0.56 nan nan nan +0.56 0.56 nan nan nan +0.57 0.56 nan nan nan +0.58 0.56 nan nan nan +0.58 0.56 nan nan nan +0.59 0.56 nan nan nan +0.60 0.56 nan nan nan +0.61 0.56 nan nan nan +0.62 0.56 nan nan nan +0.64 0.56 nan nan nan +0.65 0.56 nan nan nan +0.66 0.56 nan nan nan +0.67 0.56 nan nan nan +0.68 0.56 nan nan nan +0.69 0.56 nan nan nan +0.70 0.56 nan nan nan +0.70 0.56 nan nan nan +0.71 0.56 nan nan nan +0.72 0.56 nan nan nan +0.73 0.56 nan nan nan +0.74 0.56 nan nan nan +0.76 0.56 nan nan nan +0.77 0.56 nan nan nan +0.78 0.56 nan nan nan +0.79 0.56 nan nan nan +0.80 0.56 nan nan nan +0.81 0.56 nan nan nan +0.82 0.56 nan nan nan +0.83 0.56 nan nan nan +0.83 0.56 nan nan nan +0.84 0.56 nan nan nan +0.85 0.56 nan nan nan +0.86 0.56 nan nan nan +0.88 0.56 nan nan nan +0.89 0.56 nan nan nan +0.90 0.56 nan nan nan +0.91 0.56 nan nan nan +0.92 0.56 nan nan nan +0.93 0.56 nan nan nan +0.94 0.56 nan nan nan +0.95 0.56 nan nan nan +0.96 0.56 nan nan nan +0.96 0.56 nan nan nan +0.97 0.56 nan nan nan +0.98 0.56 nan nan nan +0.01 0.57 nan nan nan +0.01 0.57 nan nan nan +0.03 0.57 nan nan nan +0.04 0.57 nan nan nan +0.04 0.57 nan nan nan +0.06 0.57 nan nan nan +0.07 0.57 nan nan nan +0.07 0.57 nan nan nan +0.09 0.57 nan nan nan +0.10 0.57 nan nan nan +0.10 0.57 nan nan nan +0.12 0.57 nan nan nan +0.12 0.57 nan nan nan +0.14 0.57 nan nan nan +0.14 0.57 nan nan nan +0.15 0.57 nan nan nan +0.17 0.57 nan nan nan +0.18 0.57 nan nan nan +0.18 0.57 nan nan nan +0.20 0.57 nan nan nan +0.21 0.57 nan nan nan +0.21 0.57 nan nan nan +0.23 0.57 nan nan nan +0.24 0.57 nan nan nan +0.24 0.57 nan nan nan +0.26 0.57 nan nan nan +0.27 0.57 nan nan nan +0.28 0.57 nan nan nan +0.29 0.57 nan nan nan +0.29 0.57 nan nan nan +0.30 0.57 nan nan nan +0.32 0.57 nan nan nan +0.33 0.57 nan nan nan +0.34 0.57 nan nan nan +0.35 0.57 nan nan nan +0.35 0.57 nan nan nan +0.36 0.57 nan nan nan +0.38 0.57 nan nan nan +0.39 0.57 nan nan nan +0.40 0.57 nan nan nan +0.41 0.57 nan nan nan +0.42 0.57 nan nan nan +0.42 0.57 nan nan nan +0.43 0.57 nan nan nan +0.45 0.57 nan nan nan +0.46 0.57 nan nan nan +0.47 0.57 nan nan nan +0.48 0.57 nan nan nan +0.48 0.57 nan nan nan +0.49 0.57 nan nan nan +0.51 0.57 nan nan nan +0.52 0.57 nan nan nan +0.53 0.57 nan nan nan +0.54 0.57 nan nan nan +0.55 0.57 nan nan nan +0.56 0.57 nan nan nan +0.57 0.57 nan nan nan +0.58 0.57 nan nan nan +0.58 0.57 nan nan nan +0.59 0.57 nan nan nan +0.60 0.57 nan nan nan +0.61 0.57 nan nan nan +0.62 0.57 nan nan nan +0.64 0.57 nan nan nan +0.65 0.57 nan nan nan +0.66 0.57 nan nan nan +0.67 0.57 nan nan nan +0.68 0.57 nan nan nan +0.69 0.57 nan nan nan +0.70 0.57 nan nan nan +0.70 0.57 nan nan nan +0.71 0.57 nan nan nan +0.72 0.57 nan nan nan +0.73 0.57 nan nan nan +0.74 0.57 nan nan nan +0.76 0.57 nan nan nan +0.77 0.57 nan nan nan +0.78 0.57 nan nan nan +0.79 0.57 nan nan nan +0.80 0.57 nan nan nan +0.81 0.57 nan nan nan +0.82 0.57 nan nan nan +0.83 0.57 nan nan nan +0.83 0.57 nan nan nan +0.84 0.57 nan nan nan +0.85 0.57 nan nan nan +0.86 0.57 nan nan nan +0.88 0.57 nan nan nan +0.89 0.57 nan nan nan +0.90 0.57 nan nan nan +0.91 0.57 nan nan nan +0.92 0.57 nan nan nan +0.93 0.57 nan nan nan +0.94 0.57 nan nan nan +0.95 0.57 nan nan nan +0.96 0.57 nan nan nan +0.96 0.57 nan nan nan +0.97 0.57 nan nan nan +0.98 0.57 nan nan nan +0.01 0.58 nan nan nan +0.01 0.58 nan nan nan +0.03 0.58 nan nan nan +0.04 0.58 nan nan nan +0.04 0.58 nan nan nan +0.06 0.58 nan nan nan +0.07 0.58 nan nan nan +0.07 0.58 nan nan nan +0.09 0.58 nan nan nan +0.10 0.58 nan nan nan +0.10 0.58 nan nan nan +0.12 0.58 nan nan nan +0.12 0.58 nan nan nan +0.14 0.58 nan nan nan +0.14 0.58 nan nan nan +0.15 0.58 nan nan nan +0.17 0.58 nan nan nan +0.18 0.58 nan nan nan +0.18 0.58 nan nan nan +0.20 0.58 nan nan nan +0.21 0.58 nan nan nan +0.21 0.58 nan nan nan +0.23 0.58 nan nan nan +0.24 0.58 nan nan nan +0.24 0.58 nan nan nan +0.26 0.58 nan nan nan +0.27 0.58 nan nan nan +0.28 0.58 nan nan nan +0.29 0.58 nan nan nan +0.29 0.58 nan nan nan +0.30 0.58 nan nan nan +0.32 0.58 nan nan nan +0.33 0.58 nan nan nan +0.34 0.58 nan nan nan +0.35 0.58 nan nan nan +0.35 0.58 nan nan nan +0.36 0.58 nan nan nan +0.38 0.58 nan nan nan +0.39 0.58 nan nan nan +0.40 0.58 nan nan nan +0.41 0.58 nan nan nan +0.42 0.58 nan nan nan +0.42 0.58 nan nan nan +0.43 0.58 nan nan nan +0.45 0.58 nan nan nan +0.46 0.58 nan nan nan +0.47 0.58 nan nan nan +0.48 0.58 nan nan nan +0.48 0.58 nan nan nan +0.49 0.58 nan nan nan +0.51 0.58 nan nan nan +0.52 0.58 nan nan nan +0.53 0.58 nan nan nan +0.54 0.58 nan nan nan +0.55 0.58 nan nan nan +0.56 0.58 nan nan nan +0.57 0.58 nan nan nan +0.58 0.58 nan nan nan +0.58 0.58 nan nan nan +0.59 0.58 nan nan nan +0.60 0.58 nan nan nan +0.61 0.58 nan nan nan +0.62 0.58 nan nan nan +0.64 0.58 nan nan nan +0.65 0.58 nan nan nan +0.66 0.58 nan nan nan +0.67 0.58 nan nan nan +0.68 0.58 nan nan nan +0.69 0.58 nan nan nan +0.70 0.58 nan nan nan +0.70 0.58 nan nan nan +0.71 0.58 nan nan nan +0.72 0.58 nan nan nan +0.73 0.58 nan nan nan +0.74 0.58 nan nan nan +0.76 0.58 nan nan nan +0.77 0.58 nan nan nan +0.78 0.58 nan nan nan +0.79 0.58 nan nan nan +0.80 0.58 nan nan nan +0.81 0.58 nan nan nan +0.82 0.58 nan nan nan +0.83 0.58 nan nan nan +0.83 0.58 nan nan nan +0.84 0.58 nan nan nan +0.85 0.58 nan nan nan +0.86 0.58 nan nan nan +0.88 0.58 nan nan nan +0.89 0.58 nan nan nan +0.90 0.58 nan nan nan +0.91 0.58 nan nan nan +0.92 0.58 nan nan nan +0.93 0.58 nan nan nan +0.94 0.58 nan nan nan +0.95 0.58 nan nan nan +0.96 0.58 nan nan nan +0.96 0.58 nan nan nan +0.97 0.58 nan nan nan +0.98 0.58 nan nan nan +0.01 0.58 nan nan nan +0.01 0.58 nan nan nan +0.03 0.58 nan nan nan +0.04 0.58 nan nan nan +0.04 0.58 nan nan nan +0.06 0.58 nan nan nan +0.07 0.58 nan nan nan +0.07 0.58 nan nan nan +0.09 0.58 nan nan nan +0.10 0.58 nan nan nan +0.10 0.58 nan nan nan +0.12 0.58 nan nan nan +0.12 0.58 nan nan nan +0.14 0.58 nan nan nan +0.14 0.58 nan nan nan +0.15 0.58 nan nan nan +0.17 0.58 nan nan nan +0.18 0.58 nan nan nan +0.18 0.58 nan nan nan +0.20 0.58 nan nan nan +0.21 0.58 nan nan nan +0.21 0.58 nan nan nan +0.23 0.58 nan nan nan +0.24 0.58 nan nan nan +0.24 0.58 nan nan nan +0.26 0.58 nan nan nan +0.27 0.58 nan nan nan +0.28 0.58 nan nan nan +0.29 0.58 nan nan nan +0.29 0.58 nan nan nan +0.30 0.58 nan nan nan +0.32 0.58 nan nan nan +0.33 0.58 nan nan nan +0.34 0.58 nan nan nan +0.35 0.58 nan nan nan +0.35 0.58 nan nan nan +0.36 0.58 nan nan nan +0.38 0.58 nan nan nan +0.39 0.58 nan nan nan +0.40 0.58 nan nan nan +0.41 0.58 nan nan nan +0.42 0.58 nan nan nan +0.42 0.58 nan nan nan +0.43 0.58 nan nan nan +0.45 0.58 nan nan nan +0.46 0.58 nan nan nan +0.47 0.58 nan nan nan +0.48 0.58 nan nan nan +0.48 0.58 nan nan nan +0.49 0.58 nan nan nan +0.51 0.58 nan nan nan +0.52 0.58 nan nan nan +0.53 0.58 nan nan nan +0.54 0.58 nan nan nan +0.55 0.58 nan nan nan +0.56 0.58 nan nan nan +0.57 0.58 nan nan nan +0.58 0.58 nan nan nan +0.58 0.58 nan nan nan +0.59 0.58 nan nan nan +0.60 0.58 nan nan nan +0.61 0.58 nan nan nan +0.62 0.58 nan nan nan +0.64 0.58 nan nan nan +0.65 0.58 nan nan nan +0.66 0.58 nan nan nan +0.67 0.58 nan nan nan +0.68 0.58 nan nan nan +0.69 0.58 nan nan nan +0.70 0.58 nan nan nan +0.70 0.58 nan nan nan +0.71 0.58 nan nan nan +0.72 0.58 nan nan nan +0.73 0.58 nan nan nan +0.74 0.58 nan nan nan +0.76 0.58 nan nan nan +0.77 0.58 nan nan nan +0.78 0.58 nan nan nan +0.79 0.58 nan nan nan +0.80 0.58 nan nan nan +0.81 0.58 nan nan nan +0.82 0.58 nan nan nan +0.83 0.58 nan nan nan +0.83 0.58 nan nan nan +0.84 0.58 nan nan nan +0.85 0.58 nan nan nan +0.86 0.58 nan nan nan +0.88 0.58 nan nan nan +0.89 0.58 nan nan nan +0.90 0.58 nan nan nan +0.91 0.58 nan nan nan +0.92 0.58 nan nan nan +0.93 0.58 nan nan nan +0.94 0.58 nan nan nan +0.95 0.58 nan nan nan +0.96 0.58 nan nan nan +0.96 0.58 nan nan nan +0.97 0.58 nan nan nan +0.98 0.58 nan nan nan +0.01 0.59 nan nan nan +0.01 0.59 nan nan nan +0.03 0.59 nan nan nan +0.04 0.59 nan nan nan +0.04 0.59 nan nan nan +0.06 0.59 nan nan nan +0.07 0.59 nan nan nan +0.07 0.59 nan nan nan +0.09 0.59 nan nan nan +0.10 0.59 nan nan nan +0.10 0.59 nan nan nan +0.12 0.59 nan nan nan +0.12 0.59 nan nan nan +0.14 0.59 nan nan nan +0.14 0.59 nan nan nan +0.15 0.59 nan nan nan +0.17 0.59 nan nan nan +0.18 0.59 nan nan nan +0.18 0.59 nan nan nan +0.20 0.59 nan nan nan +0.21 0.59 nan nan nan +0.21 0.59 nan nan nan +0.23 0.59 nan nan nan +0.24 0.59 nan nan nan +0.24 0.59 nan nan nan +0.26 0.59 nan nan nan +0.27 0.59 nan nan nan +0.28 0.59 nan nan nan +0.29 0.59 nan nan nan +0.29 0.59 nan nan nan +0.30 0.59 nan nan nan +0.32 0.59 nan nan nan +0.33 0.59 nan nan nan +0.34 0.59 nan nan nan +0.35 0.59 nan nan nan +0.35 0.59 nan nan nan +0.36 0.59 nan nan nan +0.38 0.59 nan nan nan +0.39 0.59 nan nan nan +0.40 0.59 nan nan nan +0.41 0.59 nan nan nan +0.42 0.59 nan nan nan +0.42 0.59 nan nan nan +0.43 0.59 nan nan nan +0.45 0.59 nan nan nan +0.46 0.59 nan nan nan +0.47 0.59 nan nan nan +0.48 0.59 nan nan nan +0.48 0.59 nan nan nan +0.49 0.59 nan nan nan +0.51 0.59 nan nan nan +0.52 0.59 nan nan nan +0.53 0.59 nan nan nan +0.54 0.59 nan nan nan +0.55 0.59 nan nan nan +0.56 0.59 nan nan nan +0.57 0.59 nan nan nan +0.58 0.59 nan nan nan +0.58 0.59 nan nan nan +0.59 0.59 nan nan nan +0.60 0.59 nan nan nan +0.61 0.59 nan nan nan +0.62 0.59 nan nan nan +0.64 0.59 nan nan nan +0.65 0.59 nan nan nan +0.66 0.59 nan nan nan +0.67 0.59 nan nan nan +0.68 0.59 nan nan nan +0.69 0.59 nan nan nan +0.70 0.59 nan nan nan +0.70 0.59 nan nan nan +0.71 0.59 nan nan nan +0.72 0.59 nan nan nan +0.73 0.59 nan nan nan +0.74 0.59 nan nan nan +0.76 0.59 nan nan nan +0.77 0.59 nan nan nan +0.78 0.59 nan nan nan +0.79 0.59 nan nan nan +0.80 0.59 nan nan nan +0.81 0.59 nan nan nan +0.82 0.59 nan nan nan +0.83 0.59 nan nan nan +0.83 0.59 nan nan nan +0.84 0.59 nan nan nan +0.85 0.59 nan nan nan +0.86 0.59 nan nan nan +0.88 0.59 nan nan nan +0.89 0.59 nan nan nan +0.90 0.59 nan nan nan +0.91 0.59 nan nan nan +0.92 0.59 nan nan nan +0.93 0.59 nan nan nan +0.94 0.59 nan nan nan +0.95 0.59 nan nan nan +0.96 0.59 nan nan nan +0.96 0.59 nan nan nan +0.97 0.59 nan nan nan +0.98 0.59 nan nan nan +0.01 0.60 nan nan nan +0.01 0.60 nan nan nan +0.03 0.60 nan nan nan +0.04 0.60 nan nan nan +0.04 0.60 nan nan nan +0.06 0.60 nan nan nan +0.07 0.60 nan nan nan +0.07 0.60 nan nan nan +0.09 0.60 nan nan nan +0.10 0.60 nan nan nan +0.10 0.60 nan nan nan +0.12 0.60 nan nan nan +0.12 0.60 nan nan nan +0.14 0.60 nan nan nan +0.14 0.60 nan nan nan +0.15 0.60 nan nan nan +0.17 0.60 nan nan nan +0.18 0.60 nan nan nan +0.18 0.60 nan nan nan +0.20 0.60 nan nan nan +0.21 0.60 nan nan nan +0.21 0.60 nan nan nan +0.23 0.60 nan nan nan +0.24 0.60 nan nan nan +0.24 0.60 nan nan nan +0.26 0.60 nan nan nan +0.27 0.60 nan nan nan +0.28 0.60 nan nan nan +0.29 0.60 nan nan nan +0.29 0.60 nan nan nan +0.30 0.60 nan nan nan +0.32 0.60 nan nan nan +0.33 0.60 nan nan nan +0.34 0.60 nan nan nan +0.35 0.60 nan nan nan +0.35 0.60 nan nan nan +0.36 0.60 nan nan nan +0.38 0.60 nan nan nan +0.39 0.60 nan nan nan +0.40 0.60 nan nan nan +0.41 0.60 nan nan nan +0.42 0.60 nan nan nan +0.42 0.60 nan nan nan +0.43 0.60 nan nan nan +0.45 0.60 nan nan nan +0.46 0.60 nan nan nan +0.47 0.60 nan nan nan +0.48 0.60 nan nan nan +0.48 0.60 nan nan nan +0.49 0.60 nan nan nan +0.51 0.60 nan nan nan +0.52 0.60 nan nan nan +0.53 0.60 nan nan nan +0.54 0.60 nan nan nan +0.55 0.60 nan nan nan +0.56 0.60 nan nan nan +0.57 0.60 nan nan nan +0.58 0.60 nan nan nan +0.58 0.60 nan nan nan +0.59 0.60 nan nan nan +0.60 0.60 nan nan nan +0.61 0.60 nan nan nan +0.62 0.60 nan nan nan +0.64 0.60 nan nan nan +0.65 0.60 nan nan nan +0.66 0.60 nan nan nan +0.67 0.60 nan nan nan +0.68 0.60 nan nan nan +0.69 0.60 nan nan nan +0.70 0.60 nan nan nan +0.70 0.60 nan nan nan +0.71 0.60 nan nan nan +0.72 0.60 nan nan nan +0.73 0.60 nan nan nan +0.74 0.60 nan nan nan +0.76 0.60 nan nan nan +0.77 0.60 nan nan nan +0.78 0.60 nan nan nan +0.79 0.60 nan nan nan +0.80 0.60 nan nan nan +0.81 0.60 nan nan nan +0.82 0.60 nan nan nan +0.83 0.60 nan nan nan +0.83 0.60 nan nan nan +0.84 0.60 nan nan nan +0.85 0.60 nan nan nan +0.86 0.60 nan nan nan +0.88 0.60 nan nan nan +0.89 0.60 nan nan nan +0.90 0.60 nan nan nan +0.91 0.60 nan nan nan +0.92 0.60 nan nan nan +0.93 0.60 nan nan nan +0.94 0.60 nan nan nan +0.95 0.60 nan nan nan +0.96 0.60 nan nan nan +0.96 0.60 nan nan nan +0.97 0.60 nan nan nan +0.98 0.60 nan nan nan +0.01 0.61 nan nan nan +0.01 0.61 nan nan nan +0.03 0.61 nan nan nan +0.04 0.61 nan nan nan +0.04 0.61 nan nan nan +0.06 0.61 nan nan nan +0.07 0.61 nan nan nan +0.07 0.61 nan nan nan +0.09 0.61 nan nan nan +0.10 0.61 nan nan nan +0.10 0.61 nan nan nan +0.12 0.61 nan nan nan +0.12 0.61 nan nan nan +0.14 0.61 nan nan nan +0.14 0.61 nan nan nan +0.15 0.61 nan nan nan +0.17 0.61 nan nan nan +0.18 0.61 nan nan nan +0.18 0.61 nan nan nan +0.20 0.61 nan nan nan +0.21 0.61 nan nan nan +0.21 0.61 nan nan nan +0.23 0.61 nan nan nan +0.24 0.61 nan nan nan +0.24 0.61 nan nan nan +0.26 0.61 nan nan nan +0.27 0.61 nan nan nan +0.28 0.61 nan nan nan +0.29 0.61 nan nan nan +0.29 0.61 nan nan nan +0.30 0.61 nan nan nan +0.32 0.61 nan nan nan +0.33 0.61 nan nan nan +0.34 0.61 nan nan nan +0.35 0.61 nan nan nan +0.35 0.61 nan nan nan +0.36 0.61 nan nan nan +0.38 0.61 nan nan nan +0.39 0.61 nan nan nan +0.40 0.61 nan nan nan +0.41 0.61 nan nan nan +0.42 0.61 nan nan nan +0.42 0.61 nan nan nan +0.43 0.61 nan nan nan +0.45 0.61 nan nan nan +0.46 0.61 nan nan nan +0.47 0.61 nan nan nan +0.48 0.61 nan nan nan +0.48 0.61 nan nan nan +0.49 0.61 nan nan nan +0.51 0.61 nan nan nan +0.52 0.61 nan nan nan +0.53 0.61 nan nan nan +0.54 0.61 nan nan nan +0.55 0.61 nan nan nan +0.56 0.61 nan nan nan +0.57 0.61 nan nan nan +0.58 0.61 nan nan nan +0.58 0.61 nan nan nan +0.59 0.61 nan nan nan +0.60 0.61 nan nan nan +0.61 0.61 nan nan nan +0.62 0.61 nan nan nan +0.64 0.61 nan nan nan +0.65 0.61 nan nan nan +0.66 0.61 nan nan nan +0.67 0.61 nan nan nan +0.68 0.61 nan nan nan +0.69 0.61 nan nan nan +0.70 0.61 nan nan nan +0.70 0.61 nan nan nan +0.71 0.61 nan nan nan +0.72 0.61 nan nan nan +0.73 0.61 nan nan nan +0.74 0.61 nan nan nan +0.76 0.61 nan nan nan +0.77 0.61 nan nan nan +0.78 0.61 nan nan nan +0.79 0.61 nan nan nan +0.80 0.61 nan nan nan +0.81 0.61 nan nan nan +0.82 0.61 nan nan nan +0.83 0.61 nan nan nan +0.83 0.61 nan nan nan +0.84 0.61 nan nan nan +0.85 0.61 nan nan nan +0.86 0.61 nan nan nan +0.88 0.61 nan nan nan +0.89 0.61 nan nan nan +0.90 0.61 nan nan nan +0.91 0.61 nan nan nan +0.92 0.61 nan nan nan +0.93 0.61 nan nan nan +0.94 0.61 nan nan nan +0.95 0.61 nan nan nan +0.96 0.61 nan nan nan +0.96 0.61 nan nan nan +0.97 0.61 nan nan nan +0.98 0.61 nan nan nan +0.01 0.62 nan nan nan +0.01 0.62 nan nan nan +0.03 0.62 nan nan nan +0.04 0.62 nan nan nan +0.04 0.62 nan nan nan +0.06 0.62 nan nan nan +0.07 0.62 nan nan nan +0.07 0.62 nan nan nan +0.09 0.62 nan nan nan +0.10 0.62 nan nan nan +0.10 0.62 nan nan nan +0.12 0.62 nan nan nan +0.12 0.62 nan nan nan +0.14 0.62 nan nan nan +0.14 0.62 nan nan nan +0.15 0.62 nan nan nan +0.17 0.62 nan nan nan +0.18 0.62 nan nan nan +0.18 0.62 nan nan nan +0.20 0.62 nan nan nan +0.21 0.62 nan nan nan +0.21 0.62 nan nan nan +0.23 0.62 nan nan nan +0.24 0.62 nan nan nan +0.24 0.62 nan nan nan +0.26 0.62 nan nan nan +0.27 0.62 nan nan nan +0.28 0.62 nan nan nan +0.29 0.62 nan nan nan +0.29 0.62 nan nan nan +0.30 0.62 nan nan nan +0.32 0.62 nan nan nan +0.33 0.62 nan nan nan +0.34 0.62 nan nan nan +0.35 0.62 nan nan nan +0.35 0.62 nan nan nan +0.36 0.62 nan nan nan +0.38 0.62 nan nan nan +0.39 0.62 nan nan nan +0.40 0.62 nan nan nan +0.41 0.62 nan nan nan +0.42 0.62 nan nan nan +0.42 0.62 nan nan nan +0.43 0.62 nan nan nan +0.45 0.62 nan nan nan +0.46 0.62 nan nan nan +0.47 0.62 nan nan nan +0.48 0.62 nan nan nan +0.48 0.62 nan nan nan +0.49 0.62 nan nan nan +0.51 0.62 nan nan nan +0.52 0.62 nan nan nan +0.53 0.62 nan nan nan +0.54 0.62 nan nan nan +0.55 0.62 nan nan nan +0.56 0.62 nan nan nan +0.57 0.62 nan nan nan +0.58 0.62 nan nan nan +0.58 0.62 nan nan nan +0.59 0.62 nan nan nan +0.60 0.62 nan nan nan +0.61 0.62 nan nan nan +0.62 0.62 nan nan nan +0.64 0.62 nan nan nan +0.65 0.62 nan nan nan +0.66 0.62 nan nan nan +0.67 0.62 nan nan nan +0.68 0.62 nan nan nan +0.69 0.62 nan nan nan +0.70 0.62 nan nan nan +0.70 0.62 nan nan nan +0.71 0.62 nan nan nan +0.72 0.62 nan nan nan +0.73 0.62 nan nan nan +0.74 0.62 nan nan nan +0.76 0.62 nan nan nan +0.77 0.62 nan nan nan +0.78 0.62 nan nan nan +0.79 0.62 nan nan nan +0.80 0.62 nan nan nan +0.81 0.62 nan nan nan +0.82 0.62 nan nan nan +0.83 0.62 nan nan nan +0.83 0.62 nan nan nan +0.84 0.62 nan nan nan +0.85 0.62 nan nan nan +0.86 0.62 nan nan nan +0.88 0.62 nan nan nan +0.89 0.62 nan nan nan +0.90 0.62 nan nan nan +0.91 0.62 nan nan nan +0.92 0.62 nan nan nan +0.93 0.62 nan nan nan +0.94 0.62 nan nan nan +0.95 0.62 nan nan nan +0.96 0.62 nan nan nan +0.96 0.62 nan nan nan +0.97 0.62 nan nan nan +0.98 0.62 nan nan nan +0.01 0.64 nan nan nan +0.01 0.64 nan nan nan +0.03 0.64 nan nan nan +0.04 0.64 nan nan nan +0.04 0.64 nan nan nan +0.06 0.64 nan nan nan +0.07 0.64 nan nan nan +0.07 0.64 nan nan nan +0.09 0.64 nan nan nan +0.10 0.64 nan nan nan +0.10 0.64 nan nan nan +0.12 0.64 nan nan nan +0.12 0.64 nan nan nan +0.14 0.64 nan nan nan +0.14 0.64 nan nan nan +0.15 0.64 nan nan nan +0.17 0.64 nan nan nan +0.18 0.64 nan nan nan +0.18 0.64 nan nan nan +0.20 0.64 nan nan nan +0.21 0.64 nan nan nan +0.21 0.64 nan nan nan +0.23 0.64 nan nan nan +0.24 0.64 nan nan nan +0.24 0.64 nan nan nan +0.26 0.64 nan nan nan +0.27 0.64 nan nan nan +0.28 0.64 nan nan nan +0.29 0.64 nan nan nan +0.29 0.64 nan nan nan +0.30 0.64 nan nan nan +0.32 0.64 nan nan nan +0.33 0.64 nan nan nan +0.34 0.64 nan nan nan +0.35 0.64 nan nan nan +0.35 0.64 nan nan nan +0.36 0.64 nan nan nan +0.38 0.64 nan nan nan +0.39 0.64 nan nan nan +0.40 0.64 nan nan nan +0.41 0.64 nan nan nan +0.42 0.64 nan nan nan +0.42 0.64 nan nan nan +0.43 0.64 nan nan nan +0.45 0.64 nan nan nan +0.46 0.64 nan nan nan +0.47 0.64 nan nan nan +0.48 0.64 nan nan nan +0.48 0.64 nan nan nan +0.49 0.64 nan nan nan +0.51 0.64 nan nan nan +0.52 0.64 nan nan nan +0.53 0.64 nan nan nan +0.54 0.64 nan nan nan +0.55 0.64 nan nan nan +0.56 0.64 nan nan nan +0.57 0.64 nan nan nan +0.58 0.64 nan nan nan +0.58 0.64 nan nan nan +0.59 0.64 nan nan nan +0.60 0.64 nan nan nan +0.61 0.64 nan nan nan +0.62 0.64 nan nan nan +0.64 0.64 nan nan nan +0.65 0.64 nan nan nan +0.66 0.64 nan nan nan +0.67 0.64 nan nan nan +0.68 0.64 nan nan nan +0.69 0.64 nan nan nan +0.70 0.64 nan nan nan +0.70 0.64 nan nan nan +0.71 0.64 nan nan nan +0.72 0.64 nan nan nan +0.73 0.64 nan nan nan +0.74 0.64 nan nan nan +0.76 0.64 nan nan nan +0.77 0.64 nan nan nan +0.78 0.64 nan nan nan +0.79 0.64 nan nan nan +0.80 0.64 nan nan nan +0.81 0.64 nan nan nan +0.82 0.64 nan nan nan +0.83 0.64 nan nan nan +0.83 0.64 nan nan nan +0.84 0.64 nan nan nan +0.85 0.64 nan nan nan +0.86 0.64 nan nan nan +0.88 0.64 nan nan nan +0.89 0.64 nan nan nan +0.90 0.64 nan nan nan +0.91 0.64 nan nan nan +0.92 0.64 nan nan nan +0.93 0.64 nan nan nan +0.94 0.64 nan nan nan +0.95 0.64 nan nan nan +0.96 0.64 nan nan nan +0.96 0.64 nan nan nan +0.97 0.64 nan nan nan +0.98 0.64 nan nan nan +0.01 0.65 nan nan nan +0.01 0.65 nan nan nan +0.03 0.65 nan nan nan +0.04 0.65 nan nan nan +0.04 0.65 nan nan nan +0.06 0.65 nan nan nan +0.07 0.65 nan nan nan +0.07 0.65 nan nan nan +0.09 0.65 nan nan nan +0.10 0.65 nan nan nan +0.10 0.65 nan nan nan +0.12 0.65 nan nan nan +0.12 0.65 nan nan nan +0.14 0.65 nan nan nan +0.14 0.65 nan nan nan +0.15 0.65 nan nan nan +0.17 0.65 nan nan nan +0.18 0.65 nan nan nan +0.18 0.65 nan nan nan +0.20 0.65 nan nan nan +0.21 0.65 nan nan nan +0.21 0.65 nan nan nan +0.23 0.65 nan nan nan +0.24 0.65 nan nan nan +0.24 0.65 nan nan nan +0.26 0.65 nan nan nan +0.27 0.65 nan nan nan +0.28 0.65 nan nan nan +0.29 0.65 nan nan nan +0.29 0.65 nan nan nan +0.30 0.65 nan nan nan +0.32 0.65 nan nan nan +0.33 0.65 nan nan nan +0.34 0.65 nan nan nan +0.35 0.65 nan nan nan +0.35 0.65 nan nan nan +0.36 0.65 nan nan nan +0.38 0.65 nan nan nan +0.39 0.65 nan nan nan +0.40 0.65 nan nan nan +0.41 0.65 nan nan nan +0.42 0.65 nan nan nan +0.42 0.65 nan nan nan +0.43 0.65 nan nan nan +0.45 0.65 nan nan nan +0.46 0.65 nan nan nan +0.47 0.65 nan nan nan +0.48 0.65 nan nan nan +0.48 0.65 nan nan nan +0.49 0.65 nan nan nan +0.51 0.65 nan nan nan +0.52 0.65 nan nan nan +0.53 0.65 nan nan nan +0.54 0.65 nan nan nan +0.55 0.65 nan nan nan +0.56 0.65 nan nan nan +0.57 0.65 nan nan nan +0.58 0.65 nan nan nan +0.58 0.65 nan nan nan +0.59 0.65 nan nan nan +0.60 0.65 nan nan nan +0.61 0.65 nan nan nan +0.62 0.65 nan nan nan +0.64 0.65 nan nan nan +0.65 0.65 nan nan nan +0.66 0.65 nan nan nan +0.67 0.65 nan nan nan +0.68 0.65 nan nan nan +0.69 0.65 nan nan nan +0.70 0.65 nan nan nan +0.70 0.65 nan nan nan +0.71 0.65 nan nan nan +0.72 0.65 nan nan nan +0.73 0.65 nan nan nan +0.74 0.65 nan nan nan +0.76 0.65 nan nan nan +0.77 0.65 nan nan nan +0.78 0.65 nan nan nan +0.79 0.65 nan nan nan +0.80 0.65 nan nan nan +0.81 0.65 nan nan nan +0.82 0.65 nan nan nan +0.83 0.65 nan nan nan +0.83 0.65 nan nan nan +0.84 0.65 nan nan nan +0.85 0.65 nan nan nan +0.86 0.65 nan nan nan +0.88 0.65 nan nan nan +0.89 0.65 nan nan nan +0.90 0.65 nan nan nan +0.91 0.65 nan nan nan +0.92 0.65 nan nan nan +0.93 0.65 nan nan nan +0.94 0.65 nan nan nan +0.95 0.65 nan nan nan +0.96 0.65 nan nan nan +0.96 0.65 nan nan nan +0.97 0.65 nan nan nan +0.98 0.65 nan nan nan +0.01 0.66 nan nan nan +0.01 0.66 nan nan nan +0.03 0.66 nan nan nan +0.04 0.66 nan nan nan +0.04 0.66 nan nan nan +0.06 0.66 nan nan nan +0.07 0.66 nan nan nan +0.07 0.66 nan nan nan +0.09 0.66 nan nan nan +0.10 0.66 nan nan nan +0.10 0.66 nan nan nan +0.12 0.66 nan nan nan +0.12 0.66 nan nan nan +0.14 0.66 nan nan nan +0.14 0.66 nan nan nan +0.15 0.66 nan nan nan +0.17 0.66 nan nan nan +0.18 0.66 nan nan nan +0.18 0.66 nan nan nan +0.20 0.66 nan nan nan +0.21 0.66 nan nan nan +0.21 0.66 nan nan nan +0.23 0.66 nan nan nan +0.24 0.66 nan nan nan +0.24 0.66 nan nan nan +0.26 0.66 nan nan nan +0.27 0.66 nan nan nan +0.28 0.66 nan nan nan +0.29 0.66 nan nan nan +0.29 0.66 nan nan nan +0.30 0.66 nan nan nan +0.32 0.66 nan nan nan +0.33 0.66 nan nan nan +0.34 0.66 nan nan nan +0.35 0.66 nan nan nan +0.35 0.66 nan nan nan +0.36 0.66 nan nan nan +0.38 0.66 nan nan nan +0.39 0.66 nan nan nan +0.40 0.66 nan nan nan +0.41 0.66 nan nan nan +0.42 0.66 nan nan nan +0.42 0.66 nan nan nan +0.43 0.66 nan nan nan +0.45 0.66 nan nan nan +0.46 0.66 nan nan nan +0.47 0.66 nan nan nan +0.48 0.66 nan nan nan +0.48 0.66 nan nan nan +0.49 0.66 nan nan nan +0.51 0.66 nan nan nan +0.52 0.66 nan nan nan +0.53 0.66 nan nan nan +0.54 0.66 nan nan nan +0.55 0.66 nan nan nan +0.56 0.66 nan nan nan +0.57 0.66 nan nan nan +0.58 0.66 nan nan nan +0.58 0.66 nan nan nan +0.59 0.66 nan nan nan +0.60 0.66 nan nan nan +0.61 0.66 nan nan nan +0.62 0.66 nan nan nan +0.64 0.66 nan nan nan +0.65 0.66 nan nan nan +0.66 0.66 nan nan nan +0.67 0.66 nan nan nan +0.68 0.66 nan nan nan +0.69 0.66 nan nan nan +0.70 0.66 nan nan nan +0.70 0.66 nan nan nan +0.71 0.66 nan nan nan +0.72 0.66 nan nan nan +0.73 0.66 nan nan nan +0.74 0.66 nan nan nan +0.76 0.66 nan nan nan +0.77 0.66 nan nan nan +0.78 0.66 nan nan nan +0.79 0.66 nan nan nan +0.80 0.66 nan nan nan +0.81 0.66 nan nan nan +0.82 0.66 nan nan nan +0.83 0.66 nan nan nan +0.83 0.66 nan nan nan +0.84 0.66 nan nan nan +0.85 0.66 nan nan nan +0.86 0.66 nan nan nan +0.88 0.66 nan nan nan +0.89 0.66 nan nan nan +0.90 0.66 nan nan nan +0.91 0.66 nan nan nan +0.92 0.66 nan nan nan +0.93 0.66 nan nan nan +0.94 0.66 nan nan nan +0.95 0.66 nan nan nan +0.96 0.66 nan nan nan +0.96 0.66 nan nan nan +0.97 0.66 nan nan nan +0.98 0.66 nan nan nan +0.01 0.67 nan nan nan +0.01 0.67 nan nan nan +0.03 0.67 nan nan nan +0.04 0.67 nan nan nan +0.04 0.67 nan nan nan +0.06 0.67 nan nan nan +0.07 0.67 nan nan nan +0.07 0.67 nan nan nan +0.09 0.67 nan nan nan +0.10 0.67 nan nan nan +0.10 0.67 nan nan nan +0.12 0.67 nan nan nan +0.12 0.67 nan nan nan +0.14 0.67 nan nan nan +0.14 0.67 nan nan nan +0.15 0.67 nan nan nan +0.17 0.67 nan nan nan +0.18 0.67 nan nan nan +0.18 0.67 nan nan nan +0.20 0.67 nan nan nan +0.21 0.67 nan nan nan +0.21 0.67 nan nan nan +0.23 0.67 nan nan nan +0.24 0.67 nan nan nan +0.24 0.67 nan nan nan +0.26 0.67 nan nan nan +0.27 0.67 nan nan nan +0.28 0.67 nan nan nan +0.29 0.67 nan nan nan +0.29 0.67 nan nan nan +0.30 0.67 nan nan nan +0.32 0.67 nan nan nan +0.33 0.67 nan nan nan +0.34 0.67 nan nan nan +0.35 0.67 nan nan nan +0.35 0.67 nan nan nan +0.36 0.67 nan nan nan +0.38 0.67 nan nan nan +0.39 0.67 nan nan nan +0.40 0.67 nan nan nan +0.41 0.67 nan nan nan +0.42 0.67 nan nan nan +0.42 0.67 nan nan nan +0.43 0.67 nan nan nan +0.45 0.67 nan nan nan +0.46 0.67 nan nan nan +0.47 0.67 nan nan nan +0.48 0.67 nan nan nan +0.48 0.67 nan nan nan +0.49 0.67 nan nan nan +0.51 0.67 nan nan nan +0.52 0.67 nan nan nan +0.53 0.67 nan nan nan +0.54 0.67 nan nan nan +0.55 0.67 nan nan nan +0.56 0.67 nan nan nan +0.57 0.67 nan nan nan +0.58 0.67 nan nan nan +0.58 0.67 nan nan nan +0.59 0.67 nan nan nan +0.60 0.67 nan nan nan +0.61 0.67 nan nan nan +0.62 0.67 nan nan nan +0.64 0.67 nan nan nan +0.65 0.67 nan nan nan +0.66 0.67 nan nan nan +0.67 0.67 nan nan nan +0.68 0.67 nan nan nan +0.69 0.67 nan nan nan +0.70 0.67 nan nan nan +0.70 0.67 nan nan nan +0.71 0.67 nan nan nan +0.72 0.67 nan nan nan +0.73 0.67 nan nan nan +0.74 0.67 nan nan nan +0.76 0.67 nan nan nan +0.77 0.67 nan nan nan +0.78 0.67 nan nan nan +0.79 0.67 nan nan nan +0.80 0.67 nan nan nan +0.81 0.67 nan nan nan +0.82 0.67 nan nan nan +0.83 0.67 nan nan nan +0.83 0.67 nan nan nan +0.84 0.67 nan nan nan +0.85 0.67 nan nan nan +0.86 0.67 nan nan nan +0.88 0.67 nan nan nan +0.89 0.67 nan nan nan +0.90 0.67 nan nan nan +0.91 0.67 nan nan nan +0.92 0.67 nan nan nan +0.93 0.67 nan nan nan +0.94 0.67 nan nan nan +0.95 0.67 nan nan nan +0.96 0.67 nan nan nan +0.96 0.67 nan nan nan +0.97 0.67 nan nan nan +0.98 0.67 nan nan nan +0.01 0.68 nan nan nan +0.01 0.68 nan nan nan +0.03 0.68 nan nan nan +0.04 0.68 nan nan nan +0.04 0.68 nan nan nan +0.06 0.68 nan nan nan +0.07 0.68 nan nan nan +0.07 0.68 nan nan nan +0.09 0.68 nan nan nan +0.10 0.68 nan nan nan +0.10 0.68 nan nan nan +0.12 0.68 nan nan nan +0.12 0.68 nan nan nan +0.14 0.68 nan nan nan +0.14 0.68 nan nan nan +0.15 0.68 nan nan nan +0.17 0.68 nan nan nan +0.18 0.68 nan nan nan +0.18 0.68 nan nan nan +0.20 0.68 nan nan nan +0.21 0.68 nan nan nan +0.21 0.68 nan nan nan +0.23 0.68 nan nan nan +0.24 0.68 nan nan nan +0.24 0.68 nan nan nan +0.26 0.68 nan nan nan +0.27 0.68 nan nan nan +0.28 0.68 nan nan nan +0.29 0.68 nan nan nan +0.29 0.68 nan nan nan +0.30 0.68 nan nan nan +0.32 0.68 nan nan nan +0.33 0.68 nan nan nan +0.34 0.68 nan nan nan +0.35 0.68 nan nan nan +0.35 0.68 nan nan nan +0.36 0.68 nan nan nan +0.38 0.68 nan nan nan +0.39 0.68 nan nan nan +0.40 0.68 nan nan nan +0.41 0.68 nan nan nan +0.42 0.68 nan nan nan +0.42 0.68 nan nan nan +0.43 0.68 nan nan nan +0.45 0.68 nan nan nan +0.46 0.68 nan nan nan +0.47 0.68 nan nan nan +0.48 0.68 nan nan nan +0.48 0.68 nan nan nan +0.49 0.68 nan nan nan +0.51 0.68 nan nan nan +0.52 0.68 nan nan nan +0.53 0.68 nan nan nan +0.54 0.68 nan nan nan +0.55 0.68 nan nan nan +0.56 0.68 nan nan nan +0.57 0.68 nan nan nan +0.58 0.68 nan nan nan +0.58 0.68 nan nan nan +0.59 0.68 nan nan nan +0.60 0.68 nan nan nan +0.61 0.68 nan nan nan +0.62 0.68 nan nan nan +0.64 0.68 nan nan nan +0.65 0.68 nan nan nan +0.66 0.68 nan nan nan +0.67 0.68 nan nan nan +0.68 0.68 nan nan nan +0.69 0.68 nan nan nan +0.70 0.68 nan nan nan +0.70 0.68 nan nan nan +0.71 0.68 nan nan nan +0.72 0.68 nan nan nan +0.73 0.68 nan nan nan +0.74 0.68 nan nan nan +0.76 0.68 nan nan nan +0.77 0.68 nan nan nan +0.78 0.68 nan nan nan +0.79 0.68 nan nan nan +0.80 0.68 nan nan nan +0.81 0.68 nan nan nan +0.82 0.68 nan nan nan +0.83 0.68 nan nan nan +0.83 0.68 nan nan nan +0.84 0.68 nan nan nan +0.85 0.68 nan nan nan +0.86 0.68 nan nan nan +0.88 0.68 nan nan nan +0.89 0.68 nan nan nan +0.90 0.68 nan nan nan +0.91 0.68 nan nan nan +0.92 0.68 nan nan nan +0.93 0.68 nan nan nan +0.94 0.68 nan nan nan +0.95 0.68 nan nan nan +0.96 0.68 nan nan nan +0.96 0.68 nan nan nan +0.97 0.68 nan nan nan +0.98 0.68 nan nan nan +0.01 0.69 nan nan nan +0.01 0.69 nan nan nan +0.03 0.69 nan nan nan +0.04 0.69 nan nan nan +0.04 0.69 nan nan nan +0.06 0.69 nan nan nan +0.07 0.69 nan nan nan +0.07 0.69 nan nan nan +0.09 0.69 nan nan nan +0.10 0.69 nan nan nan +0.10 0.69 nan nan nan +0.12 0.69 nan nan nan +0.12 0.69 nan nan nan +0.14 0.69 nan nan nan +0.14 0.69 nan nan nan +0.15 0.69 nan nan nan +0.17 0.69 nan nan nan +0.18 0.69 nan nan nan +0.18 0.69 nan nan nan +0.20 0.69 nan nan nan +0.21 0.69 nan nan nan +0.21 0.69 nan nan nan +0.23 0.69 nan nan nan +0.24 0.69 nan nan nan +0.24 0.69 nan nan nan +0.26 0.69 nan nan nan +0.27 0.69 nan nan nan +0.28 0.69 nan nan nan +0.29 0.69 nan nan nan +0.29 0.69 nan nan nan +0.30 0.69 nan nan nan +0.32 0.69 nan nan nan +0.33 0.69 nan nan nan +0.34 0.69 nan nan nan +0.35 0.69 nan nan nan +0.35 0.69 nan nan nan +0.36 0.69 nan nan nan +0.38 0.69 nan nan nan +0.39 0.69 nan nan nan +0.40 0.69 nan nan nan +0.41 0.69 nan nan nan +0.42 0.69 nan nan nan +0.42 0.69 nan nan nan +0.43 0.69 nan nan nan +0.45 0.69 nan nan nan +0.46 0.69 nan nan nan +0.47 0.69 nan nan nan +0.48 0.69 nan nan nan +0.48 0.69 nan nan nan +0.49 0.69 nan nan nan +0.51 0.69 nan nan nan +0.52 0.69 nan nan nan +0.53 0.69 nan nan nan +0.54 0.69 nan nan nan +0.55 0.69 nan nan nan +0.56 0.69 nan nan nan +0.57 0.69 nan nan nan +0.58 0.69 nan nan nan +0.58 0.69 nan nan nan +0.59 0.69 nan nan nan +0.60 0.69 nan nan nan +0.61 0.69 nan nan nan +0.62 0.69 nan nan nan +0.64 0.69 nan nan nan +0.65 0.69 nan nan nan +0.66 0.69 nan nan nan +0.67 0.69 nan nan nan +0.68 0.69 nan nan nan +0.69 0.69 nan nan nan +0.70 0.69 nan nan nan +0.70 0.69 nan nan nan +0.71 0.69 nan nan nan +0.72 0.69 nan nan nan +0.73 0.69 nan nan nan +0.74 0.69 nan nan nan +0.76 0.69 nan nan nan +0.77 0.69 nan nan nan +0.78 0.69 nan nan nan +0.79 0.69 nan nan nan +0.80 0.69 nan nan nan +0.81 0.69 nan nan nan +0.82 0.69 nan nan nan +0.83 0.69 nan nan nan +0.83 0.69 nan nan nan +0.84 0.69 nan nan nan +0.85 0.69 nan nan nan +0.86 0.69 nan nan nan +0.88 0.69 nan nan nan +0.89 0.69 nan nan nan +0.90 0.69 nan nan nan +0.91 0.69 nan nan nan +0.92 0.69 nan nan nan +0.93 0.69 nan nan nan +0.94 0.69 nan nan nan +0.95 0.69 nan nan nan +0.96 0.69 nan nan nan +0.96 0.69 nan nan nan +0.97 0.69 nan nan nan +0.98 0.69 nan nan nan +0.01 0.70 nan nan nan +0.01 0.70 nan nan nan +0.03 0.70 nan nan nan +0.04 0.70 nan nan nan +0.04 0.70 nan nan nan +0.06 0.70 nan nan nan +0.07 0.70 nan nan nan +0.07 0.70 nan nan nan +0.09 0.70 nan nan nan +0.10 0.70 nan nan nan +0.10 0.70 nan nan nan +0.12 0.70 nan nan nan +0.12 0.70 nan nan nan +0.14 0.70 nan nan nan +0.14 0.70 nan nan nan +0.15 0.70 nan nan nan +0.17 0.70 nan nan nan +0.18 0.70 nan nan nan +0.18 0.70 nan nan nan +0.20 0.70 nan nan nan +0.21 0.70 nan nan nan +0.21 0.70 nan nan nan +0.23 0.70 nan nan nan +0.24 0.70 nan nan nan +0.24 0.70 nan nan nan +0.26 0.70 nan nan nan +0.27 0.70 nan nan nan +0.28 0.70 nan nan nan +0.29 0.70 nan nan nan +0.29 0.70 nan nan nan +0.30 0.70 nan nan nan +0.32 0.70 nan nan nan +0.33 0.70 nan nan nan +0.34 0.70 nan nan nan +0.35 0.70 nan nan nan +0.35 0.70 nan nan nan +0.36 0.70 nan nan nan +0.38 0.70 nan nan nan +0.39 0.70 nan nan nan +0.40 0.70 nan nan nan +0.41 0.70 nan nan nan +0.42 0.70 nan nan nan +0.42 0.70 nan nan nan +0.43 0.70 nan nan nan +0.45 0.70 nan nan nan +0.46 0.70 nan nan nan +0.47 0.70 nan nan nan +0.48 0.70 nan nan nan +0.48 0.70 nan nan nan +0.49 0.70 nan nan nan +0.51 0.70 nan nan nan +0.52 0.70 nan nan nan +0.53 0.70 nan nan nan +0.54 0.70 nan nan nan +0.55 0.70 nan nan nan +0.56 0.70 nan nan nan +0.57 0.70 nan nan nan +0.58 0.70 nan nan nan +0.58 0.70 nan nan nan +0.59 0.70 nan nan nan +0.60 0.70 nan nan nan +0.61 0.70 nan nan nan +0.62 0.70 nan nan nan +0.64 0.70 nan nan nan +0.65 0.70 nan nan nan +0.66 0.70 nan nan nan +0.67 0.70 nan nan nan +0.68 0.70 nan nan nan +0.69 0.70 nan nan nan +0.70 0.70 nan nan nan +0.70 0.70 nan nan nan +0.71 0.70 nan nan nan +0.72 0.70 nan nan nan +0.73 0.70 nan nan nan +0.74 0.70 nan nan nan +0.76 0.70 nan nan nan +0.77 0.70 nan nan nan +0.78 0.70 nan nan nan +0.79 0.70 nan nan nan +0.80 0.70 nan nan nan +0.81 0.70 nan nan nan +0.82 0.70 nan nan nan +0.83 0.70 nan nan nan +0.83 0.70 nan nan nan +0.84 0.70 nan nan nan +0.85 0.70 nan nan nan +0.86 0.70 nan nan nan +0.88 0.70 nan nan nan +0.89 0.70 nan nan nan +0.90 0.70 nan nan nan +0.91 0.70 nan nan nan +0.92 0.70 nan nan nan +0.93 0.70 nan nan nan +0.94 0.70 nan nan nan +0.95 0.70 nan nan nan +0.96 0.70 nan nan nan +0.96 0.70 nan nan nan +0.97 0.70 nan nan nan +0.98 0.70 nan nan nan +0.01 0.70 nan nan nan +0.01 0.70 nan nan nan +0.03 0.70 nan nan nan +0.04 0.70 nan nan nan +0.04 0.70 nan nan nan +0.06 0.70 nan nan nan +0.07 0.70 nan nan nan +0.07 0.70 nan nan nan +0.09 0.70 nan nan nan +0.10 0.70 nan nan nan +0.10 0.70 nan nan nan +0.12 0.70 nan nan nan +0.12 0.70 nan nan nan +0.14 0.70 nan nan nan +0.14 0.70 nan nan nan +0.15 0.70 nan nan nan +0.17 0.70 nan nan nan +0.18 0.70 nan nan nan +0.18 0.70 nan nan nan +0.20 0.70 nan nan nan +0.21 0.70 nan nan nan +0.21 0.70 nan nan nan +0.23 0.70 nan nan nan +0.24 0.70 nan nan nan +0.24 0.70 nan nan nan +0.26 0.70 nan nan nan +0.27 0.70 nan nan nan +0.28 0.70 nan nan nan +0.29 0.70 nan nan nan +0.29 0.70 nan nan nan +0.30 0.70 nan nan nan +0.32 0.70 nan nan nan +0.33 0.70 nan nan nan +0.34 0.70 nan nan nan +0.35 0.70 nan nan nan +0.35 0.70 nan nan nan +0.36 0.70 nan nan nan +0.38 0.70 nan nan nan +0.39 0.70 nan nan nan +0.40 0.70 nan nan nan +0.41 0.70 nan nan nan +0.42 0.70 nan nan nan +0.42 0.70 nan nan nan +0.43 0.70 nan nan nan +0.45 0.70 nan nan nan +0.46 0.70 nan nan nan +0.47 0.70 nan nan nan +0.48 0.70 nan nan nan +0.48 0.70 nan nan nan +0.49 0.70 nan nan nan +0.51 0.70 nan nan nan +0.52 0.70 nan nan nan +0.53 0.70 nan nan nan +0.54 0.70 nan nan nan +0.55 0.70 nan nan nan +0.56 0.70 nan nan nan +0.57 0.70 nan nan nan +0.58 0.70 nan nan nan +0.58 0.70 nan nan nan +0.59 0.70 nan nan nan +0.60 0.70 nan nan nan +0.61 0.70 nan nan nan +0.62 0.70 nan nan nan +0.64 0.70 nan nan nan +0.65 0.70 nan nan nan +0.66 0.70 nan nan nan +0.67 0.70 nan nan nan +0.68 0.70 nan nan nan +0.69 0.70 nan nan nan +0.70 0.70 nan nan nan +0.70 0.70 nan nan nan +0.71 0.70 nan nan nan +0.72 0.70 nan nan nan +0.73 0.70 nan nan nan +0.74 0.70 nan nan nan +0.76 0.70 nan nan nan +0.77 0.70 nan nan nan +0.78 0.70 nan nan nan +0.79 0.70 nan nan nan +0.80 0.70 nan nan nan +0.81 0.70 nan nan nan +0.82 0.70 nan nan nan +0.83 0.70 nan nan nan +0.83 0.70 nan nan nan +0.84 0.70 nan nan nan +0.85 0.70 nan nan nan +0.86 0.70 nan nan nan +0.88 0.70 nan nan nan +0.89 0.70 nan nan nan +0.90 0.70 nan nan nan +0.91 0.70 nan nan nan +0.92 0.70 nan nan nan +0.93 0.70 nan nan nan +0.94 0.70 nan nan nan +0.95 0.70 nan nan nan +0.96 0.70 nan nan nan +0.96 0.70 nan nan nan +0.97 0.70 nan nan nan +0.98 0.70 nan nan nan +0.01 0.71 nan nan nan +0.01 0.71 nan nan nan +0.03 0.71 nan nan nan +0.04 0.71 nan nan nan +0.04 0.71 nan nan nan +0.06 0.71 nan nan nan +0.07 0.71 nan nan nan +0.07 0.71 nan nan nan +0.09 0.71 nan nan nan +0.10 0.71 nan nan nan +0.10 0.71 nan nan nan +0.12 0.71 nan nan nan +0.12 0.71 nan nan nan +0.14 0.71 nan nan nan +0.14 0.71 nan nan nan +0.15 0.71 nan nan nan +0.17 0.71 nan nan nan +0.18 0.71 nan nan nan +0.18 0.71 nan nan nan +0.20 0.71 nan nan nan +0.21 0.71 nan nan nan +0.21 0.71 nan nan nan +0.23 0.71 nan nan nan +0.24 0.71 nan nan nan +0.24 0.71 nan nan nan +0.26 0.71 nan nan nan +0.27 0.71 nan nan nan +0.28 0.71 nan nan nan +0.29 0.71 nan nan nan +0.29 0.71 nan nan nan +0.30 0.71 nan nan nan +0.32 0.71 nan nan nan +0.33 0.71 nan nan nan +0.34 0.71 nan nan nan +0.35 0.71 nan nan nan +0.35 0.71 nan nan nan +0.36 0.71 nan nan nan +0.38 0.71 nan nan nan +0.39 0.71 nan nan nan +0.40 0.71 nan nan nan +0.41 0.71 nan nan nan +0.42 0.71 nan nan nan +0.42 0.71 nan nan nan +0.43 0.71 nan nan nan +0.45 0.71 nan nan nan +0.46 0.71 nan nan nan +0.47 0.71 nan nan nan +0.48 0.71 nan nan nan +0.48 0.71 nan nan nan +0.49 0.71 nan nan nan +0.51 0.71 nan nan nan +0.52 0.71 nan nan nan +0.53 0.71 nan nan nan +0.54 0.71 nan nan nan +0.55 0.71 nan nan nan +0.56 0.71 nan nan nan +0.57 0.71 nan nan nan +0.58 0.71 nan nan nan +0.58 0.71 nan nan nan +0.59 0.71 nan nan nan +0.60 0.71 nan nan nan +0.61 0.71 nan nan nan +0.62 0.71 nan nan nan +0.64 0.71 nan nan nan +0.65 0.71 nan nan nan +0.66 0.71 nan nan nan +0.67 0.71 nan nan nan +0.68 0.71 nan nan nan +0.69 0.71 nan nan nan +0.70 0.71 nan nan nan +0.70 0.71 nan nan nan +0.71 0.71 nan nan nan +0.72 0.71 nan nan nan +0.73 0.71 nan nan nan +0.74 0.71 nan nan nan +0.76 0.71 nan nan nan +0.77 0.71 nan nan nan +0.78 0.71 nan nan nan +0.79 0.71 nan nan nan +0.80 0.71 nan nan nan +0.81 0.71 nan nan nan +0.82 0.71 nan nan nan +0.83 0.71 nan nan nan +0.83 0.71 nan nan nan +0.84 0.71 nan nan nan +0.85 0.71 nan nan nan +0.86 0.71 nan nan nan +0.88 0.71 nan nan nan +0.89 0.71 nan nan nan +0.90 0.71 nan nan nan +0.91 0.71 nan nan nan +0.92 0.71 nan nan nan +0.93 0.71 nan nan nan +0.94 0.71 nan nan nan +0.95 0.71 nan nan nan +0.96 0.71 nan nan nan +0.96 0.71 nan nan nan +0.97 0.71 nan nan nan +0.98 0.71 nan nan nan +0.01 0.72 nan nan nan +0.01 0.72 nan nan nan +0.03 0.72 nan nan nan +0.04 0.72 nan nan nan +0.04 0.72 nan nan nan +0.06 0.72 nan nan nan +0.07 0.72 nan nan nan +0.07 0.72 nan nan nan +0.09 0.72 nan nan nan +0.10 0.72 nan nan nan +0.10 0.72 nan nan nan +0.12 0.72 nan nan nan +0.12 0.72 nan nan nan +0.14 0.72 nan nan nan +0.14 0.72 nan nan nan +0.15 0.72 nan nan nan +0.17 0.72 nan nan nan +0.18 0.72 nan nan nan +0.18 0.72 nan nan nan +0.20 0.72 nan nan nan +0.21 0.72 nan nan nan +0.21 0.72 nan nan nan +0.23 0.72 nan nan nan +0.24 0.72 nan nan nan +0.24 0.72 nan nan nan +0.26 0.72 nan nan nan +0.27 0.72 nan nan nan +0.28 0.72 nan nan nan +0.29 0.72 nan nan nan +0.29 0.72 nan nan nan +0.30 0.72 nan nan nan +0.32 0.72 nan nan nan +0.33 0.72 nan nan nan +0.34 0.72 nan nan nan +0.35 0.72 nan nan nan +0.35 0.72 nan nan nan +0.36 0.72 nan nan nan +0.38 0.72 nan nan nan +0.39 0.72 nan nan nan +0.40 0.72 nan nan nan +0.41 0.72 nan nan nan +0.42 0.72 nan nan nan +0.42 0.72 nan nan nan +0.43 0.72 nan nan nan +0.45 0.72 nan nan nan +0.46 0.72 nan nan nan +0.47 0.72 nan nan nan +0.48 0.72 nan nan nan +0.48 0.72 nan nan nan +0.49 0.72 nan nan nan +0.51 0.72 nan nan nan +0.52 0.72 nan nan nan +0.53 0.72 nan nan nan +0.54 0.72 nan nan nan +0.55 0.72 nan nan nan +0.56 0.72 nan nan nan +0.57 0.72 nan nan nan +0.58 0.72 nan nan nan +0.58 0.72 nan nan nan +0.59 0.72 nan nan nan +0.60 0.72 nan nan nan +0.61 0.72 nan nan nan +0.62 0.72 nan nan nan +0.64 0.72 nan nan nan +0.65 0.72 nan nan nan +0.66 0.72 nan nan nan +0.67 0.72 nan nan nan +0.68 0.72 nan nan nan +0.69 0.72 nan nan nan +0.70 0.72 nan nan nan +0.70 0.72 nan nan nan +0.71 0.72 nan nan nan +0.72 0.72 nan nan nan +0.73 0.72 nan nan nan +0.74 0.72 nan nan nan +0.76 0.72 nan nan nan +0.77 0.72 nan nan nan +0.78 0.72 nan nan nan +0.79 0.72 nan nan nan +0.80 0.72 nan nan nan +0.81 0.72 nan nan nan +0.82 0.72 nan nan nan +0.83 0.72 nan nan nan +0.83 0.72 nan nan nan +0.84 0.72 nan nan nan +0.85 0.72 nan nan nan +0.86 0.72 nan nan nan +0.88 0.72 nan nan nan +0.89 0.72 nan nan nan +0.90 0.72 nan nan nan +0.91 0.72 nan nan nan +0.92 0.72 nan nan nan +0.93 0.72 nan nan nan +0.94 0.72 nan nan nan +0.95 0.72 nan nan nan +0.96 0.72 nan nan nan +0.96 0.72 nan nan nan +0.97 0.72 nan nan nan +0.98 0.72 nan nan nan +0.01 0.73 nan nan nan +0.01 0.73 nan nan nan +0.03 0.73 nan nan nan +0.04 0.73 nan nan nan +0.04 0.73 nan nan nan +0.06 0.73 nan nan nan +0.07 0.73 nan nan nan +0.07 0.73 nan nan nan +0.09 0.73 nan nan nan +0.10 0.73 nan nan nan +0.10 0.73 nan nan nan +0.12 0.73 nan nan nan +0.12 0.73 nan nan nan +0.14 0.73 nan nan nan +0.14 0.73 nan nan nan +0.15 0.73 nan nan nan +0.17 0.73 nan nan nan +0.18 0.73 nan nan nan +0.18 0.73 nan nan nan +0.20 0.73 nan nan nan +0.21 0.73 nan nan nan +0.21 0.73 nan nan nan +0.23 0.73 nan nan nan +0.24 0.73 nan nan nan +0.24 0.73 nan nan nan +0.26 0.73 nan nan nan +0.27 0.73 nan nan nan +0.28 0.73 nan nan nan +0.29 0.73 nan nan nan +0.29 0.73 nan nan nan +0.30 0.73 nan nan nan +0.32 0.73 nan nan nan +0.33 0.73 nan nan nan +0.34 0.73 nan nan nan +0.35 0.73 nan nan nan +0.35 0.73 nan nan nan +0.36 0.73 nan nan nan +0.38 0.73 nan nan nan +0.39 0.73 nan nan nan +0.40 0.73 nan nan nan +0.41 0.73 nan nan nan +0.42 0.73 nan nan nan +0.42 0.73 nan nan nan +0.43 0.73 nan nan nan +0.45 0.73 nan nan nan +0.46 0.73 nan nan nan +0.47 0.73 nan nan nan +0.48 0.73 nan nan nan +0.48 0.73 nan nan nan +0.49 0.73 nan nan nan +0.51 0.73 nan nan nan +0.52 0.73 nan nan nan +0.53 0.73 nan nan nan +0.54 0.73 nan nan nan +0.55 0.73 nan nan nan +0.56 0.73 nan nan nan +0.57 0.73 nan nan nan +0.58 0.73 nan nan nan +0.58 0.73 nan nan nan +0.59 0.73 nan nan nan +0.60 0.73 nan nan nan +0.61 0.73 nan nan nan +0.62 0.73 nan nan nan +0.64 0.73 nan nan nan +0.65 0.73 nan nan nan +0.66 0.73 nan nan nan +0.67 0.73 nan nan nan +0.68 0.73 nan nan nan +0.69 0.73 nan nan nan +0.70 0.73 nan nan nan +0.70 0.73 nan nan nan +0.71 0.73 nan nan nan +0.72 0.73 nan nan nan +0.73 0.73 nan nan nan +0.74 0.73 nan nan nan +0.76 0.73 nan nan nan +0.77 0.73 nan nan nan +0.78 0.73 nan nan nan +0.79 0.73 nan nan nan +0.80 0.73 nan nan nan +0.81 0.73 nan nan nan +0.82 0.73 nan nan nan +0.83 0.73 nan nan nan +0.83 0.73 nan nan nan +0.84 0.73 nan nan nan +0.85 0.73 nan nan nan +0.86 0.73 nan nan nan +0.88 0.73 nan nan nan +0.89 0.73 nan nan nan +0.90 0.73 nan nan nan +0.91 0.73 nan nan nan +0.92 0.73 nan nan nan +0.93 0.73 nan nan nan +0.94 0.73 nan nan nan +0.95 0.73 nan nan nan +0.96 0.73 nan nan nan +0.96 0.73 nan nan nan +0.97 0.73 nan nan nan +0.98 0.73 nan nan nan +0.01 0.74 nan nan nan +0.01 0.74 nan nan nan +0.03 0.74 nan nan nan +0.04 0.74 nan nan nan +0.04 0.74 nan nan nan +0.06 0.74 nan nan nan +0.07 0.74 nan nan nan +0.07 0.74 nan nan nan +0.09 0.74 nan nan nan +0.10 0.74 nan nan nan +0.10 0.74 nan nan nan +0.12 0.74 nan nan nan +0.12 0.74 nan nan nan +0.14 0.74 nan nan nan +0.14 0.74 nan nan nan +0.15 0.74 nan nan nan +0.17 0.74 nan nan nan +0.18 0.74 nan nan nan +0.18 0.74 nan nan nan +0.20 0.74 nan nan nan +0.21 0.74 nan nan nan +0.21 0.74 nan nan nan +0.23 0.74 nan nan nan +0.24 0.74 nan nan nan +0.24 0.74 nan nan nan +0.26 0.74 nan nan nan +0.27 0.74 nan nan nan +0.28 0.74 nan nan nan +0.29 0.74 nan nan nan +0.29 0.74 nan nan nan +0.30 0.74 nan nan nan +0.32 0.74 nan nan nan +0.33 0.74 nan nan nan +0.34 0.74 nan nan nan +0.35 0.74 nan nan nan +0.35 0.74 nan nan nan +0.36 0.74 nan nan nan +0.38 0.74 nan nan nan +0.39 0.74 nan nan nan +0.40 0.74 nan nan nan +0.41 0.74 nan nan nan +0.42 0.74 nan nan nan +0.42 0.74 nan nan nan +0.43 0.74 nan nan nan +0.45 0.74 nan nan nan +0.46 0.74 nan nan nan +0.47 0.74 nan nan nan +0.48 0.74 nan nan nan +0.48 0.74 nan nan nan +0.49 0.74 nan nan nan +0.51 0.74 nan nan nan +0.52 0.74 nan nan nan +0.53 0.74 nan nan nan +0.54 0.74 nan nan nan +0.55 0.74 nan nan nan +0.56 0.74 nan nan nan +0.57 0.74 nan nan nan +0.58 0.74 nan nan nan +0.58 0.74 nan nan nan +0.59 0.74 nan nan nan +0.60 0.74 nan nan nan +0.61 0.74 nan nan nan +0.62 0.74 nan nan nan +0.64 0.74 nan nan nan +0.65 0.74 nan nan nan +0.66 0.74 nan nan nan +0.67 0.74 nan nan nan +0.68 0.74 nan nan nan +0.69 0.74 nan nan nan +0.70 0.74 nan nan nan +0.70 0.74 nan nan nan +0.71 0.74 nan nan nan +0.72 0.74 nan nan nan +0.73 0.74 nan nan nan +0.74 0.74 nan nan nan +0.76 0.74 nan nan nan +0.77 0.74 nan nan nan +0.78 0.74 nan nan nan +0.79 0.74 nan nan nan +0.80 0.74 nan nan nan +0.81 0.74 nan nan nan +0.82 0.74 nan nan nan +0.83 0.74 nan nan nan +0.83 0.74 nan nan nan +0.84 0.74 nan nan nan +0.85 0.74 nan nan nan +0.86 0.74 nan nan nan +0.88 0.74 nan nan nan +0.89 0.74 nan nan nan +0.90 0.74 nan nan nan +0.91 0.74 nan nan nan +0.92 0.74 nan nan nan +0.93 0.74 nan nan nan +0.94 0.74 nan nan nan +0.95 0.74 nan nan nan +0.96 0.74 nan nan nan +0.96 0.74 nan nan nan +0.97 0.74 nan nan nan +0.98 0.74 nan nan nan +0.01 0.76 nan nan nan +0.01 0.76 nan nan nan +0.03 0.76 nan nan nan +0.04 0.76 nan nan nan +0.04 0.76 nan nan nan +0.06 0.76 nan nan nan +0.07 0.76 nan nan nan +0.07 0.76 nan nan nan +0.09 0.76 nan nan nan +0.10 0.76 nan nan nan +0.10 0.76 nan nan nan +0.12 0.76 nan nan nan +0.12 0.76 nan nan nan +0.14 0.76 nan nan nan +0.14 0.76 nan nan nan +0.15 0.76 nan nan nan +0.17 0.76 nan nan nan +0.18 0.76 nan nan nan +0.18 0.76 nan nan nan +0.20 0.76 nan nan nan +0.21 0.76 nan nan nan +0.21 0.76 nan nan nan +0.23 0.76 nan nan nan +0.24 0.76 nan nan nan +0.24 0.76 nan nan nan +0.26 0.76 nan nan nan +0.27 0.76 nan nan nan +0.28 0.76 nan nan nan +0.29 0.76 nan nan nan +0.29 0.76 nan nan nan +0.30 0.76 nan nan nan +0.32 0.76 nan nan nan +0.33 0.76 nan nan nan +0.34 0.76 nan nan nan +0.35 0.76 nan nan nan +0.35 0.76 nan nan nan +0.36 0.76 nan nan nan +0.38 0.76 nan nan nan +0.39 0.76 nan nan nan +0.40 0.76 nan nan nan +0.41 0.76 nan nan nan +0.42 0.76 nan nan nan +0.42 0.76 nan nan nan +0.43 0.76 nan nan nan +0.45 0.76 nan nan nan +0.46 0.76 nan nan nan +0.47 0.76 nan nan nan +0.48 0.76 nan nan nan +0.48 0.76 nan nan nan +0.49 0.76 nan nan nan +0.51 0.76 nan nan nan +0.52 0.76 nan nan nan +0.53 0.76 nan nan nan +0.54 0.76 nan nan nan +0.55 0.76 nan nan nan +0.56 0.76 nan nan nan +0.57 0.76 nan nan nan +0.58 0.76 nan nan nan +0.58 0.76 nan nan nan +0.59 0.76 nan nan nan +0.60 0.76 nan nan nan +0.61 0.76 nan nan nan +0.62 0.76 nan nan nan +0.64 0.76 nan nan nan +0.65 0.76 nan nan nan +0.66 0.76 nan nan nan +0.67 0.76 nan nan nan +0.68 0.76 nan nan nan +0.69 0.76 nan nan nan +0.70 0.76 nan nan nan +0.70 0.76 nan nan nan +0.71 0.76 nan nan nan +0.72 0.76 nan nan nan +0.73 0.76 nan nan nan +0.74 0.76 nan nan nan +0.76 0.76 nan nan nan +0.77 0.76 nan nan nan +0.78 0.76 nan nan nan +0.79 0.76 nan nan nan +0.80 0.76 nan nan nan +0.81 0.76 nan nan nan +0.82 0.76 nan nan nan +0.83 0.76 nan nan nan +0.83 0.76 nan nan nan +0.84 0.76 nan nan nan +0.85 0.76 nan nan nan +0.86 0.76 nan nan nan +0.88 0.76 nan nan nan +0.89 0.76 nan nan nan +0.90 0.76 nan nan nan +0.91 0.76 nan nan nan +0.92 0.76 nan nan nan +0.93 0.76 nan nan nan +0.94 0.76 nan nan nan +0.95 0.76 nan nan nan +0.96 0.76 nan nan nan +0.96 0.76 nan nan nan +0.97 0.76 nan nan nan +0.98 0.76 nan nan nan +0.01 0.77 nan nan nan +0.01 0.77 nan nan nan +0.03 0.77 nan nan nan +0.04 0.77 nan nan nan +0.04 0.77 nan nan nan +0.06 0.77 nan nan nan +0.07 0.77 nan nan nan +0.07 0.77 nan nan nan +0.09 0.77 nan nan nan +0.10 0.77 nan nan nan +0.10 0.77 nan nan nan +0.12 0.77 nan nan nan +0.12 0.77 nan nan nan +0.14 0.77 nan nan nan +0.14 0.77 nan nan nan +0.15 0.77 nan nan nan +0.17 0.77 nan nan nan +0.18 0.77 nan nan nan +0.18 0.77 nan nan nan +0.20 0.77 nan nan nan +0.21 0.77 nan nan nan +0.21 0.77 nan nan nan +0.23 0.77 nan nan nan +0.24 0.77 nan nan nan +0.24 0.77 nan nan nan +0.26 0.77 nan nan nan +0.27 0.77 nan nan nan +0.28 0.77 nan nan nan +0.29 0.77 nan nan nan +0.29 0.77 nan nan nan +0.30 0.77 nan nan nan +0.32 0.77 nan nan nan +0.33 0.77 nan nan nan +0.34 0.77 nan nan nan +0.35 0.77 nan nan nan +0.35 0.77 nan nan nan +0.36 0.77 nan nan nan +0.38 0.77 nan nan nan +0.39 0.77 nan nan nan +0.40 0.77 nan nan nan +0.41 0.77 nan nan nan +0.42 0.77 nan nan nan +0.42 0.77 nan nan nan +0.43 0.77 nan nan nan +0.45 0.77 nan nan nan +0.46 0.77 nan nan nan +0.47 0.77 nan nan nan +0.48 0.77 nan nan nan +0.48 0.77 nan nan nan +0.49 0.77 nan nan nan +0.51 0.77 nan nan nan +0.52 0.77 nan nan nan +0.53 0.77 nan nan nan +0.54 0.77 nan nan nan +0.55 0.77 nan nan nan +0.56 0.77 nan nan nan +0.57 0.77 nan nan nan +0.58 0.77 nan nan nan +0.58 0.77 nan nan nan +0.59 0.77 nan nan nan +0.60 0.77 nan nan nan +0.61 0.77 nan nan nan +0.62 0.77 nan nan nan +0.64 0.77 nan nan nan +0.65 0.77 nan nan nan +0.66 0.77 nan nan nan +0.67 0.77 nan nan nan +0.68 0.77 nan nan nan +0.69 0.77 nan nan nan +0.70 0.77 nan nan nan +0.70 0.77 nan nan nan +0.71 0.77 nan nan nan +0.72 0.77 nan nan nan +0.73 0.77 nan nan nan +0.74 0.77 nan nan nan +0.76 0.77 nan nan nan +0.77 0.77 nan nan nan +0.78 0.77 nan nan nan +0.79 0.77 nan nan nan +0.80 0.77 nan nan nan +0.81 0.77 nan nan nan +0.82 0.77 nan nan nan +0.83 0.77 nan nan nan +0.83 0.77 nan nan nan +0.84 0.77 nan nan nan +0.85 0.77 nan nan nan +0.86 0.77 nan nan nan +0.88 0.77 nan nan nan +0.89 0.77 nan nan nan +0.90 0.77 nan nan nan +0.91 0.77 nan nan nan +0.92 0.77 nan nan nan +0.93 0.77 nan nan nan +0.94 0.77 nan nan nan +0.95 0.77 nan nan nan +0.96 0.77 nan nan nan +0.96 0.77 nan nan nan +0.97 0.77 nan nan nan +0.98 0.77 nan nan nan +0.01 0.78 nan nan nan +0.01 0.78 nan nan nan +0.03 0.78 nan nan nan +0.04 0.78 nan nan nan +0.04 0.78 nan nan nan +0.06 0.78 nan nan nan +0.07 0.78 nan nan nan +0.07 0.78 nan nan nan +0.09 0.78 nan nan nan +0.10 0.78 nan nan nan +0.10 0.78 nan nan nan +0.12 0.78 nan nan nan +0.12 0.78 nan nan nan +0.14 0.78 nan nan nan +0.14 0.78 nan nan nan +0.15 0.78 nan nan nan +0.17 0.78 nan nan nan +0.18 0.78 nan nan nan +0.18 0.78 nan nan nan +0.20 0.78 nan nan nan +0.21 0.78 nan nan nan +0.21 0.78 nan nan nan +0.23 0.78 nan nan nan +0.24 0.78 nan nan nan +0.24 0.78 nan nan nan +0.26 0.78 nan nan nan +0.27 0.78 nan nan nan +0.28 0.78 nan nan nan +0.29 0.78 nan nan nan +0.29 0.78 nan nan nan +0.30 0.78 nan nan nan +0.32 0.78 nan nan nan +0.33 0.78 nan nan nan +0.34 0.78 nan nan nan +0.35 0.78 nan nan nan +0.35 0.78 nan nan nan +0.36 0.78 nan nan nan +0.38 0.78 nan nan nan +0.39 0.78 nan nan nan +0.40 0.78 nan nan nan +0.41 0.78 nan nan nan +0.42 0.78 nan nan nan +0.42 0.78 nan nan nan +0.43 0.78 nan nan nan +0.45 0.78 nan nan nan +0.46 0.78 nan nan nan +0.47 0.78 nan nan nan +0.48 0.78 nan nan nan +0.48 0.78 nan nan nan +0.49 0.78 nan nan nan +0.51 0.78 nan nan nan +0.52 0.78 nan nan nan +0.53 0.78 nan nan nan +0.54 0.78 nan nan nan +0.55 0.78 nan nan nan +0.56 0.78 nan nan nan +0.57 0.78 nan nan nan +0.58 0.78 nan nan nan +0.58 0.78 nan nan nan +0.59 0.78 nan nan nan +0.60 0.78 nan nan nan +0.61 0.78 nan nan nan +0.62 0.78 nan nan nan +0.64 0.78 nan nan nan +0.65 0.78 nan nan nan +0.66 0.78 nan nan nan +0.67 0.78 nan nan nan +0.68 0.78 nan nan nan +0.69 0.78 nan nan nan +0.70 0.78 nan nan nan +0.70 0.78 nan nan nan +0.71 0.78 nan nan nan +0.72 0.78 nan nan nan +0.73 0.78 nan nan nan +0.74 0.78 nan nan nan +0.76 0.78 nan nan nan +0.77 0.78 nan nan nan +0.78 0.78 nan nan nan +0.79 0.78 nan nan nan +0.80 0.78 nan nan nan +0.81 0.78 nan nan nan +0.82 0.78 nan nan nan +0.83 0.78 nan nan nan +0.83 0.78 nan nan nan +0.84 0.78 nan nan nan +0.85 0.78 nan nan nan +0.86 0.78 nan nan nan +0.88 0.78 nan nan nan +0.89 0.78 nan nan nan +0.90 0.78 nan nan nan +0.91 0.78 nan nan nan +0.92 0.78 nan nan nan +0.93 0.78 nan nan nan +0.94 0.78 nan nan nan +0.95 0.78 nan nan nan +0.96 0.78 nan nan nan +0.96 0.78 nan nan nan +0.97 0.78 nan nan nan +0.98 0.78 nan nan nan +0.01 0.79 nan nan nan +0.01 0.79 nan nan nan +0.03 0.79 nan nan nan +0.04 0.79 nan nan nan +0.04 0.79 nan nan nan +0.06 0.79 nan nan nan +0.07 0.79 nan nan nan +0.07 0.79 nan nan nan +0.09 0.79 nan nan nan +0.10 0.79 nan nan nan +0.10 0.79 nan nan nan +0.12 0.79 nan nan nan +0.12 0.79 nan nan nan +0.14 0.79 nan nan nan +0.14 0.79 nan nan nan +0.15 0.79 nan nan nan +0.17 0.79 nan nan nan +0.18 0.79 nan nan nan +0.18 0.79 nan nan nan +0.20 0.79 nan nan nan +0.21 0.79 nan nan nan +0.21 0.79 nan nan nan +0.23 0.79 nan nan nan +0.24 0.79 nan nan nan +0.24 0.79 nan nan nan +0.26 0.79 nan nan nan +0.27 0.79 nan nan nan +0.28 0.79 nan nan nan +0.29 0.79 nan nan nan +0.29 0.79 nan nan nan +0.30 0.79 nan nan nan +0.32 0.79 nan nan nan +0.33 0.79 nan nan nan +0.34 0.79 nan nan nan +0.35 0.79 nan nan nan +0.35 0.79 nan nan nan +0.36 0.79 nan nan nan +0.38 0.79 nan nan nan +0.39 0.79 nan nan nan +0.40 0.79 nan nan nan +0.41 0.79 nan nan nan +0.42 0.79 nan nan nan +0.42 0.79 nan nan nan +0.43 0.79 nan nan nan +0.45 0.79 nan nan nan +0.46 0.79 nan nan nan +0.47 0.79 nan nan nan +0.48 0.79 nan nan nan +0.48 0.79 nan nan nan +0.49 0.79 nan nan nan +0.51 0.79 nan nan nan +0.52 0.79 nan nan nan +0.53 0.79 nan nan nan +0.54 0.79 nan nan nan +0.55 0.79 nan nan nan +0.56 0.79 nan nan nan +0.57 0.79 nan nan nan +0.58 0.79 nan nan nan +0.58 0.79 nan nan nan +0.59 0.79 nan nan nan +0.60 0.79 nan nan nan +0.61 0.79 nan nan nan +0.62 0.79 nan nan nan +0.64 0.79 nan nan nan +0.65 0.79 nan nan nan +0.66 0.79 nan nan nan +0.67 0.79 nan nan nan +0.68 0.79 nan nan nan +0.69 0.79 nan nan nan +0.70 0.79 nan nan nan +0.70 0.79 nan nan nan +0.71 0.79 nan nan nan +0.72 0.79 nan nan nan +0.73 0.79 nan nan nan +0.74 0.79 nan nan nan +0.76 0.79 nan nan nan +0.77 0.79 nan nan nan +0.78 0.79 nan nan nan +0.79 0.79 nan nan nan +0.80 0.79 nan nan nan +0.81 0.79 nan nan nan +0.82 0.79 nan nan nan +0.83 0.79 nan nan nan +0.83 0.79 nan nan nan +0.84 0.79 nan nan nan +0.85 0.79 nan nan nan +0.86 0.79 nan nan nan +0.88 0.79 nan nan nan +0.89 0.79 nan nan nan +0.90 0.79 nan nan nan +0.91 0.79 nan nan nan +0.92 0.79 nan nan nan +0.93 0.79 nan nan nan +0.94 0.79 nan nan nan +0.95 0.79 nan nan nan +0.96 0.79 nan nan nan +0.96 0.79 nan nan nan +0.97 0.79 nan nan nan +0.98 0.79 nan nan nan +0.01 0.80 nan nan nan +0.01 0.80 nan nan nan +0.03 0.80 nan nan nan +0.04 0.80 nan nan nan +0.04 0.80 nan nan nan +0.06 0.80 nan nan nan +0.07 0.80 nan nan nan +0.07 0.80 nan nan nan +0.09 0.80 nan nan nan +0.10 0.80 nan nan nan +0.10 0.80 nan nan nan +0.12 0.80 nan nan nan +0.12 0.80 nan nan nan +0.14 0.80 nan nan nan +0.14 0.80 nan nan nan +0.15 0.80 nan nan nan +0.17 0.80 nan nan nan +0.18 0.80 nan nan nan +0.18 0.80 nan nan nan +0.20 0.80 nan nan nan +0.21 0.80 nan nan nan +0.21 0.80 nan nan nan +0.23 0.80 nan nan nan +0.24 0.80 nan nan nan +0.24 0.80 nan nan nan +0.26 0.80 nan nan nan +0.27 0.80 nan nan nan +0.28 0.80 nan nan nan +0.29 0.80 nan nan nan +0.29 0.80 nan nan nan +0.30 0.80 nan nan nan +0.32 0.80 nan nan nan +0.33 0.80 nan nan nan +0.34 0.80 nan nan nan +0.35 0.80 nan nan nan +0.35 0.80 nan nan nan +0.36 0.80 nan nan nan +0.38 0.80 nan nan nan +0.39 0.80 nan nan nan +0.40 0.80 nan nan nan +0.41 0.80 nan nan nan +0.42 0.80 nan nan nan +0.42 0.80 nan nan nan +0.43 0.80 nan nan nan +0.45 0.80 nan nan nan +0.46 0.80 nan nan nan +0.47 0.80 nan nan nan +0.48 0.80 nan nan nan +0.48 0.80 nan nan nan +0.49 0.80 nan nan nan +0.51 0.80 nan nan nan +0.52 0.80 nan nan nan +0.53 0.80 nan nan nan +0.54 0.80 nan nan nan +0.55 0.80 nan nan nan +0.56 0.80 nan nan nan +0.57 0.80 nan nan nan +0.58 0.80 nan nan nan +0.58 0.80 nan nan nan +0.59 0.80 nan nan nan +0.60 0.80 nan nan nan +0.61 0.80 nan nan nan +0.62 0.80 nan nan nan +0.64 0.80 nan nan nan +0.65 0.80 nan nan nan +0.66 0.80 nan nan nan +0.67 0.80 nan nan nan +0.68 0.80 nan nan nan +0.69 0.80 nan nan nan +0.70 0.80 nan nan nan +0.70 0.80 nan nan nan +0.71 0.80 nan nan nan +0.72 0.80 nan nan nan +0.73 0.80 nan nan nan +0.74 0.80 nan nan nan +0.76 0.80 nan nan nan +0.77 0.80 nan nan nan +0.78 0.80 nan nan nan +0.79 0.80 nan nan nan +0.80 0.80 nan nan nan +0.81 0.80 nan nan nan +0.82 0.80 nan nan nan +0.83 0.80 nan nan nan +0.83 0.80 nan nan nan +0.84 0.80 nan nan nan +0.85 0.80 nan nan nan +0.86 0.80 nan nan nan +0.88 0.80 nan nan nan +0.89 0.80 nan nan nan +0.90 0.80 nan nan nan +0.91 0.80 nan nan nan +0.92 0.80 nan nan nan +0.93 0.80 nan nan nan +0.94 0.80 nan nan nan +0.95 0.80 nan nan nan +0.96 0.80 nan nan nan +0.96 0.80 nan nan nan +0.97 0.80 nan nan nan +0.98 0.80 nan nan nan +0.01 0.81 nan nan nan +0.01 0.81 nan nan nan +0.03 0.81 nan nan nan +0.04 0.81 nan nan nan +0.04 0.81 nan nan nan +0.06 0.81 nan nan nan +0.07 0.81 nan nan nan +0.07 0.81 nan nan nan +0.09 0.81 nan nan nan +0.10 0.81 nan nan nan +0.10 0.81 nan nan nan +0.12 0.81 nan nan nan +0.12 0.81 nan nan nan +0.14 0.81 nan nan nan +0.14 0.81 nan nan nan +0.15 0.81 nan nan nan +0.17 0.81 nan nan nan +0.18 0.81 nan nan nan +0.18 0.81 nan nan nan +0.20 0.81 nan nan nan +0.21 0.81 nan nan nan +0.21 0.81 nan nan nan +0.23 0.81 nan nan nan +0.24 0.81 nan nan nan +0.24 0.81 nan nan nan +0.26 0.81 nan nan nan +0.27 0.81 nan nan nan +0.28 0.81 nan nan nan +0.29 0.81 nan nan nan +0.29 0.81 nan nan nan +0.30 0.81 nan nan nan +0.32 0.81 nan nan nan +0.33 0.81 nan nan nan +0.34 0.81 nan nan nan +0.35 0.81 nan nan nan +0.35 0.81 nan nan nan +0.36 0.81 nan nan nan +0.38 0.81 nan nan nan +0.39 0.81 nan nan nan +0.40 0.81 nan nan nan +0.41 0.81 nan nan nan +0.42 0.81 nan nan nan +0.42 0.81 nan nan nan +0.43 0.81 nan nan nan +0.45 0.81 nan nan nan +0.46 0.81 nan nan nan +0.47 0.81 nan nan nan +0.48 0.81 nan nan nan +0.48 0.81 nan nan nan +0.49 0.81 nan nan nan +0.51 0.81 nan nan nan +0.52 0.81 nan nan nan +0.53 0.81 nan nan nan +0.54 0.81 nan nan nan +0.55 0.81 nan nan nan +0.56 0.81 nan nan nan +0.57 0.81 nan nan nan +0.58 0.81 nan nan nan +0.58 0.81 nan nan nan +0.59 0.81 nan nan nan +0.60 0.81 nan nan nan +0.61 0.81 nan nan nan +0.62 0.81 nan nan nan +0.64 0.81 nan nan nan +0.65 0.81 nan nan nan +0.66 0.81 nan nan nan +0.67 0.81 nan nan nan +0.68 0.81 nan nan nan +0.69 0.81 nan nan nan +0.70 0.81 nan nan nan +0.70 0.81 nan nan nan +0.71 0.81 nan nan nan +0.72 0.81 nan nan nan +0.73 0.81 nan nan nan +0.74 0.81 nan nan nan +0.76 0.81 nan nan nan +0.77 0.81 nan nan nan +0.78 0.81 nan nan nan +0.79 0.81 nan nan nan +0.80 0.81 nan nan nan +0.81 0.81 nan nan nan +0.82 0.81 nan nan nan +0.83 0.81 nan nan nan +0.83 0.81 nan nan nan +0.84 0.81 nan nan nan +0.85 0.81 nan nan nan +0.86 0.81 nan nan nan +0.88 0.81 nan nan nan +0.89 0.81 nan nan nan +0.90 0.81 nan nan nan +0.91 0.81 nan nan nan +0.92 0.81 nan nan nan +0.93 0.81 nan nan nan +0.94 0.81 nan nan nan +0.95 0.81 nan nan nan +0.96 0.81 nan nan nan +0.96 0.81 nan nan nan +0.97 0.81 nan nan nan +0.98 0.81 nan nan nan +0.01 0.82 nan nan nan +0.01 0.82 nan nan nan +0.03 0.82 nan nan nan +0.04 0.82 nan nan nan +0.04 0.82 nan nan nan +0.06 0.82 nan nan nan +0.07 0.82 nan nan nan +0.07 0.82 nan nan nan +0.09 0.82 nan nan nan +0.10 0.82 nan nan nan +0.10 0.82 nan nan nan +0.12 0.82 nan nan nan +0.12 0.82 nan nan nan +0.14 0.82 nan nan nan +0.14 0.82 nan nan nan +0.15 0.82 nan nan nan +0.17 0.82 nan nan nan +0.18 0.82 nan nan nan +0.18 0.82 nan nan nan +0.20 0.82 nan nan nan +0.21 0.82 nan nan nan +0.21 0.82 nan nan nan +0.23 0.82 nan nan nan +0.24 0.82 nan nan nan +0.24 0.82 nan nan nan +0.26 0.82 nan nan nan +0.27 0.82 nan nan nan +0.28 0.82 nan nan nan +0.29 0.82 nan nan nan +0.29 0.82 nan nan nan +0.30 0.82 nan nan nan +0.32 0.82 nan nan nan +0.33 0.82 nan nan nan +0.34 0.82 nan nan nan +0.35 0.82 nan nan nan +0.35 0.82 nan nan nan +0.36 0.82 nan nan nan +0.38 0.82 nan nan nan +0.39 0.82 nan nan nan +0.40 0.82 nan nan nan +0.41 0.82 nan nan nan +0.42 0.82 nan nan nan +0.42 0.82 nan nan nan +0.43 0.82 nan nan nan +0.45 0.82 nan nan nan +0.46 0.82 nan nan nan +0.47 0.82 nan nan nan +0.48 0.82 nan nan nan +0.48 0.82 nan nan nan +0.49 0.82 nan nan nan +0.51 0.82 nan nan nan +0.52 0.82 nan nan nan +0.53 0.82 nan nan nan +0.54 0.82 nan nan nan +0.55 0.82 nan nan nan +0.56 0.82 nan nan nan +0.57 0.82 nan nan nan +0.58 0.82 nan nan nan +0.58 0.82 nan nan nan +0.59 0.82 nan nan nan +0.60 0.82 nan nan nan +0.61 0.82 nan nan nan +0.62 0.82 nan nan nan +0.64 0.82 nan nan nan +0.65 0.82 nan nan nan +0.66 0.82 nan nan nan +0.67 0.82 nan nan nan +0.68 0.82 nan nan nan +0.69 0.82 nan nan nan +0.70 0.82 nan nan nan +0.70 0.82 nan nan nan +0.71 0.82 nan nan nan +0.72 0.82 nan nan nan +0.73 0.82 nan nan nan +0.74 0.82 nan nan nan +0.76 0.82 nan nan nan +0.77 0.82 nan nan nan +0.78 0.82 nan nan nan +0.79 0.82 nan nan nan +0.80 0.82 nan nan nan +0.81 0.82 nan nan nan +0.82 0.82 nan nan nan +0.83 0.82 nan nan nan +0.83 0.82 nan nan nan +0.84 0.82 nan nan nan +0.85 0.82 nan nan nan +0.86 0.82 nan nan nan +0.88 0.82 nan nan nan +0.89 0.82 nan nan nan +0.90 0.82 nan nan nan +0.91 0.82 nan nan nan +0.92 0.82 nan nan nan +0.93 0.82 nan nan nan +0.94 0.82 nan nan nan +0.95 0.82 nan nan nan +0.96 0.82 nan nan nan +0.96 0.82 nan nan nan +0.97 0.82 nan nan nan +0.98 0.82 nan nan nan +0.01 0.83 nan nan nan +0.01 0.83 nan nan nan +0.03 0.83 nan nan nan +0.04 0.83 nan nan nan +0.04 0.83 nan nan nan +0.06 0.83 nan nan nan +0.07 0.83 nan nan nan +0.07 0.83 nan nan nan +0.09 0.83 nan nan nan +0.10 0.83 nan nan nan +0.10 0.83 nan nan nan +0.12 0.83 nan nan nan +0.12 0.83 nan nan nan +0.14 0.83 nan nan nan +0.14 0.83 nan nan nan +0.15 0.83 nan nan nan +0.17 0.83 nan nan nan +0.18 0.83 nan nan nan +0.18 0.83 nan nan nan +0.20 0.83 nan nan nan +0.21 0.83 nan nan nan +0.21 0.83 nan nan nan +0.23 0.83 nan nan nan +0.24 0.83 nan nan nan +0.24 0.83 nan nan nan +0.26 0.83 nan nan nan +0.27 0.83 nan nan nan +0.28 0.83 nan nan nan +0.29 0.83 nan nan nan +0.29 0.83 nan nan nan +0.30 0.83 nan nan nan +0.32 0.83 nan nan nan +0.33 0.83 nan nan nan +0.34 0.83 nan nan nan +0.35 0.83 nan nan nan +0.35 0.83 nan nan nan +0.36 0.83 nan nan nan +0.38 0.83 nan nan nan +0.39 0.83 nan nan nan +0.40 0.83 nan nan nan +0.41 0.83 nan nan nan +0.42 0.83 nan nan nan +0.42 0.83 nan nan nan +0.43 0.83 nan nan nan +0.45 0.83 nan nan nan +0.46 0.83 nan nan nan +0.47 0.83 nan nan nan +0.48 0.83 nan nan nan +0.48 0.83 nan nan nan +0.49 0.83 nan nan nan +0.51 0.83 nan nan nan +0.52 0.83 nan nan nan +0.53 0.83 nan nan nan +0.54 0.83 nan nan nan +0.55 0.83 nan nan nan +0.56 0.83 nan nan nan +0.57 0.83 nan nan nan +0.58 0.83 nan nan nan +0.58 0.83 nan nan nan +0.59 0.83 nan nan nan +0.60 0.83 nan nan nan +0.61 0.83 nan nan nan +0.62 0.83 nan nan nan +0.64 0.83 nan nan nan +0.65 0.83 nan nan nan +0.66 0.83 nan nan nan +0.67 0.83 nan nan nan +0.68 0.83 nan nan nan +0.69 0.83 nan nan nan +0.70 0.83 nan nan nan +0.70 0.83 nan nan nan +0.71 0.83 nan nan nan +0.72 0.83 nan nan nan +0.73 0.83 nan nan nan +0.74 0.83 nan nan nan +0.76 0.83 nan nan nan +0.77 0.83 nan nan nan +0.78 0.83 nan nan nan +0.79 0.83 nan nan nan +0.80 0.83 nan nan nan +0.81 0.83 nan nan nan +0.82 0.83 nan nan nan +0.83 0.83 nan nan nan +0.83 0.83 nan nan nan +0.84 0.83 nan nan nan +0.85 0.83 nan nan nan +0.86 0.83 nan nan nan +0.88 0.83 nan nan nan +0.89 0.83 nan nan nan +0.90 0.83 nan nan nan +0.91 0.83 nan nan nan +0.92 0.83 nan nan nan +0.93 0.83 nan nan nan +0.94 0.83 nan nan nan +0.95 0.83 nan nan nan +0.96 0.83 nan nan nan +0.96 0.83 nan nan nan +0.97 0.83 nan nan nan +0.98 0.83 nan nan nan +0.01 0.83 nan nan nan +0.01 0.83 nan nan nan +0.03 0.83 nan nan nan +0.04 0.83 nan nan nan +0.04 0.83 nan nan nan +0.06 0.83 nan nan nan +0.07 0.83 nan nan nan +0.07 0.83 nan nan nan +0.09 0.83 nan nan nan +0.10 0.83 nan nan nan +0.10 0.83 nan nan nan +0.12 0.83 nan nan nan +0.12 0.83 nan nan nan +0.14 0.83 nan nan nan +0.14 0.83 nan nan nan +0.15 0.83 nan nan nan +0.17 0.83 nan nan nan +0.18 0.83 nan nan nan +0.18 0.83 nan nan nan +0.20 0.83 nan nan nan +0.21 0.83 nan nan nan +0.21 0.83 nan nan nan +0.23 0.83 nan nan nan +0.24 0.83 nan nan nan +0.24 0.83 nan nan nan +0.26 0.83 nan nan nan +0.27 0.83 nan nan nan +0.28 0.83 nan nan nan +0.29 0.83 nan nan nan +0.29 0.83 nan nan nan +0.30 0.83 nan nan nan +0.32 0.83 nan nan nan +0.33 0.83 nan nan nan +0.34 0.83 nan nan nan +0.35 0.83 nan nan nan +0.35 0.83 nan nan nan +0.36 0.83 nan nan nan +0.38 0.83 nan nan nan +0.39 0.83 nan nan nan +0.40 0.83 nan nan nan +0.41 0.83 nan nan nan +0.42 0.83 nan nan nan +0.42 0.83 nan nan nan +0.43 0.83 nan nan nan +0.45 0.83 nan nan nan +0.46 0.83 nan nan nan +0.47 0.83 nan nan nan +0.48 0.83 nan nan nan +0.48 0.83 nan nan nan +0.49 0.83 nan nan nan +0.51 0.83 nan nan nan +0.52 0.83 nan nan nan +0.53 0.83 nan nan nan +0.54 0.83 nan nan nan +0.55 0.83 nan nan nan +0.56 0.83 nan nan nan +0.57 0.83 nan nan nan +0.58 0.83 nan nan nan +0.58 0.83 nan nan nan +0.59 0.83 nan nan nan +0.60 0.83 nan nan nan +0.61 0.83 nan nan nan +0.62 0.83 nan nan nan +0.64 0.83 nan nan nan +0.65 0.83 nan nan nan +0.66 0.83 nan nan nan +0.67 0.83 nan nan nan +0.68 0.83 nan nan nan +0.69 0.83 nan nan nan +0.70 0.83 nan nan nan +0.70 0.83 nan nan nan +0.71 0.83 nan nan nan +0.72 0.83 nan nan nan +0.73 0.83 nan nan nan +0.74 0.83 nan nan nan +0.76 0.83 nan nan nan +0.77 0.83 nan nan nan +0.78 0.83 nan nan nan +0.79 0.83 nan nan nan +0.80 0.83 nan nan nan +0.81 0.83 nan nan nan +0.82 0.83 nan nan nan +0.83 0.83 nan nan nan +0.83 0.83 nan nan nan +0.84 0.83 nan nan nan +0.85 0.83 nan nan nan +0.86 0.83 nan nan nan +0.88 0.83 nan nan nan +0.89 0.83 nan nan nan +0.90 0.83 nan nan nan +0.91 0.83 nan nan nan +0.92 0.83 nan nan nan +0.93 0.83 nan nan nan +0.94 0.83 nan nan nan +0.95 0.83 nan nan nan +0.96 0.83 nan nan nan +0.96 0.83 nan nan nan +0.97 0.83 nan nan nan +0.98 0.83 nan nan nan +0.01 0.84 nan nan nan +0.01 0.84 nan nan nan +0.03 0.84 nan nan nan +0.04 0.84 nan nan nan +0.04 0.84 nan nan nan +0.06 0.84 nan nan nan +0.07 0.84 nan nan nan +0.07 0.84 nan nan nan +0.09 0.84 nan nan nan +0.10 0.84 nan nan nan +0.10 0.84 nan nan nan +0.12 0.84 nan nan nan +0.12 0.84 nan nan nan +0.14 0.84 nan nan nan +0.14 0.84 nan nan nan +0.15 0.84 nan nan nan +0.17 0.84 nan nan nan +0.18 0.84 nan nan nan +0.18 0.84 nan nan nan +0.20 0.84 nan nan nan +0.21 0.84 nan nan nan +0.21 0.84 nan nan nan +0.23 0.84 nan nan nan +0.24 0.84 nan nan nan +0.24 0.84 nan nan nan +0.26 0.84 nan nan nan +0.27 0.84 nan nan nan +0.28 0.84 nan nan nan +0.29 0.84 nan nan nan +0.29 0.84 nan nan nan +0.30 0.84 nan nan nan +0.32 0.84 nan nan nan +0.33 0.84 nan nan nan +0.34 0.84 nan nan nan +0.35 0.84 nan nan nan +0.35 0.84 nan nan nan +0.36 0.84 nan nan nan +0.38 0.84 nan nan nan +0.39 0.84 nan nan nan +0.40 0.84 nan nan nan +0.41 0.84 nan nan nan +0.42 0.84 nan nan nan +0.42 0.84 nan nan nan +0.43 0.84 nan nan nan +0.45 0.84 nan nan nan +0.46 0.84 nan nan nan +0.47 0.84 nan nan nan +0.48 0.84 nan nan nan +0.48 0.84 nan nan nan +0.49 0.84 nan nan nan +0.51 0.84 nan nan nan +0.52 0.84 nan nan nan +0.53 0.84 nan nan nan +0.54 0.84 nan nan nan +0.55 0.84 nan nan nan +0.56 0.84 nan nan nan +0.57 0.84 nan nan nan +0.58 0.84 nan nan nan +0.58 0.84 nan nan nan +0.59 0.84 nan nan nan +0.60 0.84 nan nan nan +0.61 0.84 nan nan nan +0.62 0.84 nan nan nan +0.64 0.84 nan nan nan +0.65 0.84 nan nan nan +0.66 0.84 nan nan nan +0.67 0.84 nan nan nan +0.68 0.84 nan nan nan +0.69 0.84 nan nan nan +0.70 0.84 nan nan nan +0.70 0.84 nan nan nan +0.71 0.84 nan nan nan +0.72 0.84 nan nan nan +0.73 0.84 nan nan nan +0.74 0.84 nan nan nan +0.76 0.84 nan nan nan +0.77 0.84 nan nan nan +0.78 0.84 nan nan nan +0.79 0.84 nan nan nan +0.80 0.84 nan nan nan +0.81 0.84 nan nan nan +0.82 0.84 nan nan nan +0.83 0.84 nan nan nan +0.83 0.84 nan nan nan +0.84 0.84 nan nan nan +0.85 0.84 nan nan nan +0.86 0.84 nan nan nan +0.88 0.84 nan nan nan +0.89 0.84 nan nan nan +0.90 0.84 nan nan nan +0.91 0.84 nan nan nan +0.92 0.84 nan nan nan +0.93 0.84 nan nan nan +0.94 0.84 nan nan nan +0.95 0.84 nan nan nan +0.96 0.84 nan nan nan +0.96 0.84 nan nan nan +0.97 0.84 nan nan nan +0.98 0.84 nan nan nan +0.01 0.85 nan nan nan +0.01 0.85 nan nan nan +0.03 0.85 nan nan nan +0.04 0.85 nan nan nan +0.04 0.85 nan nan nan +0.06 0.85 nan nan nan +0.07 0.85 nan nan nan +0.07 0.85 nan nan nan +0.09 0.85 nan nan nan +0.10 0.85 nan nan nan +0.10 0.85 nan nan nan +0.12 0.85 nan nan nan +0.12 0.85 nan nan nan +0.14 0.85 nan nan nan +0.14 0.85 nan nan nan +0.15 0.85 nan nan nan +0.17 0.85 nan nan nan +0.18 0.85 nan nan nan +0.18 0.85 nan nan nan +0.20 0.85 nan nan nan +0.21 0.85 nan nan nan +0.21 0.85 nan nan nan +0.23 0.85 nan nan nan +0.24 0.85 nan nan nan +0.24 0.85 nan nan nan +0.26 0.85 nan nan nan +0.27 0.85 nan nan nan +0.28 0.85 nan nan nan +0.29 0.85 nan nan nan +0.29 0.85 nan nan nan +0.30 0.85 nan nan nan +0.32 0.85 nan nan nan +0.33 0.85 nan nan nan +0.34 0.85 nan nan nan +0.35 0.85 nan nan nan +0.35 0.85 nan nan nan +0.36 0.85 nan nan nan +0.38 0.85 nan nan nan +0.39 0.85 nan nan nan +0.40 0.85 nan nan nan +0.41 0.85 nan nan nan +0.42 0.85 nan nan nan +0.42 0.85 nan nan nan +0.43 0.85 nan nan nan +0.45 0.85 nan nan nan +0.46 0.85 nan nan nan +0.47 0.85 nan nan nan +0.48 0.85 nan nan nan +0.48 0.85 nan nan nan +0.49 0.85 nan nan nan +0.51 0.85 nan nan nan +0.52 0.85 nan nan nan +0.53 0.85 nan nan nan +0.54 0.85 nan nan nan +0.55 0.85 nan nan nan +0.56 0.85 nan nan nan +0.57 0.85 nan nan nan +0.58 0.85 nan nan nan +0.58 0.85 nan nan nan +0.59 0.85 nan nan nan +0.60 0.85 nan nan nan +0.61 0.85 nan nan nan +0.62 0.85 nan nan nan +0.64 0.85 nan nan nan +0.65 0.85 nan nan nan +0.66 0.85 nan nan nan +0.67 0.85 nan nan nan +0.68 0.85 nan nan nan +0.69 0.85 nan nan nan +0.70 0.85 nan nan nan +0.70 0.85 nan nan nan +0.71 0.85 nan nan nan +0.72 0.85 nan nan nan +0.73 0.85 nan nan nan +0.74 0.85 nan nan nan +0.76 0.85 nan nan nan +0.77 0.85 nan nan nan +0.78 0.85 nan nan nan +0.79 0.85 nan nan nan +0.80 0.85 nan nan nan +0.81 0.85 nan nan nan +0.82 0.85 nan nan nan +0.83 0.85 nan nan nan +0.83 0.85 nan nan nan +0.84 0.85 nan nan nan +0.85 0.85 nan nan nan +0.86 0.85 nan nan nan +0.88 0.85 nan nan nan +0.89 0.85 nan nan nan +0.90 0.85 nan nan nan +0.91 0.85 nan nan nan +0.92 0.85 nan nan nan +0.93 0.85 nan nan nan +0.94 0.85 nan nan nan +0.95 0.85 nan nan nan +0.96 0.85 nan nan nan +0.96 0.85 nan nan nan +0.97 0.85 nan nan nan +0.98 0.85 nan nan nan +0.01 0.86 nan nan nan +0.01 0.86 nan nan nan +0.03 0.86 nan nan nan +0.04 0.86 nan nan nan +0.04 0.86 nan nan nan +0.06 0.86 nan nan nan +0.07 0.86 nan nan nan +0.07 0.86 nan nan nan +0.09 0.86 nan nan nan +0.10 0.86 nan nan nan +0.10 0.86 nan nan nan +0.12 0.86 nan nan nan +0.12 0.86 nan nan nan +0.14 0.86 nan nan nan +0.14 0.86 nan nan nan +0.15 0.86 nan nan nan +0.17 0.86 nan nan nan +0.18 0.86 nan nan nan +0.18 0.86 nan nan nan +0.20 0.86 nan nan nan +0.21 0.86 nan nan nan +0.21 0.86 nan nan nan +0.23 0.86 nan nan nan +0.24 0.86 nan nan nan +0.24 0.86 nan nan nan +0.26 0.86 nan nan nan +0.27 0.86 nan nan nan +0.28 0.86 nan nan nan +0.29 0.86 nan nan nan +0.29 0.86 nan nan nan +0.30 0.86 nan nan nan +0.32 0.86 nan nan nan +0.33 0.86 nan nan nan +0.34 0.86 nan nan nan +0.35 0.86 nan nan nan +0.35 0.86 nan nan nan +0.36 0.86 nan nan nan +0.38 0.86 nan nan nan +0.39 0.86 nan nan nan +0.40 0.86 nan nan nan +0.41 0.86 nan nan nan +0.42 0.86 nan nan nan +0.42 0.86 nan nan nan +0.43 0.86 nan nan nan +0.45 0.86 nan nan nan +0.46 0.86 nan nan nan +0.47 0.86 nan nan nan +0.48 0.86 nan nan nan +0.48 0.86 nan nan nan +0.49 0.86 nan nan nan +0.51 0.86 nan nan nan +0.52 0.86 nan nan nan +0.53 0.86 nan nan nan +0.54 0.86 nan nan nan +0.55 0.86 nan nan nan +0.56 0.86 nan nan nan +0.57 0.86 nan nan nan +0.58 0.86 nan nan nan +0.58 0.86 nan nan nan +0.59 0.86 nan nan nan +0.60 0.86 nan nan nan +0.61 0.86 nan nan nan +0.62 0.86 nan nan nan +0.64 0.86 nan nan nan +0.65 0.86 nan nan nan +0.66 0.86 nan nan nan +0.67 0.86 nan nan nan +0.68 0.86 nan nan nan +0.69 0.86 nan nan nan +0.70 0.86 nan nan nan +0.70 0.86 nan nan nan +0.71 0.86 nan nan nan +0.72 0.86 nan nan nan +0.73 0.86 nan nan nan +0.74 0.86 nan nan nan +0.76 0.86 nan nan nan +0.77 0.86 nan nan nan +0.78 0.86 nan nan nan +0.79 0.86 nan nan nan +0.80 0.86 nan nan nan +0.81 0.86 nan nan nan +0.82 0.86 nan nan nan +0.83 0.86 nan nan nan +0.83 0.86 nan nan nan +0.84 0.86 nan nan nan +0.85 0.86 nan nan nan +0.86 0.86 nan nan nan +0.88 0.86 nan nan nan +0.89 0.86 nan nan nan +0.90 0.86 nan nan nan +0.91 0.86 nan nan nan +0.92 0.86 nan nan nan +0.93 0.86 nan nan nan +0.94 0.86 nan nan nan +0.95 0.86 nan nan nan +0.96 0.86 nan nan nan +0.96 0.86 nan nan nan +0.97 0.86 nan nan nan +0.98 0.86 nan nan nan +0.01 0.88 nan nan nan +0.01 0.88 nan nan nan +0.03 0.88 nan nan nan +0.04 0.88 nan nan nan +0.04 0.88 nan nan nan +0.06 0.88 nan nan nan +0.07 0.88 nan nan nan +0.07 0.88 nan nan nan +0.09 0.88 nan nan nan +0.10 0.88 nan nan nan +0.10 0.88 nan nan nan +0.12 0.88 nan nan nan +0.12 0.88 nan nan nan +0.14 0.88 nan nan nan +0.14 0.88 nan nan nan +0.15 0.88 nan nan nan +0.17 0.88 nan nan nan +0.18 0.88 nan nan nan +0.18 0.88 nan nan nan +0.20 0.88 nan nan nan +0.21 0.88 nan nan nan +0.21 0.88 nan nan nan +0.23 0.88 nan nan nan +0.24 0.88 nan nan nan +0.24 0.88 nan nan nan +0.26 0.88 nan nan nan +0.27 0.88 nan nan nan +0.28 0.88 nan nan nan +0.29 0.88 nan nan nan +0.29 0.88 nan nan nan +0.30 0.88 nan nan nan +0.32 0.88 nan nan nan +0.33 0.88 nan nan nan +0.34 0.88 nan nan nan +0.35 0.88 nan nan nan +0.35 0.88 nan nan nan +0.36 0.88 nan nan nan +0.38 0.88 nan nan nan +0.39 0.88 nan nan nan +0.40 0.88 nan nan nan +0.41 0.88 nan nan nan +0.42 0.88 nan nan nan +0.42 0.88 nan nan nan +0.43 0.88 nan nan nan +0.45 0.88 nan nan nan +0.46 0.88 nan nan nan +0.47 0.88 nan nan nan +0.48 0.88 nan nan nan +0.48 0.88 nan nan nan +0.49 0.88 nan nan nan +0.51 0.88 nan nan nan +0.52 0.88 nan nan nan +0.53 0.88 nan nan nan +0.54 0.88 nan nan nan +0.55 0.88 nan nan nan +0.56 0.88 nan nan nan +0.57 0.88 nan nan nan +0.58 0.88 nan nan nan +0.58 0.88 nan nan nan +0.59 0.88 nan nan nan +0.60 0.88 nan nan nan +0.61 0.88 nan nan nan +0.62 0.88 nan nan nan +0.64 0.88 nan nan nan +0.65 0.88 nan nan nan +0.66 0.88 nan nan nan +0.67 0.88 nan nan nan +0.68 0.88 nan nan nan +0.69 0.88 nan nan nan +0.70 0.88 nan nan nan +0.70 0.88 nan nan nan +0.71 0.88 nan nan nan +0.72 0.88 nan nan nan +0.73 0.88 nan nan nan +0.74 0.88 nan nan nan +0.76 0.88 nan nan nan +0.77 0.88 nan nan nan +0.78 0.88 nan nan nan +0.79 0.88 nan nan nan +0.80 0.88 nan nan nan +0.81 0.88 nan nan nan +0.82 0.88 nan nan nan +0.83 0.88 nan nan nan +0.83 0.88 nan nan nan +0.84 0.88 nan nan nan +0.85 0.88 nan nan nan +0.86 0.88 nan nan nan +0.88 0.88 nan nan nan +0.89 0.88 nan nan nan +0.90 0.88 nan nan nan +0.91 0.88 nan nan nan +0.92 0.88 nan nan nan +0.93 0.88 nan nan nan +0.94 0.88 nan nan nan +0.95 0.88 nan nan nan +0.96 0.88 nan nan nan +0.96 0.88 nan nan nan +0.97 0.88 nan nan nan +0.98 0.88 nan nan nan +0.01 0.89 nan nan nan +0.01 0.89 nan nan nan +0.03 0.89 nan nan nan +0.04 0.89 nan nan nan +0.04 0.89 nan nan nan +0.06 0.89 nan nan nan +0.07 0.89 nan nan nan +0.07 0.89 nan nan nan +0.09 0.89 nan nan nan +0.10 0.89 nan nan nan +0.10 0.89 nan nan nan +0.12 0.89 nan nan nan +0.12 0.89 nan nan nan +0.14 0.89 nan nan nan +0.14 0.89 nan nan nan +0.15 0.89 nan nan nan +0.17 0.89 nan nan nan +0.18 0.89 nan nan nan +0.18 0.89 nan nan nan +0.20 0.89 nan nan nan +0.21 0.89 nan nan nan +0.21 0.89 nan nan nan +0.23 0.89 nan nan nan +0.24 0.89 nan nan nan +0.24 0.89 nan nan nan +0.26 0.89 nan nan nan +0.27 0.89 nan nan nan +0.28 0.89 nan nan nan +0.29 0.89 nan nan nan +0.29 0.89 nan nan nan +0.30 0.89 nan nan nan +0.32 0.89 nan nan nan +0.33 0.89 nan nan nan +0.34 0.89 nan nan nan +0.35 0.89 nan nan nan +0.35 0.89 nan nan nan +0.36 0.89 nan nan nan +0.38 0.89 nan nan nan +0.39 0.89 nan nan nan +0.40 0.89 nan nan nan +0.41 0.89 nan nan nan +0.42 0.89 nan nan nan +0.42 0.89 nan nan nan +0.43 0.89 nan nan nan +0.45 0.89 nan nan nan +0.46 0.89 nan nan nan +0.47 0.89 nan nan nan +0.48 0.89 nan nan nan +0.48 0.89 nan nan nan +0.49 0.89 nan nan nan +0.51 0.89 nan nan nan +0.52 0.89 nan nan nan +0.53 0.89 nan nan nan +0.54 0.89 nan nan nan +0.55 0.89 nan nan nan +0.56 0.89 nan nan nan +0.57 0.89 nan nan nan +0.58 0.89 nan nan nan +0.58 0.89 nan nan nan +0.59 0.89 nan nan nan +0.60 0.89 nan nan nan +0.61 0.89 nan nan nan +0.62 0.89 nan nan nan +0.64 0.89 nan nan nan +0.65 0.89 nan nan nan +0.66 0.89 nan nan nan +0.67 0.89 nan nan nan +0.68 0.89 nan nan nan +0.69 0.89 nan nan nan +0.70 0.89 nan nan nan +0.70 0.89 nan nan nan +0.71 0.89 nan nan nan +0.72 0.89 nan nan nan +0.73 0.89 nan nan nan +0.74 0.89 nan nan nan +0.76 0.89 nan nan nan +0.77 0.89 nan nan nan +0.78 0.89 nan nan nan +0.79 0.89 nan nan nan +0.80 0.89 nan nan nan +0.81 0.89 nan nan nan +0.82 0.89 nan nan nan +0.83 0.89 nan nan nan +0.83 0.89 nan nan nan +0.84 0.89 nan nan nan +0.85 0.89 nan nan nan +0.86 0.89 nan nan nan +0.88 0.89 nan nan nan +0.89 0.89 nan nan nan +0.90 0.89 nan nan nan +0.91 0.89 nan nan nan +0.92 0.89 nan nan nan +0.93 0.89 nan nan nan +0.94 0.89 nan nan nan +0.95 0.89 nan nan nan +0.96 0.89 nan nan nan +0.96 0.89 nan nan nan +0.97 0.89 nan nan nan +0.98 0.89 nan nan nan +0.01 0.90 nan nan nan +0.01 0.90 nan nan nan +0.03 0.90 nan nan nan +0.04 0.90 nan nan nan +0.04 0.90 nan nan nan +0.06 0.90 nan nan nan +0.07 0.90 nan nan nan +0.07 0.90 nan nan nan +0.09 0.90 nan nan nan +0.10 0.90 nan nan nan +0.10 0.90 nan nan nan +0.12 0.90 nan nan nan +0.12 0.90 nan nan nan +0.14 0.90 nan nan nan +0.14 0.90 nan nan nan +0.15 0.90 nan nan nan +0.17 0.90 nan nan nan +0.18 0.90 nan nan nan +0.18 0.90 nan nan nan +0.20 0.90 nan nan nan +0.21 0.90 nan nan nan +0.21 0.90 nan nan nan +0.23 0.90 nan nan nan +0.24 0.90 nan nan nan +0.24 0.90 nan nan nan +0.26 0.90 nan nan nan +0.27 0.90 nan nan nan +0.28 0.90 nan nan nan +0.29 0.90 nan nan nan +0.29 0.90 nan nan nan +0.30 0.90 nan nan nan +0.32 0.90 nan nan nan +0.33 0.90 nan nan nan +0.34 0.90 nan nan nan +0.35 0.90 nan nan nan +0.35 0.90 nan nan nan +0.36 0.90 nan nan nan +0.38 0.90 nan nan nan +0.39 0.90 nan nan nan +0.40 0.90 nan nan nan +0.41 0.90 nan nan nan +0.42 0.90 nan nan nan +0.42 0.90 nan nan nan +0.43 0.90 nan nan nan +0.45 0.90 nan nan nan +0.46 0.90 nan nan nan +0.47 0.90 nan nan nan +0.48 0.90 nan nan nan +0.48 0.90 nan nan nan +0.49 0.90 nan nan nan +0.51 0.90 nan nan nan +0.52 0.90 nan nan nan +0.53 0.90 nan nan nan +0.54 0.90 nan nan nan +0.55 0.90 nan nan nan +0.56 0.90 nan nan nan +0.57 0.90 nan nan nan +0.58 0.90 nan nan nan +0.58 0.90 nan nan nan +0.59 0.90 nan nan nan +0.60 0.90 nan nan nan +0.61 0.90 nan nan nan +0.62 0.90 nan nan nan +0.64 0.90 nan nan nan +0.65 0.90 nan nan nan +0.66 0.90 nan nan nan +0.67 0.90 nan nan nan +0.68 0.90 nan nan nan +0.69 0.90 nan nan nan +0.70 0.90 nan nan nan +0.70 0.90 nan nan nan +0.71 0.90 nan nan nan +0.72 0.90 nan nan nan +0.73 0.90 nan nan nan +0.74 0.90 nan nan nan +0.76 0.90 nan nan nan +0.77 0.90 nan nan nan +0.78 0.90 nan nan nan +0.79 0.90 nan nan nan +0.80 0.90 nan nan nan +0.81 0.90 nan nan nan +0.82 0.90 nan nan nan +0.83 0.90 nan nan nan +0.83 0.90 nan nan nan +0.84 0.90 nan nan nan +0.85 0.90 nan nan nan +0.86 0.90 nan nan nan +0.88 0.90 nan nan nan +0.89 0.90 nan nan nan +0.90 0.90 nan nan nan +0.91 0.90 nan nan nan +0.92 0.90 nan nan nan +0.93 0.90 nan nan nan +0.94 0.90 nan nan nan +0.95 0.90 nan nan nan +0.96 0.90 nan nan nan +0.96 0.90 nan nan nan +0.97 0.90 nan nan nan +0.98 0.90 nan nan nan +0.01 0.91 nan nan nan +0.01 0.91 nan nan nan +0.03 0.91 nan nan nan +0.04 0.91 nan nan nan +0.04 0.91 nan nan nan +0.06 0.91 nan nan nan +0.07 0.91 nan nan nan +0.07 0.91 nan nan nan +0.09 0.91 nan nan nan +0.10 0.91 nan nan nan +0.10 0.91 nan nan nan +0.12 0.91 nan nan nan +0.12 0.91 nan nan nan +0.14 0.91 nan nan nan +0.14 0.91 nan nan nan +0.15 0.91 nan nan nan +0.17 0.91 nan nan nan +0.18 0.91 nan nan nan +0.18 0.91 nan nan nan +0.20 0.91 nan nan nan +0.21 0.91 nan nan nan +0.21 0.91 nan nan nan +0.23 0.91 nan nan nan +0.24 0.91 nan nan nan +0.24 0.91 nan nan nan +0.26 0.91 nan nan nan +0.27 0.91 nan nan nan +0.28 0.91 nan nan nan +0.29 0.91 nan nan nan +0.29 0.91 nan nan nan +0.30 0.91 nan nan nan +0.32 0.91 nan nan nan +0.33 0.91 nan nan nan +0.34 0.91 nan nan nan +0.35 0.91 nan nan nan +0.35 0.91 nan nan nan +0.36 0.91 nan nan nan +0.38 0.91 nan nan nan +0.39 0.91 nan nan nan +0.40 0.91 nan nan nan +0.41 0.91 nan nan nan +0.42 0.91 nan nan nan +0.42 0.91 nan nan nan +0.43 0.91 nan nan nan +0.45 0.91 nan nan nan +0.46 0.91 nan nan nan +0.47 0.91 nan nan nan +0.48 0.91 nan nan nan +0.48 0.91 nan nan nan +0.49 0.91 nan nan nan +0.51 0.91 nan nan nan +0.52 0.91 nan nan nan +0.53 0.91 nan nan nan +0.54 0.91 nan nan nan +0.55 0.91 nan nan nan +0.56 0.91 nan nan nan +0.57 0.91 nan nan nan +0.58 0.91 nan nan nan +0.58 0.91 nan nan nan +0.59 0.91 nan nan nan +0.60 0.91 nan nan nan +0.61 0.91 nan nan nan +0.62 0.91 nan nan nan +0.64 0.91 nan nan nan +0.65 0.91 nan nan nan +0.66 0.91 nan nan nan +0.67 0.91 nan nan nan +0.68 0.91 nan nan nan +0.69 0.91 nan nan nan +0.70 0.91 nan nan nan +0.70 0.91 nan nan nan +0.71 0.91 nan nan nan +0.72 0.91 nan nan nan +0.73 0.91 nan nan nan +0.74 0.91 nan nan nan +0.76 0.91 nan nan nan +0.77 0.91 nan nan nan +0.78 0.91 nan nan nan +0.79 0.91 nan nan nan +0.80 0.91 nan nan nan +0.81 0.91 nan nan nan +0.82 0.91 nan nan nan +0.83 0.91 nan nan nan +0.83 0.91 nan nan nan +0.84 0.91 nan nan nan +0.85 0.91 nan nan nan +0.86 0.91 nan nan nan +0.88 0.91 nan nan nan +0.89 0.91 nan nan nan +0.90 0.91 nan nan nan +0.91 0.91 nan nan nan +0.92 0.91 nan nan nan +0.93 0.91 nan nan nan +0.94 0.91 nan nan nan +0.95 0.91 nan nan nan +0.96 0.91 nan nan nan +0.96 0.91 nan nan nan +0.97 0.91 nan nan nan +0.98 0.91 nan nan nan +0.01 0.92 nan nan nan +0.01 0.92 nan nan nan +0.03 0.92 nan nan nan +0.04 0.92 nan nan nan +0.04 0.92 nan nan nan +0.06 0.92 nan nan nan +0.07 0.92 nan nan nan +0.07 0.92 nan nan nan +0.09 0.92 nan nan nan +0.10 0.92 nan nan nan +0.10 0.92 nan nan nan +0.12 0.92 nan nan nan +0.12 0.92 nan nan nan +0.14 0.92 nan nan nan +0.14 0.92 nan nan nan +0.15 0.92 nan nan nan +0.17 0.92 nan nan nan +0.18 0.92 nan nan nan +0.18 0.92 nan nan nan +0.20 0.92 nan nan nan +0.21 0.92 nan nan nan +0.21 0.92 nan nan nan +0.23 0.92 nan nan nan +0.24 0.92 nan nan nan +0.24 0.92 nan nan nan +0.26 0.92 nan nan nan +0.27 0.92 nan nan nan +0.28 0.92 nan nan nan +0.29 0.92 nan nan nan +0.29 0.92 nan nan nan +0.30 0.92 nan nan nan +0.32 0.92 nan nan nan +0.33 0.92 nan nan nan +0.34 0.92 nan nan nan +0.35 0.92 nan nan nan +0.35 0.92 nan nan nan +0.36 0.92 nan nan nan +0.38 0.92 nan nan nan +0.39 0.92 nan nan nan +0.40 0.92 nan nan nan +0.41 0.92 nan nan nan +0.42 0.92 nan nan nan +0.42 0.92 nan nan nan +0.43 0.92 nan nan nan +0.45 0.92 nan nan nan +0.46 0.92 nan nan nan +0.47 0.92 nan nan nan +0.48 0.92 nan nan nan +0.48 0.92 nan nan nan +0.49 0.92 nan nan nan +0.51 0.92 nan nan nan +0.52 0.92 nan nan nan +0.53 0.92 nan nan nan +0.54 0.92 nan nan nan +0.55 0.92 nan nan nan +0.56 0.92 nan nan nan +0.57 0.92 nan nan nan +0.58 0.92 nan nan nan +0.58 0.92 nan nan nan +0.59 0.92 nan nan nan +0.60 0.92 nan nan nan +0.61 0.92 nan nan nan +0.62 0.92 nan nan nan +0.64 0.92 nan nan nan +0.65 0.92 nan nan nan +0.66 0.92 nan nan nan +0.67 0.92 nan nan nan +0.68 0.92 nan nan nan +0.69 0.92 nan nan nan +0.70 0.92 nan nan nan +0.70 0.92 nan nan nan +0.71 0.92 nan nan nan +0.72 0.92 nan nan nan +0.73 0.92 nan nan nan +0.74 0.92 nan nan nan +0.76 0.92 nan nan nan +0.77 0.92 nan nan nan +0.78 0.92 nan nan nan +0.79 0.92 nan nan nan +0.80 0.92 nan nan nan +0.81 0.92 nan nan nan +0.82 0.92 nan nan nan +0.83 0.92 nan nan nan +0.83 0.92 nan nan nan +0.84 0.92 nan nan nan +0.85 0.92 nan nan nan +0.86 0.92 nan nan nan +0.88 0.92 nan nan nan +0.89 0.92 nan nan nan +0.90 0.92 nan nan nan +0.91 0.92 nan nan nan +0.92 0.92 nan nan nan +0.93 0.92 nan nan nan +0.94 0.92 nan nan nan +0.95 0.92 nan nan nan +0.96 0.92 nan nan nan +0.96 0.92 nan nan nan +0.97 0.92 nan nan nan +0.98 0.92 nan nan nan +0.01 0.93 nan nan nan +0.01 0.93 nan nan nan +0.03 0.93 nan nan nan +0.04 0.93 nan nan nan +0.04 0.93 nan nan nan +0.06 0.93 nan nan nan +0.07 0.93 nan nan nan +0.07 0.93 nan nan nan +0.09 0.93 nan nan nan +0.10 0.93 nan nan nan +0.10 0.93 nan nan nan +0.12 0.93 nan nan nan +0.12 0.93 nan nan nan +0.14 0.93 nan nan nan +0.14 0.93 nan nan nan +0.15 0.93 nan nan nan +0.17 0.93 nan nan nan +0.18 0.93 nan nan nan +0.18 0.93 nan nan nan +0.20 0.93 nan nan nan +0.21 0.93 nan nan nan +0.21 0.93 nan nan nan +0.23 0.93 nan nan nan +0.24 0.93 nan nan nan +0.24 0.93 nan nan nan +0.26 0.93 nan nan nan +0.27 0.93 nan nan nan +0.28 0.93 nan nan nan +0.29 0.93 nan nan nan +0.29 0.93 nan nan nan +0.30 0.93 nan nan nan +0.32 0.93 nan nan nan +0.33 0.93 nan nan nan +0.34 0.93 nan nan nan +0.35 0.93 nan nan nan +0.35 0.93 nan nan nan +0.36 0.93 nan nan nan +0.38 0.93 nan nan nan +0.39 0.93 nan nan nan +0.40 0.93 nan nan nan +0.41 0.93 nan nan nan +0.42 0.93 nan nan nan +0.42 0.93 nan nan nan +0.43 0.93 nan nan nan +0.45 0.93 nan nan nan +0.46 0.93 nan nan nan +0.47 0.93 nan nan nan +0.48 0.93 nan nan nan +0.48 0.93 nan nan nan +0.49 0.93 nan nan nan +0.51 0.93 nan nan nan +0.52 0.93 nan nan nan +0.53 0.93 nan nan nan +0.54 0.93 nan nan nan +0.55 0.93 nan nan nan +0.56 0.93 nan nan nan +0.57 0.93 nan nan nan +0.58 0.93 nan nan nan +0.58 0.93 nan nan nan +0.59 0.93 nan nan nan +0.60 0.93 nan nan nan +0.61 0.93 nan nan nan +0.62 0.93 nan nan nan +0.64 0.93 nan nan nan +0.65 0.93 nan nan nan +0.66 0.93 nan nan nan +0.67 0.93 nan nan nan +0.68 0.93 nan nan nan +0.69 0.93 nan nan nan +0.70 0.93 nan nan nan +0.70 0.93 nan nan nan +0.71 0.93 nan nan nan +0.72 0.93 nan nan nan +0.73 0.93 nan nan nan +0.74 0.93 nan nan nan +0.76 0.93 nan nan nan +0.77 0.93 nan nan nan +0.78 0.93 nan nan nan +0.79 0.93 nan nan nan +0.80 0.93 nan nan nan +0.81 0.93 nan nan nan +0.82 0.93 nan nan nan +0.83 0.93 nan nan nan +0.83 0.93 nan nan nan +0.84 0.93 nan nan nan +0.85 0.93 nan nan nan +0.86 0.93 nan nan nan +0.88 0.93 nan nan nan +0.89 0.93 nan nan nan +0.90 0.93 nan nan nan +0.91 0.93 nan nan nan +0.92 0.93 nan nan nan +0.93 0.93 nan nan nan +0.94 0.93 nan nan nan +0.95 0.93 nan nan nan +0.96 0.93 nan nan nan +0.96 0.93 nan nan nan +0.97 0.93 nan nan nan +0.98 0.93 nan nan nan +0.01 0.94 nan nan nan +0.01 0.94 nan nan nan +0.03 0.94 nan nan nan +0.04 0.94 nan nan nan +0.04 0.94 nan nan nan +0.06 0.94 nan nan nan +0.07 0.94 nan nan nan +0.07 0.94 nan nan nan +0.09 0.94 nan nan nan +0.10 0.94 nan nan nan +0.10 0.94 nan nan nan +0.12 0.94 nan nan nan +0.12 0.94 nan nan nan +0.14 0.94 nan nan nan +0.14 0.94 nan nan nan +0.15 0.94 nan nan nan +0.17 0.94 nan nan nan +0.18 0.94 nan nan nan +0.18 0.94 nan nan nan +0.20 0.94 nan nan nan +0.21 0.94 nan nan nan +0.21 0.94 nan nan nan +0.23 0.94 nan nan nan +0.24 0.94 nan nan nan +0.24 0.94 nan nan nan +0.26 0.94 nan nan nan +0.27 0.94 nan nan nan +0.28 0.94 nan nan nan +0.29 0.94 nan nan nan +0.29 0.94 nan nan nan +0.30 0.94 nan nan nan +0.32 0.94 nan nan nan +0.33 0.94 nan nan nan +0.34 0.94 nan nan nan +0.35 0.94 nan nan nan +0.35 0.94 nan nan nan +0.36 0.94 nan nan nan +0.38 0.94 nan nan nan +0.39 0.94 nan nan nan +0.40 0.94 nan nan nan +0.41 0.94 nan nan nan +0.42 0.94 nan nan nan +0.42 0.94 nan nan nan +0.43 0.94 nan nan nan +0.45 0.94 nan nan nan +0.46 0.94 nan nan nan +0.47 0.94 nan nan nan +0.48 0.94 nan nan nan +0.48 0.94 nan nan nan +0.49 0.94 nan nan nan +0.51 0.94 nan nan nan +0.52 0.94 nan nan nan +0.53 0.94 nan nan nan +0.54 0.94 nan nan nan +0.55 0.94 nan nan nan +0.56 0.94 nan nan nan +0.57 0.94 nan nan nan +0.58 0.94 nan nan nan +0.58 0.94 nan nan nan +0.59 0.94 nan nan nan +0.60 0.94 nan nan nan +0.61 0.94 nan nan nan +0.62 0.94 nan nan nan +0.64 0.94 nan nan nan +0.65 0.94 nan nan nan +0.66 0.94 nan nan nan +0.67 0.94 nan nan nan +0.68 0.94 nan nan nan +0.69 0.94 nan nan nan +0.70 0.94 nan nan nan +0.70 0.94 nan nan nan +0.71 0.94 nan nan nan +0.72 0.94 nan nan nan +0.73 0.94 nan nan nan +0.74 0.94 nan nan nan +0.76 0.94 nan nan nan +0.77 0.94 nan nan nan +0.78 0.94 nan nan nan +0.79 0.94 nan nan nan +0.80 0.94 nan nan nan +0.81 0.94 nan nan nan +0.82 0.94 nan nan nan +0.83 0.94 nan nan nan +0.83 0.94 nan nan nan +0.84 0.94 nan nan nan +0.85 0.94 nan nan nan +0.86 0.94 nan nan nan +0.88 0.94 nan nan nan +0.89 0.94 nan nan nan +0.90 0.94 nan nan nan +0.91 0.94 nan nan nan +0.92 0.94 nan nan nan +0.93 0.94 nan nan nan +0.94 0.94 nan nan nan +0.95 0.94 nan nan nan +0.96 0.94 nan nan nan +0.96 0.94 nan nan nan +0.97 0.94 nan nan nan +0.98 0.94 nan nan nan +0.01 0.95 nan nan nan +0.01 0.95 nan nan nan +0.03 0.95 nan nan nan +0.04 0.95 nan nan nan +0.04 0.95 nan nan nan +0.06 0.95 nan nan nan +0.07 0.95 nan nan nan +0.07 0.95 nan nan nan +0.09 0.95 nan nan nan +0.10 0.95 nan nan nan +0.10 0.95 nan nan nan +0.12 0.95 nan nan nan +0.12 0.95 nan nan nan +0.14 0.95 nan nan nan +0.14 0.95 nan nan nan +0.15 0.95 nan nan nan +0.17 0.95 nan nan nan +0.18 0.95 nan nan nan +0.18 0.95 nan nan nan +0.20 0.95 nan nan nan +0.21 0.95 nan nan nan +0.21 0.95 nan nan nan +0.23 0.95 nan nan nan +0.24 0.95 nan nan nan +0.24 0.95 nan nan nan +0.26 0.95 nan nan nan +0.27 0.95 nan nan nan +0.28 0.95 nan nan nan +0.29 0.95 nan nan nan +0.29 0.95 nan nan nan +0.30 0.95 nan nan nan +0.32 0.95 nan nan nan +0.33 0.95 nan nan nan +0.34 0.95 nan nan nan +0.35 0.95 nan nan nan +0.35 0.95 nan nan nan +0.36 0.95 nan nan nan +0.38 0.95 nan nan nan +0.39 0.95 nan nan nan +0.40 0.95 nan nan nan +0.41 0.95 nan nan nan +0.42 0.95 nan nan nan +0.42 0.95 nan nan nan +0.43 0.95 nan nan nan +0.45 0.95 nan nan nan +0.46 0.95 nan nan nan +0.47 0.95 nan nan nan +0.48 0.95 nan nan nan +0.48 0.95 nan nan nan +0.49 0.95 nan nan nan +0.51 0.95 nan nan nan +0.52 0.95 nan nan nan +0.53 0.95 nan nan nan +0.54 0.95 nan nan nan +0.55 0.95 nan nan nan +0.56 0.95 nan nan nan +0.57 0.95 nan nan nan +0.58 0.95 nan nan nan +0.58 0.95 nan nan nan +0.59 0.95 nan nan nan +0.60 0.95 nan nan nan +0.61 0.95 nan nan nan +0.62 0.95 nan nan nan +0.64 0.95 nan nan nan +0.65 0.95 nan nan nan +0.66 0.95 nan nan nan +0.67 0.95 nan nan nan +0.68 0.95 nan nan nan +0.69 0.95 nan nan nan +0.70 0.95 nan nan nan +0.70 0.95 nan nan nan +0.71 0.95 nan nan nan +0.72 0.95 nan nan nan +0.73 0.95 nan nan nan +0.74 0.95 nan nan nan +0.76 0.95 nan nan nan +0.77 0.95 nan nan nan +0.78 0.95 nan nan nan +0.79 0.95 nan nan nan +0.80 0.95 nan nan nan +0.81 0.95 nan nan nan +0.82 0.95 nan nan nan +0.83 0.95 nan nan nan +0.83 0.95 nan nan nan +0.84 0.95 nan nan nan +0.85 0.95 nan nan nan +0.86 0.95 nan nan nan +0.88 0.95 nan nan nan +0.89 0.95 nan nan nan +0.90 0.95 nan nan nan +0.91 0.95 nan nan nan +0.92 0.95 nan nan nan +0.93 0.95 nan nan nan +0.94 0.95 nan nan nan +0.95 0.95 nan nan nan +0.96 0.95 nan nan nan +0.96 0.95 nan nan nan +0.97 0.95 nan nan nan +0.98 0.95 nan nan nan +0.01 0.96 nan nan nan +0.01 0.96 nan nan nan +0.03 0.96 nan nan nan +0.04 0.96 nan nan nan +0.04 0.96 nan nan nan +0.06 0.96 nan nan nan +0.07 0.96 nan nan nan +0.07 0.96 nan nan nan +0.09 0.96 nan nan nan +0.10 0.96 nan nan nan +0.10 0.96 nan nan nan +0.12 0.96 nan nan nan +0.12 0.96 nan nan nan +0.14 0.96 nan nan nan +0.14 0.96 nan nan nan +0.15 0.96 nan nan nan +0.17 0.96 nan nan nan +0.18 0.96 nan nan nan +0.18 0.96 nan nan nan +0.20 0.96 nan nan nan +0.21 0.96 nan nan nan +0.21 0.96 nan nan nan +0.23 0.96 nan nan nan +0.24 0.96 nan nan nan +0.24 0.96 nan nan nan +0.26 0.96 nan nan nan +0.27 0.96 nan nan nan +0.28 0.96 nan nan nan +0.29 0.96 nan nan nan +0.29 0.96 nan nan nan +0.30 0.96 nan nan nan +0.32 0.96 nan nan nan +0.33 0.96 nan nan nan +0.34 0.96 nan nan nan +0.35 0.96 nan nan nan +0.35 0.96 nan nan nan +0.36 0.96 nan nan nan +0.38 0.96 nan nan nan +0.39 0.96 nan nan nan +0.40 0.96 nan nan nan +0.41 0.96 nan nan nan +0.42 0.96 nan nan nan +0.42 0.96 nan nan nan +0.43 0.96 nan nan nan +0.45 0.96 nan nan nan +0.46 0.96 nan nan nan +0.47 0.96 nan nan nan +0.48 0.96 nan nan nan +0.48 0.96 nan nan nan +0.49 0.96 nan nan nan +0.51 0.96 nan nan nan +0.52 0.96 nan nan nan +0.53 0.96 nan nan nan +0.54 0.96 nan nan nan +0.55 0.96 nan nan nan +0.56 0.96 nan nan nan +0.57 0.96 nan nan nan +0.58 0.96 nan nan nan +0.58 0.96 nan nan nan +0.59 0.96 nan nan nan +0.60 0.96 nan nan nan +0.61 0.96 nan nan nan +0.62 0.96 nan nan nan +0.64 0.96 nan nan nan +0.65 0.96 nan nan nan +0.66 0.96 nan nan nan +0.67 0.96 nan nan nan +0.68 0.96 nan nan nan +0.69 0.96 nan nan nan +0.70 0.96 nan nan nan +0.70 0.96 nan nan nan +0.71 0.96 nan nan nan +0.72 0.96 nan nan nan +0.73 0.96 nan nan nan +0.74 0.96 nan nan nan +0.76 0.96 nan nan nan +0.77 0.96 nan nan nan +0.78 0.96 nan nan nan +0.79 0.96 nan nan nan +0.80 0.96 nan nan nan +0.81 0.96 nan nan nan +0.82 0.96 nan nan nan +0.83 0.96 nan nan nan +0.83 0.96 nan nan nan +0.84 0.96 nan nan nan +0.85 0.96 nan nan nan +0.86 0.96 nan nan nan +0.88 0.96 nan nan nan +0.89 0.96 nan nan nan +0.90 0.96 nan nan nan +0.91 0.96 nan nan nan +0.92 0.96 nan nan nan +0.93 0.96 nan nan nan +0.94 0.96 nan nan nan +0.95 0.96 nan nan nan +0.96 0.96 nan nan nan +0.96 0.96 nan nan nan +0.97 0.96 nan nan nan +0.98 0.96 nan nan nan +0.01 0.96 nan nan nan +0.01 0.96 nan nan nan +0.03 0.96 nan nan nan +0.04 0.96 nan nan nan +0.04 0.96 nan nan nan +0.06 0.96 nan nan nan +0.07 0.96 nan nan nan +0.07 0.96 nan nan nan +0.09 0.96 nan nan nan +0.10 0.96 nan nan nan +0.10 0.96 nan nan nan +0.12 0.96 nan nan nan +0.12 0.96 nan nan nan +0.14 0.96 nan nan nan +0.14 0.96 nan nan nan +0.15 0.96 nan nan nan +0.17 0.96 nan nan nan +0.18 0.96 nan nan nan +0.18 0.96 nan nan nan +0.20 0.96 nan nan nan +0.21 0.96 nan nan nan +0.21 0.96 nan nan nan +0.23 0.96 nan nan nan +0.24 0.96 nan nan nan +0.24 0.96 nan nan nan +0.26 0.96 nan nan nan +0.27 0.96 nan nan nan +0.28 0.96 nan nan nan +0.29 0.96 nan nan nan +0.29 0.96 nan nan nan +0.30 0.96 nan nan nan +0.32 0.96 nan nan nan +0.33 0.96 nan nan nan +0.34 0.96 nan nan nan +0.35 0.96 nan nan nan +0.35 0.96 nan nan nan +0.36 0.96 nan nan nan +0.38 0.96 nan nan nan +0.39 0.96 nan nan nan +0.40 0.96 nan nan nan +0.41 0.96 nan nan nan +0.42 0.96 nan nan nan +0.42 0.96 nan nan nan +0.43 0.96 nan nan nan +0.45 0.96 nan nan nan +0.46 0.96 nan nan nan +0.47 0.96 nan nan nan +0.48 0.96 nan nan nan +0.48 0.96 nan nan nan +0.49 0.96 nan nan nan +0.51 0.96 nan nan nan +0.52 0.96 nan nan nan +0.53 0.96 nan nan nan +0.54 0.96 nan nan nan +0.55 0.96 nan nan nan +0.56 0.96 nan nan nan +0.57 0.96 nan nan nan +0.58 0.96 nan nan nan +0.58 0.96 nan nan nan +0.59 0.96 nan nan nan +0.60 0.96 nan nan nan +0.61 0.96 nan nan nan +0.62 0.96 nan nan nan +0.64 0.96 nan nan nan +0.65 0.96 nan nan nan +0.66 0.96 nan nan nan +0.67 0.96 nan nan nan +0.68 0.96 nan nan nan +0.69 0.96 nan nan nan +0.70 0.96 nan nan nan +0.70 0.96 nan nan nan +0.71 0.96 nan nan nan +0.72 0.96 nan nan nan +0.73 0.96 nan nan nan +0.74 0.96 nan nan nan +0.76 0.96 nan nan nan +0.77 0.96 nan nan nan +0.78 0.96 nan nan nan +0.79 0.96 nan nan nan +0.80 0.96 nan nan nan +0.81 0.96 nan nan nan +0.82 0.96 nan nan nan +0.83 0.96 nan nan nan +0.83 0.96 nan nan nan +0.84 0.96 nan nan nan +0.85 0.96 nan nan nan +0.86 0.96 nan nan nan +0.88 0.96 nan nan nan +0.89 0.96 nan nan nan +0.90 0.96 nan nan nan +0.91 0.96 nan nan nan +0.92 0.96 nan nan nan +0.93 0.96 nan nan nan +0.94 0.96 nan nan nan +0.95 0.96 nan nan nan +0.96 0.96 nan nan nan +0.96 0.96 nan nan nan +0.97 0.96 nan nan nan +0.98 0.96 nan nan nan +0.01 0.97 nan nan nan +0.01 0.97 nan nan nan +0.03 0.97 nan nan nan +0.04 0.97 nan nan nan +0.04 0.97 nan nan nan +0.06 0.97 nan nan nan +0.07 0.97 nan nan nan +0.07 0.97 nan nan nan +0.09 0.97 nan nan nan +0.10 0.97 nan nan nan +0.10 0.97 nan nan nan +0.12 0.97 nan nan nan +0.12 0.97 nan nan nan +0.14 0.97 nan nan nan +0.14 0.97 nan nan nan +0.15 0.97 nan nan nan +0.17 0.97 nan nan nan +0.18 0.97 nan nan nan +0.18 0.97 nan nan nan +0.20 0.97 nan nan nan +0.21 0.97 nan nan nan +0.21 0.97 nan nan nan +0.23 0.97 nan nan nan +0.24 0.97 nan nan nan +0.24 0.97 nan nan nan +0.26 0.97 nan nan nan +0.27 0.97 nan nan nan +0.28 0.97 nan nan nan +0.29 0.97 nan nan nan +0.29 0.97 nan nan nan +0.30 0.97 nan nan nan +0.32 0.97 nan nan nan +0.33 0.97 nan nan nan +0.34 0.97 nan nan nan +0.35 0.97 nan nan nan +0.35 0.97 nan nan nan +0.36 0.97 nan nan nan +0.38 0.97 nan nan nan +0.39 0.97 nan nan nan +0.40 0.97 nan nan nan +0.41 0.97 nan nan nan +0.42 0.97 nan nan nan +0.42 0.97 nan nan nan +0.43 0.97 nan nan nan +0.45 0.97 nan nan nan +0.46 0.97 nan nan nan +0.47 0.97 nan nan nan +0.48 0.97 nan nan nan +0.48 0.97 nan nan nan +0.49 0.97 nan nan nan +0.51 0.97 nan nan nan +0.52 0.97 nan nan nan +0.53 0.97 nan nan nan +0.54 0.97 nan nan nan +0.55 0.97 nan nan nan +0.56 0.97 nan nan nan +0.57 0.97 nan nan nan +0.58 0.97 nan nan nan +0.58 0.97 nan nan nan +0.59 0.97 nan nan nan +0.60 0.97 nan nan nan +0.61 0.97 nan nan nan +0.62 0.97 nan nan nan +0.64 0.97 nan nan nan +0.65 0.97 nan nan nan +0.66 0.97 nan nan nan +0.67 0.97 nan nan nan +0.68 0.97 nan nan nan +0.69 0.97 nan nan nan +0.70 0.97 nan nan nan +0.70 0.97 nan nan nan +0.71 0.97 nan nan nan +0.72 0.97 nan nan nan +0.73 0.97 nan nan nan +0.74 0.97 nan nan nan +0.76 0.97 nan nan nan +0.77 0.97 nan nan nan +0.78 0.97 nan nan nan +0.79 0.97 nan nan nan +0.80 0.97 nan nan nan +0.81 0.97 nan nan nan +0.82 0.97 nan nan nan +0.83 0.97 nan nan nan +0.83 0.97 nan nan nan +0.84 0.97 nan nan nan +0.85 0.97 nan nan nan +0.86 0.97 nan nan nan +0.88 0.97 nan nan nan +0.89 0.97 nan nan nan +0.90 0.97 nan nan nan +0.91 0.97 nan nan nan +0.92 0.97 nan nan nan +0.93 0.97 nan nan nan +0.94 0.97 nan nan nan +0.95 0.97 nan nan nan +0.96 0.97 nan nan nan +0.96 0.97 nan nan nan +0.97 0.97 nan nan nan +0.98 0.97 nan nan nan +0.01 0.98 nan nan nan +0.01 0.98 nan nan nan +0.03 0.98 nan nan nan +0.04 0.98 nan nan nan +0.04 0.98 nan nan nan +0.06 0.98 nan nan nan +0.07 0.98 nan nan nan +0.07 0.98 nan nan nan +0.09 0.98 nan nan nan +0.10 0.98 nan nan nan +0.10 0.98 nan nan nan +0.12 0.98 nan nan nan +0.12 0.98 nan nan nan +0.14 0.98 nan nan nan +0.14 0.98 nan nan nan +0.15 0.98 nan nan nan +0.17 0.98 nan nan nan +0.18 0.98 nan nan nan +0.18 0.98 nan nan nan +0.20 0.98 nan nan nan +0.21 0.98 nan nan nan +0.21 0.98 nan nan nan +0.23 0.98 nan nan nan +0.24 0.98 nan nan nan +0.24 0.98 nan nan nan +0.26 0.98 nan nan nan +0.27 0.98 nan nan nan +0.28 0.98 nan nan nan +0.29 0.98 nan nan nan +0.29 0.98 nan nan nan +0.30 0.98 nan nan nan +0.32 0.98 nan nan nan +0.33 0.98 nan nan nan +0.34 0.98 nan nan nan +0.35 0.98 nan nan nan +0.35 0.98 nan nan nan +0.36 0.98 nan nan nan +0.38 0.98 nan nan nan +0.39 0.98 nan nan nan +0.40 0.98 nan nan nan +0.41 0.98 nan nan nan +0.42 0.98 nan nan nan +0.42 0.98 nan nan nan +0.43 0.98 nan nan nan +0.45 0.98 nan nan nan +0.46 0.98 nan nan nan +0.47 0.98 nan nan nan +0.48 0.98 nan nan nan +0.48 0.98 nan nan nan +0.49 0.98 nan nan nan +0.51 0.98 nan nan nan +0.52 0.98 nan nan nan +0.53 0.98 nan nan nan +0.54 0.98 nan nan nan +0.55 0.98 nan nan nan +0.56 0.98 nan nan nan +0.57 0.98 nan nan nan +0.58 0.98 nan nan nan +0.58 0.98 nan nan nan +0.59 0.98 nan nan nan +0.60 0.98 nan nan nan +0.61 0.98 nan nan nan +0.62 0.98 nan nan nan +0.64 0.98 nan nan nan +0.65 0.98 nan nan nan +0.66 0.98 nan nan nan +0.67 0.98 nan nan nan +0.68 0.98 nan nan nan +0.69 0.98 nan nan nan +0.70 0.98 nan nan nan +0.70 0.98 nan nan nan +0.71 0.98 nan nan nan +0.72 0.98 nan nan nan +0.73 0.98 nan nan nan +0.74 0.98 nan nan nan +0.76 0.98 nan nan nan +0.77 0.98 nan nan nan +0.78 0.98 nan nan nan +0.79 0.98 nan nan nan +0.80 0.98 nan nan nan +0.81 0.98 nan nan nan +0.82 0.98 nan nan nan +0.83 0.98 nan nan nan +0.83 0.98 nan nan nan +0.84 0.98 nan nan nan +0.85 0.98 nan nan nan +0.86 0.98 nan nan nan +0.88 0.98 nan nan nan +0.89 0.98 nan nan nan +0.90 0.98 nan nan nan +0.91 0.98 nan nan nan +0.92 0.98 nan nan nan +0.93 0.98 nan nan nan +0.94 0.98 nan nan nan +0.95 0.98 nan nan nan +0.96 0.98 nan nan nan +0.96 0.98 nan nan nan +0.97 0.98 nan nan nan +0.98 0.98 nan nan nan +0000 0.000000 +0.86 0.58 0.000000 0.000000 0.000000 +0.88 0.58 0.000000 0.000000 0.000000 +0.89 0.58 0.000000 0.000000 0.000000 +0.90 0.58 0.000000 0.000000 0.000000 +0.91 0.58 0.000000 0.000000 0.000000 +0.92 0.58 0.000000 0.000000 0.000000 +0.93 0.58 0.000000 0.000000 0.000000 +0.94 0.58 0.000000 0.000000 0.000000 +0.95 0.58 0.000000 0.000000 0.000000 +0.96 0.58 0.000000 0.000000 0.000000 +0.96 0.58 0.000000 0.000000 0.000000 +0.97 0.58 0.000000 0.000000 0.000000 +0.98 0.58 0.000000 0.000000 0.000000 +0.01 0.59 0.000000 0.000000 0.000000 +0.01 0.59 0.000000 0.000000 0.000000 +0.03 0.59 0.000000 0.000000 0.000000 +0.04 0.59 0.000000 0.000000 0.000000 +0.04 0.59 0.000000 0.000000 0.000000 +0.06 0.59 0.000000 0.000000 0.000000 +0.07 0.59 0.000000 0.000000 0.000000 +0.07 0.59 0.000000 0.000000 0.000000 +0.09 0.59 0.000000 0.000000 0.000000 +0.10 0.59 0.000000 0.000000 0.000000 +0.10 0.59 0.000000 0.000000 0.000000 +0.12 0.59 0.000000 0.000000 0.000000 +0.12 0.59 0.000000 0.000000 0.000000 +0.14 0.59 0.000000 0.000000 0.000000 +0.14 0.59 0.000000 0.000000 0.000000 +0.15 0.59 0.000000 0.000000 0.000000 +0.17 0.59 0.000000 0.000000 0.000000 +0.18 0.59 0.000000 0.000000 0.000000 +0.18 0.59 0.000000 0.000000 0.000000 +0.20 0.59 0.000000 0.000000 0.000000 +0.21 0.59 0.000000 0.000000 0.000000 +0.21 0.59 0.000000 0.000000 0.000000 +0.23 0.59 0.000000 0.000000 0.000000 +0.24 0.59 0.000000 0.000000 0.000000 +0.24 0.59 0.000000 0.000000 0.000000 +0.26 0.59 0.000000 0.000000 0.000000 +0.27 0.59 0.000000 0.000000 0.000000 +0.28 0.59 0.000000 0.000000 0.000000 +0.29 0.59 0.000000 0.000000 0.000000 +0.29 0.59 0.000000 0.000000 0.000000 +0.30 0.59 0.000000 0.000000 0.000000 +0.32 0.59 0.000000 0.000000 0.000000 +0.33 0.59 0.000000 0.000000 0.000000 +0.34 0.59 0.000000 0.000000 0.000000 +0.35 0.59 0.000000 0.000000 0.000000 +0.35 0.59 0.000000 0.000000 0.000000 +0.36 0.59 0.000000 0.000000 0.000000 +0.38 0.59 0.000000 0.000000 0.000000 +0.39 0.59 0.000000 0.000000 0.000000 +0.40 0.59 0.000000 0.000000 0.000000 +0.41 0.59 0.000000 0.000000 0.000000 +0.42 0.59 0.000000 0.000000 0.000000 +0.42 0.59 0.000000 0.000000 0.000000 +0.43 0.59 0.000000 0.000000 0.000000 +0.45 0.59 0.000000 0.000000 0.000000 +0.46 0.59 0.000000 0.000000 0.000000 +0.47 0.59 0.000000 0.000000 0.000000 +0.48 0.59 0.000000 0.000000 0.000000 +0.48 0.59 0.000000 0.000000 0.000000 +0.49 0.59 0.000000 0.000000 0.000000 +0.51 0.59 0.000000 0.000000 0.000000 +0.52 0.59 0.000000 0.000000 0.000000 +0.53 0.59 0.000000 0.000000 0.000000 +0.54 0.59 0.000000 0.000000 0.000000 +0.55 0.59 0.000000 0.000000 0.000000 +0.56 0.59 0.000000 0.000000 0.000000 +0.57 0.59 0.000000 0.000000 0.000000 +0.58 0.59 0.000000 0.000000 0.000000 +0.58 0.59 0.000000 0.000000 0.000000 +0.59 0.59 0.000000 0.000000 0.000000 +0.60 0.59 0.000000 0.000000 0.000000 +0.61 0.59 0.000000 0.000000 0.000000 +0.62 0.59 0.000000 0.000000 0.000000 +0.64 0.59 0.000000 0.000000 0.000000 +0.65 0.59 0.000000 0.000000 0.000000 +0.66 0.59 0.000000 0.000000 0.000000 +0.67 0.59 0.000000 0.000000 0.000000 +0.68 0.59 0.000000 0.000000 0.000000 +0.69 0.59 0.000000 0.000000 0.000000 +0.70 0.59 0.000000 0.000000 0.000000 +0.70 0.59 0.000000 0.000000 0.000000 +0.71 0.59 0.000000 0.000000 0.000000 +0.72 0.59 0.000000 0.000000 0.000000 +0.73 0.59 0.000000 0.000000 0.000000 +0.74 0.59 0.000000 0.000000 0.000000 +0.76 0.59 0.000000 0.000000 0.000000 +0.77 0.59 0.000000 0.000000 0.000000 +0.78 0.59 0.000000 0.000000 0.000000 +0.79 0.59 0.000000 0.000000 0.000000 +0.80 0.59 0.000000 0.000000 0.000000 +0.81 0.59 0.000000 0.000000 0.000000 +0.82 0.59 0.000000 0.000000 0.000000 +0.83 0.59 0.000000 0.000000 0.000000 +0.83 0.59 0.000000 0.000000 0.000000 +0.84 0.59 0.000000 0.000000 0.000000 +0.85 0.59 0.000000 0.000000 0.000000 +0.86 0.59 0.000000 0.000000 0.000000 +0.88 0.59 0.000000 0.000000 0.000000 +0.89 0.59 0.000000 0.000000 0.000000 +0.90 0.59 0.000000 0.000000 0.000000 +0.91 0.59 0.000000 0.000000 0.000000 +0.92 0.59 0.000000 0.000000 0.000000 +0.93 0.59 0.000000 0.000000 0.000000 +0.94 0.59 0.000000 0.000000 0.000000 +0.95 0.59 0.000000 0.000000 0.000000 +0.96 0.59 0.000000 0.000000 0.000000 +0.96 0.59 0.000000 0.000000 0.000000 +0.97 0.59 0.000000 0.000000 0.000000 +0.98 0.59 0.000000 0.000000 0.000000 +0.01 0.60 0.000000 0.000000 0.000000 +0.01 0.60 0.000000 0.000000 0.000000 +0.03 0.60 0.000000 0.000000 0.000000 +0.04 0.60 0.000000 0.000000 0.000000 +0.04 0.60 0.000000 0.000000 0.000000 +0.06 0.60 0.000000 0.000000 0.000000 +0.07 0.60 0.000000 0.000000 0.000000 +0.07 0.60 0.000000 0.000000 0.000000 +0.09 0.60 0.000000 0.000000 0.000000 +0.10 0.60 0.000000 0.000000 0.000000 +0.10 0.60 0.000000 0.000000 0.000000 +0.12 0.60 0.000000 0.000000 0.000000 +0.12 0.60 0.000000 0.000000 0.000000 +0.14 0.60 0.000000 0.000000 0.000000 +0.14 0.60 0.000000 0.000000 0.000000 +0.15 0.60 0.000000 0.000000 0.000000 +0.17 0.60 0.000000 0.000000 0.000000 +0.18 0.60 0.000000 0.000000 0.000000 +0.18 0.60 0.000000 0.000000 0.000000 +0.20 0.60 0.000000 0.000000 0.000000 +0.21 0.60 0.000000 0.000000 0.000000 +0.21 0.60 0.000000 0.000000 0.000000 +0.23 0.60 0.000000 0.000000 0.000000 +0.24 0.60 0.000000 0.000000 0.000000 +0.24 0.60 0.000000 0.000000 0.000000 +0.26 0.60 0.000000 0.000000 0.000000 +0.27 0.60 0.000000 0.000000 0.000000 +0.28 0.60 0.000000 0.000000 0.000000 +0.29 0.60 0.000000 0.000000 0.000000 +0.29 0.60 0.000000 0.000000 0.000000 +0.30 0.60 0.000000 0.000000 0.000000 +0.32 0.60 0.000000 0.000000 0.000000 +0.33 0.60 0.000000 0.000000 0.000000 +0.34 0.60 0.000000 0.000000 0.000000 +0.35 0.60 0.000000 0.000000 0.000000 +0.35 0.60 0.000000 0.000000 0.000000 +0.36 0.60 0.000000 0.000000 0.000000 +0.38 0.60 0.000000 0.000000 0.000000 +0.39 0.60 0.000000 0.000000 0.000000 +0.40 0.60 0.000000 0.000000 0.000000 +0.41 0.60 0.000000 0.000000 0.000000 +0.42 0.60 0.000000 0.000000 0.000000 +0.42 0.60 0.000000 0.000000 0.000000 +0.43 0.60 0.000000 0.000000 0.000000 +0.45 0.60 0.000000 0.000000 0.000000 +0.46 0.60 0.000000 0.000000 0.000000 +0.47 0.60 0.000000 0.000000 0.000000 +0.48 0.60 0.000000 0.000000 0.000000 +0.48 0.60 0.000000 0.000000 0.000000 +0.49 0.60 0.000000 0.000000 0.000000 +0.51 0.60 0.000000 0.000000 0.000000 +0.52 0.60 0.000000 0.000000 0.000000 +0.53 0.60 0.000000 0.000000 0.000000 +0.54 0.60 0.000000 0.000000 0.000000 +0.55 0.60 0.000000 0.000000 0.000000 +0.56 0.60 0.000000 0.000000 0.000000 +0.57 0.60 0.000000 0.000000 0.000000 +0.58 0.60 0.000000 0.000000 0.000000 +0.58 0.60 0.000000 0.000000 0.000000 +0.59 0.60 0.000000 0.000000 0.000000 +0.60 0.60 0.000000 0.000000 0.000000 +0.61 0.60 0.000000 0.000000 0.000000 +0.62 0.60 0.000000 0.000000 0.000000 +0.64 0.60 0.000000 0.000000 0.000000 +0.65 0.60 0.000000 0.000000 0.000000 +0.66 0.60 0.000000 0.000000 0.000000 +0.67 0.60 0.000000 0.000000 0.000000 +0.68 0.60 0.000000 0.000000 0.000000 +0.69 0.60 0.000000 0.000000 0.000000 +0.70 0.60 0.000000 0.000000 0.000000 +0.70 0.60 0.000000 0.000000 0.000000 +0.71 0.60 0.000000 0.000000 0.000000 +0.72 0.60 0.000000 0.000000 0.000000 +0.73 0.60 0.000000 0.000000 0.000000 +0.74 0.60 0.000000 0.000000 0.000000 +0.76 0.60 0.000000 0.000000 0.000000 +0.77 0.60 0.000000 0.000000 0.000000 +0.78 0.60 0.000000 0.000000 0.000000 +0.79 0.60 0.000000 0.000000 0.000000 +0.80 0.60 0.000000 0.000000 0.000000 +0.81 0.60 0.000000 0.000000 0.000000 +0.82 0.60 0.000000 0.000000 0.000000 +0.83 0.60 0.000000 0.000000 0.000000 +0.83 0.60 0.000000 0.000000 0.000000 +0.84 0.60 0.000000 0.000000 0.000000 +0.85 0.60 0.000000 0.000000 0.000000 +0.86 0.60 0.000000 0.000000 0.000000 +0.88 0.60 0.000000 0.000000 0.000000 +0.89 0.60 0.000000 0.000000 0.000000 +0.90 0.60 0.000000 0.000000 0.000000 +0.91 0.60 0.000000 0.000000 0.000000 +0.92 0.60 0.000000 0.000000 0.000000 +0.93 0.60 0.000000 0.000000 0.000000 +0.94 0.60 0.000000 0.000000 0.000000 +0.95 0.60 0.000000 0.000000 0.000000 +0.96 0.60 0.000000 0.000000 0.000000 +0.96 0.60 0.000000 0.000000 0.000000 +0.97 0.60 0.000000 0.000000 0.000000 +0.98 0.60 0.000000 0.000000 0.000000 +0.01 0.61 0.000000 0.000000 0.000000 +0.01 0.61 0.000000 0.000000 0.000000 +0.03 0.61 0.000000 0.000000 0.000000 +0.04 0.61 0.000000 0.000000 0.000000 +0.04 0.61 0.000000 0.000000 0.000000 +0.06 0.61 0.000000 0.000000 0.000000 +0.07 0.61 0.000000 0.000000 0.000000 +0.07 0.61 0.000000 0.000000 0.000000 +0.09 0.61 0.000000 0.000000 0.000000 +0.10 0.61 0.000000 0.000000 0.000000 +0.10 0.61 0.000000 0.000000 0.000000 +0.12 0.61 0.000000 0.000000 0.000000 +0.12 0.61 0.000000 0.000000 0.000000 +0.14 0.61 0.000000 0.000000 0.000000 +0.14 0.61 0.000000 0.000000 0.000000 +0.15 0.61 0.000000 0.000000 0.000000 +0.17 0.61 0.000000 0.000000 0.000000 +0.18 0.61 0.000000 0.000000 0.000000 +0.18 0.61 0.000000 0.000000 0.000000 +0.20 0.61 0.000000 0.000000 0.000000 +0.21 0.61 0.000000 0.000000 0.000000 +0.21 0.61 0.000000 0.000000 0.000000 +0.23 0.61 0.000000 0.000000 0.000000 +0.24 0.61 0.000000 0.000000 0.000000 +0.24 0.61 0.000000 0.000000 0.000000 +0.26 0.61 0.000000 0.000000 0.000000 +0.27 0.61 0.000000 0.000000 0.000000 +0.28 0.61 0.000000 0.000000 0.000000 +0.29 0.61 0.000000 0.000000 0.000000 +0.29 0.61 0.000000 0.000000 0.000000 +0.30 0.61 0.000000 0.000000 0.000000 +0.32 0.61 0.000000 0.000000 0.000000 +0.33 0.61 0.000000 0.000000 0.000000 +0.34 0.61 0.000000 0.000000 0.000000 +0.35 0.61 0.000000 0.000000 0.000000 +0.35 0.61 0.000000 0.000000 0.000000 +0.36 0.61 0.000000 0.000000 0.000000 +0.38 0.61 0.000000 0.000000 0.000000 +0.39 0.61 0.000000 0.000000 0.000000 +0.40 0.61 0.000000 0.000000 0.000000 +0.41 0.61 0.000000 0.000000 0.000000 +0.42 0.61 0.000000 0.000000 0.000000 +0.42 0.61 0.000000 0.000000 0.000000 +0.43 0.61 0.000000 0.000000 0.000000 +0.45 0.61 0.000000 0.000000 0.000000 +0.46 0.61 0.000000 0.000000 0.000000 +0.47 0.61 0.000000 0.000000 0.000000 +0.48 0.61 0.000000 0.000000 0.000000 +0.48 0.61 0.000000 0.000000 0.000000 +0.49 0.61 0.000000 0.000000 0.000000 +0.51 0.61 0.000000 0.000000 0.000000 +0.52 0.61 0.000000 0.000000 0.000000 +0.53 0.61 0.000000 0.000000 0.000000 +0.54 0.61 0.000000 0.000000 0.000000 +0.55 0.61 0.000000 0.000000 0.000000 +0.56 0.61 0.000000 0.000000 0.000000 +0.57 0.61 0.000000 0.000000 0.000000 +0.58 0.61 0.000000 0.000000 0.000000 +0.58 0.61 0.000000 0.000000 0.000000 +0.59 0.61 0.000000 0.000000 0.000000 +0.60 0.61 0.000000 0.000000 0.000000 +0.61 0.61 0.000000 0.000000 0.000000 +0.62 0.61 0.000000 0.000000 0.000000 +0.64 0.61 0.000000 0.000000 0.000000 +0.65 0.61 0.000000 0.000000 0.000000 +0.66 0.61 0.000000 0.000000 0.000000 +0.67 0.61 0.000000 0.000000 0.000000 +0.68 0.61 0.000000 0.000000 0.000000 +0.69 0.61 0.000000 0.000000 0.000000 +0.70 0.61 0.000000 0.000000 0.000000 +0.70 0.61 0.000000 0.000000 0.000000 +0.71 0.61 0.000000 0.000000 0.000000 +0.72 0.61 0.000000 0.000000 0.000000 +0.73 0.61 0.000000 0.000000 0.000000 +0.74 0.61 0.000000 0.000000 0.000000 +0.76 0.61 0.000000 0.000000 0.000000 +0.77 0.61 0.000000 0.000000 0.000000 +0.78 0.61 0.000000 0.000000 0.000000 +0.79 0.61 0.000000 0.000000 0.000000 +0.80 0.61 0.000000 0.000000 0.000000 +0.81 0.61 0.000000 0.000000 0.000000 +0.82 0.61 0.000000 0.000000 0.000000 +0.83 0.61 0.000000 0.000000 0.000000 +0.83 0.61 0.000000 0.000000 0.000000 +0.84 0.61 0.000000 0.000000 0.000000 +0.85 0.61 0.000000 0.000000 0.000000 +0.86 0.61 0.000000 0.000000 0.000000 +0.88 0.61 0.000000 0.000000 0.000000 +0.89 0.61 0.000000 0.000000 0.000000 +0.90 0.61 0.000000 0.000000 0.000000 +0.91 0.61 0.000000 0.000000 0.000000 +0.92 0.61 0.000000 0.000000 0.000000 +0.93 0.61 0.000000 0.000000 0.000000 +0.94 0.61 0.000000 0.000000 0.000000 +0.95 0.61 0.000000 0.000000 0.000000 +0.96 0.61 0.000000 0.000000 0.000000 +0.96 0.61 0.000000 0.000000 0.000000 +0.97 0.61 0.000000 0.000000 0.000000 +0.98 0.61 0.000000 0.000000 0.000000 +0.01 0.62 0.000000 0.000000 0.000000 +0.01 0.62 0.000000 0.000000 0.000000 +0.03 0.62 0.000000 0.000000 0.000000 +0.04 0.62 0.000000 0.000000 0.000000 +0.04 0.62 0.000000 0.000000 0.000000 +0.06 0.62 0.000000 0.000000 0.000000 +0.07 0.62 0.000000 0.000000 0.000000 +0.07 0.62 0.000000 0.000000 0.000000 +0.09 0.62 0.000000 0.000000 0.000000 +0.10 0.62 0.000000 0.000000 0.000000 +0.10 0.62 0.000000 0.000000 0.000000 +0.12 0.62 0.000000 0.000000 0.000000 +0.12 0.62 0.000000 0.000000 0.000000 +0.14 0.62 0.000000 0.000000 0.000000 +0.14 0.62 0.000000 0.000000 0.000000 +0.15 0.62 0.000000 0.000000 0.000000 +0.17 0.62 0.000000 0.000000 0.000000 +0.18 0.62 0.000000 0.000000 0.000000 +0.18 0.62 0.000000 0.000000 0.000000 +0.20 0.62 0.000000 0.000000 0.000000 +0.21 0.62 0.000000 0.000000 0.000000 +0.21 0.62 0.000000 0.000000 0.000000 +0.23 0.62 0.000000 0.000000 0.000000 +0.24 0.62 0.000000 0.000000 0.000000 +0.24 0.62 0.000000 0.000000 0.000000 +0.26 0.62 0.000000 0.000000 0.000000 +0.27 0.62 0.000000 0.000000 0.000000 +0.28 0.62 0.000000 0.000000 0.000000 +0.29 0.62 0.000000 0.000000 0.000000 +0.29 0.62 0.000000 0.000000 0.000000 +0.30 0.62 0.000000 0.000000 0.000000 +0.32 0.62 0.000000 0.000000 0.000000 +0.33 0.62 0.000000 0.000000 0.000000 +0.34 0.62 0.000000 0.000000 0.000000 +0.35 0.62 0.000000 0.000000 0.000000 +0.35 0.62 0.000000 0.000000 0.000000 +0.36 0.62 0.000000 0.000000 0.000000 +0.38 0.62 0.000000 0.000000 0.000000 +0.39 0.62 0.000000 0.000000 0.000000 +0.40 0.62 0.000000 0.000000 0.000000 +0.41 0.62 0.000000 0.000000 0.000000 +0.42 0.62 0.000000 0.000000 0.000000 +0.42 0.62 0.000000 0.000000 0.000000 +0.43 0.62 0.000000 0.000000 0.000000 +0.45 0.62 0.000000 0.000000 0.000000 +0.46 0.62 0.000000 0.000000 0.000000 +0.47 0.62 0.000000 0.000000 0.000000 +0.48 0.62 0.000000 0.000000 0.000000 +0.48 0.62 0.000000 0.000000 0.000000 +0.49 0.62 0.000000 0.000000 0.000000 +0.51 0.62 0.000000 0.000000 0.000000 +0.52 0.62 0.000000 0.000000 0.000000 +0.53 0.62 0.000000 0.000000 0.000000 +0.54 0.62 0.000000 0.000000 0.000000 +0.55 0.62 0.000000 0.000000 0.000000 +0.56 0.62 0.000000 0.000000 0.000000 +0.57 0.62 0.000000 0.000000 0.000000 +0.58 0.62 0.000000 0.000000 0.000000 +0.58 0.62 0.000000 0.000000 0.000000 +0.59 0.62 0.000000 0.000000 0.000000 +0.60 0.62 0.000000 0.000000 0.000000 +0.61 0.62 0.000000 0.000000 0.000000 +0.62 0.62 0.000000 0.000000 0.000000 +0.64 0.62 0.000000 0.000000 0.000000 +0.65 0.62 0.000000 0.000000 0.000000 +0.66 0.62 0.000000 0.000000 0.000000 +0.67 0.62 0.000000 0.000000 0.000000 +0.68 0.62 0.000000 0.000000 0.000000 +0.69 0.62 0.000000 0.000000 0.000000 +0.70 0.62 0.000000 0.000000 0.000000 +0.70 0.62 0.000000 0.000000 0.000000 +0.71 0.62 0.000000 0.000000 0.000000 +0.72 0.62 0.000000 0.000000 0.000000 +0.73 0.62 0.000000 0.000000 0.000000 +0.74 0.62 0.000000 0.000000 0.000000 +0.76 0.62 0.000000 0.000000 0.000000 +0.77 0.62 0.000000 0.000000 0.000000 +0.78 0.62 0.000000 0.000000 0.000000 +0.79 0.62 0.000000 0.000000 0.000000 +0.80 0.62 0.000000 0.000000 0.000000 +0.81 0.62 0.000000 0.000000 0.000000 +0.82 0.62 0.000000 0.000000 0.000000 +0.83 0.62 0.000000 0.000000 0.000000 +0.83 0.62 0.000000 0.000000 0.000000 +0.84 0.62 0.000000 0.000000 0.000000 +0.85 0.62 0.000000 0.000000 0.000000 +0.86 0.62 0.000000 0.000000 0.000000 +0.88 0.62 0.000000 0.000000 0.000000 +0.89 0.62 0.000000 0.000000 0.000000 +0.90 0.62 0.000000 0.000000 0.000000 +0.91 0.62 0.000000 0.000000 0.000000 +0.92 0.62 0.000000 0.000000 0.000000 +0.93 0.62 0.000000 0.000000 0.000000 +0.94 0.62 0.000000 0.000000 0.000000 +0.95 0.62 0.000000 0.000000 0.000000 +0.96 0.62 0.000000 0.000000 0.000000 +0.96 0.62 0.000000 0.000000 0.000000 +0.97 0.62 0.000000 0.000000 0.000000 +0.98 0.62 0.000000 0.000000 0.000000 +0.01 0.64 0.000000 0.000000 0.000000 +0.01 0.64 0.000000 0.000000 0.000000 +0.03 0.64 0.000000 0.000000 0.000000 +0.04 0.64 0.000000 0.000000 0.000000 +0.04 0.64 0.000000 0.000000 0.000000 +0.06 0.64 0.000000 0.000000 0.000000 +0.07 0.64 0.000000 0.000000 0.000000 +0.07 0.64 0.000000 0.000000 0.000000 +0.09 0.64 0.000000 0.000000 0.000000 +0.10 0.64 0.000000 0.000000 0.000000 +0.10 0.64 0.000000 0.000000 0.000000 +0.12 0.64 0.000000 0.000000 0.000000 +0.12 0.64 0.000000 0.000000 0.000000 +0.14 0.64 0.000000 0.000000 0.000000 +0.14 0.64 0.000000 0.000000 0.000000 +0.15 0.64 0.000000 0.000000 0.000000 +0.17 0.64 0.000000 0.000000 0.000000 +0.18 0.64 0.000000 0.000000 0.000000 +0.18 0.64 0.000000 0.000000 0.000000 +0.20 0.64 0.000000 0.000000 0.000000 +0.21 0.64 0.000000 0.000000 0.000000 +0.21 0.64 0.000000 0.000000 0.000000 +0.23 0.64 0.000000 0.000000 0.000000 +0.24 0.64 0.000000 0.000000 0.000000 +0.24 0.64 0.000000 0.000000 0.000000 +0.26 0.64 0.000000 0.000000 0.000000 +0.27 0.64 0.000000 0.000000 0.000000 +0.28 0.64 0.000000 0.000000 0.000000 +0.29 0.64 0.000000 0.000000 0.000000 +0.29 0.64 0.000000 0.000000 0.000000 +0.30 0.64 0.000000 0.000000 0.000000 +0.32 0.64 0.000000 0.000000 0.000000 +0.33 0.64 0.000000 0.000000 0.000000 +0.34 0.64 0.000000 0.000000 0.000000 +0.35 0.64 0.000000 0.000000 0.000000 +0.35 0.64 0.000000 0.000000 0.000000 +0.36 0.64 0.000000 0.000000 0.000000 +0.38 0.64 0.000000 0.000000 0.000000 +0.39 0.64 0.000000 0.000000 0.000000 +0.40 0.64 0.000000 0.000000 0.000000 +0.41 0.64 0.000000 0.000000 0.000000 +0.42 0.64 0.000000 0.000000 0.000000 +0.42 0.64 0.000000 0.000000 0.000000 +0.43 0.64 0.000000 0.000000 0.000000 +0.45 0.64 0.000000 0.000000 0.000000 +0.46 0.64 0.000000 0.000000 0.000000 +0.47 0.64 0.000000 0.000000 0.000000 +0.48 0.64 0.000000 0.000000 0.000000 +0.48 0.64 0.000000 0.000000 0.000000 +0.49 0.64 0.000000 0.000000 0.000000 +0.51 0.64 0.000000 0.000000 0.000000 +0.52 0.64 0.000000 0.000000 0.000000 +0.53 0.64 0.000000 0.000000 0.000000 +0.54 0.64 0.000000 0.000000 0.000000 +0.55 0.64 0.000000 0.000000 0.000000 +0.56 0.64 0.000000 0.000000 0.000000 +0.57 0.64 0.000000 0.000000 0.000000 +0.58 0.64 0.000000 0.000000 0.000000 +0.58 0.64 0.000000 0.000000 0.000000 +0.59 0.64 0.000000 0.000000 0.000000 +0.60 0.64 0.000000 0.000000 0.000000 +0.61 0.64 0.000000 0.000000 0.000000 +0.62 0.64 0.000000 0.000000 0.000000 +0.64 0.64 0.000000 0.000000 0.000000 +0.65 0.64 0.000000 0.000000 0.000000 +0.66 0.64 0.000000 0.000000 0.000000 +0.67 0.64 0.000000 0.000000 0.000000 +0.68 0.64 0.000000 0.000000 0.000000 +0.69 0.64 0.000000 0.000000 0.000000 +0.70 0.64 0.000000 0.000000 0.000000 +0.70 0.64 0.000000 0.000000 0.000000 +0.71 0.64 0.000000 0.000000 0.000000 +0.72 0.64 0.000000 0.000000 0.000000 +0.73 0.64 0.000000 0.000000 0.000000 +0.74 0.64 0.000000 0.000000 0.000000 +0.76 0.64 0.000000 0.000000 0.000000 +0.77 0.64 0.000000 0.000000 0.000000 +0.78 0.64 0.000000 0.000000 0.000000 +0.79 0.64 0.000000 0.000000 0.000000 +0.80 0.64 0.000000 0.000000 0.000000 +0.81 0.64 0.000000 0.000000 0.000000 +0.82 0.64 0.000000 0.000000 0.000000 +0.83 0.64 0.000000 0.000000 0.000000 +0.83 0.64 0.000000 0.000000 0.000000 +0.84 0.64 0.000000 0.000000 0.000000 +0.85 0.64 0.000000 0.000000 0.000000 +0.86 0.64 0.000000 0.000000 0.000000 +0.88 0.64 0.000000 0.000000 0.000000 +0.89 0.64 0.000000 0.000000 0.000000 +0.90 0.64 0.000000 0.000000 0.000000 +0.91 0.64 0.000000 0.000000 0.000000 +0.92 0.64 0.000000 0.000000 0.000000 +0.93 0.64 0.000000 0.000000 0.000000 +0.94 0.64 0.000000 0.000000 0.000000 +0.95 0.64 0.000000 0.000000 0.000000 +0.96 0.64 0.000000 0.000000 0.000000 +0.96 0.64 0.000000 0.000000 0.000000 +0.97 0.64 0.000000 0.000000 0.000000 +0.98 0.64 0.000000 0.000000 0.000000 +0.01 0.65 0.000000 0.000000 0.000000 +0.01 0.65 0.000000 0.000000 0.000000 +0.03 0.65 0.000000 0.000000 0.000000 +0.04 0.65 0.000000 0.000000 0.000000 +0.04 0.65 0.000000 0.000000 0.000000 +0.06 0.65 0.000000 0.000000 0.000000 +0.07 0.65 0.000000 0.000000 0.000000 +0.07 0.65 0.000000 0.000000 0.000000 +0.09 0.65 0.000000 0.000000 0.000000 +0.10 0.65 0.000000 0.000000 0.000000 +0.10 0.65 0.000000 0.000000 0.000000 +0.12 0.65 0.000000 0.000000 0.000000 +0.12 0.65 0.000000 0.000000 0.000000 +0.14 0.65 0.000000 0.000000 0.000000 +0.14 0.65 0.000000 0.000000 0.000000 +0.15 0.65 0.000000 0.000000 0.000000 +0.17 0.65 0.000000 0.000000 0.000000 +0.18 0.65 0.000000 0.000000 0.000000 +0.18 0.65 0.000000 0.000000 0.000000 +0.20 0.65 0.000000 0.000000 0.000000 +0.21 0.65 0.000000 0.000000 0.000000 +0.21 0.65 0.000000 0.000000 0.000000 +0.23 0.65 0.000000 0.000000 0.000000 +0.24 0.65 0.000000 0.000000 0.000000 +0.24 0.65 0.000000 0.000000 0.000000 +0.26 0.65 0.000000 0.000000 0.000000 +0.27 0.65 0.000000 0.000000 0.000000 +0.28 0.65 0.000000 0.000000 0.000000 +0.29 0.65 0.000000 0.000000 0.000000 +0.29 0.65 0.000000 0.000000 0.000000 +0.30 0.65 0.000000 0.000000 0.000000 +0.32 0.65 0.000000 0.000000 0.000000 +0.33 0.65 0.000000 0.000000 0.000000 +0.34 0.65 0.000000 0.000000 0.000000 +0.35 0.65 0.000000 0.000000 0.000000 +0.35 0.65 0.000000 0.000000 0.000000 +0.36 0.65 0.000000 0.000000 0.000000 +0.38 0.65 0.000000 0.000000 0.000000 +0.39 0.65 0.000000 0.000000 0.000000 +0.40 0.65 0.000000 0.000000 0.000000 +0.41 0.65 0.000000 0.000000 0.000000 +0.42 0.65 0.000000 0.000000 0.000000 +0.42 0.65 0.000000 0.000000 0.000000 +0.43 0.65 0.000000 0.000000 0.000000 +0.45 0.65 0.000000 0.000000 0.000000 +0.46 0.65 0.000000 0.000000 0.000000 +0.47 0.65 0.000000 0.000000 0.000000 +0.48 0.65 0.000000 0.000000 0.000000 +0.48 0.65 0.000000 0.000000 0.000000 +0.49 0.65 0.000000 0.000000 0.000000 +0.51 0.65 0.000000 0.000000 0.000000 +0.52 0.65 0.000000 0.000000 0.000000 +0.53 0.65 0.000000 0.000000 0.000000 +0.54 0.65 0.000000 0.000000 0.000000 +0.55 0.65 0.000000 0.000000 0.000000 +0.56 0.65 0.000000 0.000000 0.000000 +0.57 0.65 0.000000 0.000000 0.000000 +0.58 0.65 0.000000 0.000000 0.000000 +0.58 0.65 0.000000 0.000000 0.000000 +0.59 0.65 0.000000 0.000000 0.000000 +0.60 0.65 0.000000 0.000000 0.000000 +0.61 0.65 0.000000 0.000000 0.000000 +0.62 0.65 0.000000 0.000000 0.000000 +0.64 0.65 0.000000 0.000000 0.000000 +0.65 0.65 0.000000 0.000000 0.000000 +0.66 0.65 0.000000 0.000000 0.000000 +0.67 0.65 0.000000 0.000000 0.000000 +0.68 0.65 0.000000 0.000000 0.000000 +0.69 0.65 0.000000 0.000000 0.000000 +0.70 0.65 0.000000 0.000000 0.000000 +0.70 0.65 0.000000 0.000000 0.000000 +0.71 0.65 0.000000 0.000000 0.000000 +0.72 0.65 0.000000 0.000000 0.000000 +0.73 0.65 0.000000 0.000000 0.000000 +0.74 0.65 0.000000 0.000000 0.000000 +0.76 0.65 0.000000 0.000000 0.000000 +0.77 0.65 0.000000 0.000000 0.000000 +0.78 0.65 0.000000 0.000000 0.000000 +0.79 0.65 0.000000 0.000000 0.000000 +0.80 0.65 0.000000 0.000000 0.000000 +0.81 0.65 0.000000 0.000000 0.000000 +0.82 0.65 0.000000 0.000000 0.000000 +0.83 0.65 0.000000 0.000000 0.000000 +0.83 0.65 0.000000 0.000000 0.000000 +0.84 0.65 0.000000 0.000000 0.000000 +0.85 0.65 0.000000 0.000000 0.000000 +0.86 0.65 0.000000 0.000000 0.000000 +0.88 0.65 0.000000 0.000000 0.000000 +0.89 0.65 0.000000 0.000000 0.000000 +0.90 0.65 0.000000 0.000000 0.000000 +0.91 0.65 0.000000 0.000000 0.000000 +0.92 0.65 0.000000 0.000000 0.000000 +0.93 0.65 0.000000 0.000000 0.000000 +0.94 0.65 0.000000 0.000000 0.000000 +0.95 0.65 0.000000 0.000000 0.000000 +0.96 0.65 0.000000 0.000000 0.000000 +0.96 0.65 0.000000 0.000000 0.000000 +0.97 0.65 0.000000 0.000000 0.000000 +0.98 0.65 0.000000 0.000000 0.000000 +0.01 0.66 0.000000 0.000000 0.000000 +0.01 0.66 0.000000 0.000000 0.000000 +0.03 0.66 0.000000 0.000000 0.000000 +0.04 0.66 0.000000 0.000000 0.000000 +0.04 0.66 0.000000 0.000000 0.000000 +0.06 0.66 0.000000 0.000000 0.000000 +0.07 0.66 0.000000 0.000000 0.000000 +0.07 0.66 0.000000 0.000000 0.000000 +0.09 0.66 0.000000 0.000000 0.000000 +0.10 0.66 0.000000 0.000000 0.000000 +0.10 0.66 0.000000 0.000000 0.000000 +0.12 0.66 0.000000 0.000000 0.000000 +0.12 0.66 0.000000 0.000000 0.000000 +0.14 0.66 0.000000 0.000000 0.000000 +0.14 0.66 0.000000 0.000000 0.000000 +0.15 0.66 0.000000 0.000000 0.000000 +0.17 0.66 0.000000 0.000000 0.000000 +0.18 0.66 0.000000 0.000000 0.000000 +0.18 0.66 0.000000 0.000000 0.000000 +0.20 0.66 0.000000 0.000000 0.000000 +0.21 0.66 0.000000 0.000000 0.000000 +0.21 0.66 0.000000 0.000000 0.000000 +0.23 0.66 0.000000 0.000000 0.000000 +0.24 0.66 0.000000 0.000000 0.000000 +0.24 0.66 0.000000 0.000000 0.000000 +0.26 0.66 0.000000 0.000000 0.000000 +0.27 0.66 0.000000 0.000000 0.000000 +0.28 0.66 0.000000 0.000000 0.000000 +0.29 0.66 0.000000 0.000000 0.000000 +0.29 0.66 0.000000 0.000000 0.000000 +0.30 0.66 0.000000 0.000000 0.000000 +0.32 0.66 0.000000 0.000000 0.000000 +0.33 0.66 0.000000 0.000000 0.000000 +0.34 0.66 0.000000 0.000000 0.000000 +0.35 0.66 0.000000 0.000000 0.000000 +0.35 0.66 0.000000 0.000000 0.000000 +0.36 0.66 0.000000 0.000000 0.000000 +0.38 0.66 0.000000 0.000000 0.000000 +0.39 0.66 0.000000 0.000000 0.000000 +0.40 0.66 0.000000 0.000000 0.000000 +0.41 0.66 0.000000 0.000000 0.000000 +0.42 0.66 0.000000 0.000000 0.000000 +0.42 0.66 0.000000 0.000000 0.000000 +0.43 0.66 0.000000 0.000000 0.000000 +0.45 0.66 0.000000 0.000000 0.000000 +0.46 0.66 0.000000 0.000000 0.000000 +0.47 0.66 0.000000 0.000000 0.000000 +0.48 0.66 0.000000 0.000000 0.000000 +0.48 0.66 0.000000 0.000000 0.000000 +0.49 0.66 0.000000 0.000000 0.000000 +0.51 0.66 0.000000 0.000000 0.000000 +0.52 0.66 0.000000 0.000000 0.000000 +0.53 0.66 0.000000 0.000000 0.000000 +0.54 0.66 0.000000 0.000000 0.000000 +0.55 0.66 0.000000 0.000000 0.000000 +0.56 0.66 0.000000 0.000000 0.000000 +0.57 0.66 0.000000 0.000000 0.000000 +0.58 0.66 0.000000 0.000000 0.000000 +0.58 0.66 0.000000 0.000000 0.000000 +0.59 0.66 0.000000 0.000000 0.000000 +0.60 0.66 0.000000 0.000000 0.000000 +0.61 0.66 0.000000 0.000000 0.000000 +0.62 0.66 0.000000 0.000000 0.000000 +0.64 0.66 0.000000 0.000000 0.000000 +0.65 0.66 0.000000 0.000000 0.000000 +0.66 0.66 0.000000 0.000000 0.000000 +0.67 0.66 0.000000 0.000000 0.000000 +0.68 0.66 0.000000 0.000000 0.000000 +0.69 0.66 0.000000 0.000000 0.000000 +0.70 0.66 0.000000 0.000000 0.000000 +0.70 0.66 0.000000 0.000000 0.000000 +0.71 0.66 0.000000 0.000000 0.000000 +0.72 0.66 0.000000 0.000000 0.000000 +0.73 0.66 0.000000 0.000000 0.000000 +0.74 0.66 0.000000 0.000000 0.000000 +0.76 0.66 0.000000 0.000000 0.000000 +0.77 0.66 0.000000 0.000000 0.000000 +0.78 0.66 0.000000 0.000000 0.000000 +0.79 0.66 0.000000 0.000000 0.000000 +0.80 0.66 0.000000 0.000000 0.000000 +0.81 0.66 0.000000 0.000000 0.000000 +0.82 0.66 0.000000 0.000000 0.000000 +0.83 0.66 0.000000 0.000000 0.000000 +0.83 0.66 0.000000 0.000000 0.000000 +0.84 0.66 0.000000 0.000000 0.000000 +0.85 0.66 0.000000 0.000000 0.000000 +0.86 0.66 0.000000 0.000000 0.000000 +0.88 0.66 0.000000 0.000000 0.000000 +0.89 0.66 0.000000 0.000000 0.000000 +0.90 0.66 0.000000 0.000000 0.000000 +0.91 0.66 0.000000 0.000000 0.000000 +0.92 0.66 0.000000 0.000000 0.000000 +0.93 0.66 0.000000 0.000000 0.000000 +0.94 0.66 0.000000 0.000000 0.000000 +0.95 0.66 0.000000 0.000000 0.000000 +0.96 0.66 0.000000 0.000000 0.000000 +0.96 0.66 0.000000 0.000000 0.000000 +0.97 0.66 0.000000 0.000000 0.000000 +0.98 0.66 0.000000 0.000000 0.000000 +0.01 0.67 0.000000 0.000000 0.000000 +0.01 0.67 0.000000 0.000000 0.000000 +0.03 0.67 0.000000 0.000000 0.000000 +0.04 0.67 0.000000 0.000000 0.000000 +0.04 0.67 0.000000 0.000000 0.000000 +0.06 0.67 0.000000 0.000000 0.000000 +0.07 0.67 0.000000 0.000000 0.000000 +0.07 0.67 0.000000 0.000000 0.000000 +0.09 0.67 0.000000 0.000000 0.000000 +0.10 0.67 0.000000 0.000000 0.000000 +0.10 0.67 0.000000 0.000000 0.000000 +0.12 0.67 0.000000 0.000000 0.000000 +0.12 0.67 0.000000 0.000000 0.000000 +0.14 0.67 0.000000 0.000000 0.000000 +0.14 0.67 0.000000 0.000000 0.000000 +0.15 0.67 0.000000 0.000000 0.000000 +0.17 0.67 0.000000 0.000000 0.000000 +0.18 0.67 0.000000 0.000000 0.000000 +0.18 0.67 0.000000 0.000000 0.000000 +0.20 0.67 0.000000 0.000000 0.000000 +0.21 0.67 0.000000 0.000000 0.000000 +0.21 0.67 0.000000 0.000000 0.000000 +0.23 0.67 0.000000 0.000000 0.000000 +0.24 0.67 0.000000 0.000000 0.000000 +0.24 0.67 0.000000 0.000000 0.000000 +0.26 0.67 0.000000 0.000000 0.000000 +0.27 0.67 0.000000 0.000000 0.000000 +0.28 0.67 0.000000 0.000000 0.000000 +0.29 0.67 0.000000 0.000000 0.000000 +0.29 0.67 0.000000 0.000000 0.000000 +0.30 0.67 0.000000 0.000000 0.000000 +0.32 0.67 0.000000 0.000000 0.000000 +0.33 0.67 0.000000 0.000000 0.000000 +0.34 0.67 0.000000 0.000000 0.000000 +0.35 0.67 0.000000 0.000000 0.000000 +0.35 0.67 0.000000 0.000000 0.000000 +0.36 0.67 0.000000 0.000000 0.000000 +0.38 0.67 0.000000 0.000000 0.000000 +0.39 0.67 0.000000 0.000000 0.000000 +0.40 0.67 0.000000 0.000000 0.000000 +0.41 0.67 0.000000 0.000000 0.000000 +0.42 0.67 0.000000 0.000000 0.000000 +0.42 0.67 0.000000 0.000000 0.000000 +0.43 0.67 0.000000 0.000000 0.000000 +0.45 0.67 0.000000 0.000000 0.000000 +0.46 0.67 0.000000 0.000000 0.000000 +0.47 0.67 0.000000 0.000000 0.000000 +0.48 0.67 0.000000 0.000000 0.000000 +0.48 0.67 0.000000 0.000000 0.000000 +0.49 0.67 0.000000 0.000000 0.000000 +0.51 0.67 0.000000 0.000000 0.000000 +0.52 0.67 0.000000 0.000000 0.000000 +0.53 0.67 0.000000 0.000000 0.000000 +0.54 0.67 0.000000 0.000000 0.000000 +0.55 0.67 0.000000 0.000000 0.000000 +0.56 0.67 0.000000 0.000000 0.000000 +0.57 0.67 0.000000 0.000000 0.000000 +0.58 0.67 0.000000 0.000000 0.000000 +0.58 0.67 0.000000 0.000000 0.000000 +0.59 0.67 0.000000 0.000000 0.000000 +0.60 0.67 0.000000 0.000000 0.000000 +0.61 0.67 0.000000 0.000000 0.000000 +0.62 0.67 0.000000 0.000000 0.000000 +0.64 0.67 0.000000 0.000000 0.000000 +0.65 0.67 0.000000 0.000000 0.000000 +0.66 0.67 0.000000 0.000000 0.000000 +0.67 0.67 0.000000 0.000000 0.000000 +0.68 0.67 0.000000 0.000000 0.000000 +0.69 0.67 0.000000 0.000000 0.000000 +0.70 0.67 0.000000 0.000000 0.000000 +0.70 0.67 0.000000 0.000000 0.000000 +0.71 0.67 0.000000 0.000000 0.000000 +0.72 0.67 0.000000 0.000000 0.000000 +0.73 0.67 0.000000 0.000000 0.000000 +0.74 0.67 0.000000 0.000000 0.000000 +0.76 0.67 0.000000 0.000000 0.000000 +0.77 0.67 0.000000 0.000000 0.000000 +0.78 0.67 0.000000 0.000000 0.000000 +0.79 0.67 0.000000 0.000000 0.000000 +0.80 0.67 0.000000 0.000000 0.000000 +0.81 0.67 0.000000 0.000000 0.000000 +0.82 0.67 0.000000 0.000000 0.000000 +0.83 0.67 0.000000 0.000000 0.000000 +0.83 0.67 0.000000 0.000000 0.000000 +0.84 0.67 0.000000 0.000000 0.000000 +0.85 0.67 0.000000 0.000000 0.000000 +0.86 0.67 0.000000 0.000000 0.000000 +0.88 0.67 0.000000 0.000000 0.000000 +0.89 0.67 0.000000 0.000000 0.000000 +0.90 0.67 0.000000 0.000000 0.000000 +0.91 0.67 0.000000 0.000000 0.000000 +0.92 0.67 0.000000 0.000000 0.000000 +0.93 0.67 0.000000 0.000000 0.000000 +0.94 0.67 0.000000 0.000000 0.000000 +0.95 0.67 0.000000 0.000000 0.000000 +0.96 0.67 0.000000 0.000000 0.000000 +0.96 0.67 0.000000 0.000000 0.000000 +0.97 0.67 0.000000 0.000000 0.000000 +0.98 0.67 0.000000 0.000000 0.000000 +0.01 0.68 0.000000 0.000000 0.000000 +0.01 0.68 0.000000 0.000000 0.000000 +0.03 0.68 0.000000 0.000000 0.000000 +0.04 0.68 0.000000 0.000000 0.000000 +0.04 0.68 0.000000 0.000000 0.000000 +0.06 0.68 0.000000 0.000000 0.000000 +0.07 0.68 0.000000 0.000000 0.000000 +0.07 0.68 0.000000 0.000000 0.000000 +0.09 0.68 0.000000 0.000000 0.000000 +0.10 0.68 0.000000 0.000000 0.000000 +0.10 0.68 0.000000 0.000000 0.000000 +0.12 0.68 0.000000 0.000000 0.000000 +0.12 0.68 0.000000 0.000000 0.000000 +0.14 0.68 0.000000 0.000000 0.000000 +0.14 0.68 0.000000 0.000000 0.000000 +0.15 0.68 0.000000 0.000000 0.000000 +0.17 0.68 0.000000 0.000000 0.000000 +0.18 0.68 0.000000 0.000000 0.000000 +0.18 0.68 0.000000 0.000000 0.000000 +0.20 0.68 0.000000 0.000000 0.000000 +0.21 0.68 0.000000 0.000000 0.000000 +0.21 0.68 0.000000 0.000000 0.000000 +0.23 0.68 0.000000 0.000000 0.000000 +0.24 0.68 0.000000 0.000000 0.000000 +0.24 0.68 0.000000 0.000000 0.000000 +0.26 0.68 0.000000 0.000000 0.000000 +0.27 0.68 0.000000 0.000000 0.000000 +0.28 0.68 0.000000 0.000000 0.000000 +0.29 0.68 0.000000 0.000000 0.000000 +0.29 0.68 0.000000 0.000000 0.000000 +0.30 0.68 0.000000 0.000000 0.000000 +0.32 0.68 0.000000 0.000000 0.000000 +0.33 0.68 0.000000 0.000000 0.000000 +0.34 0.68 0.000000 0.000000 0.000000 +0.35 0.68 0.000000 0.000000 0.000000 +0.35 0.68 0.000000 0.000000 0.000000 +0.36 0.68 0.000000 0.000000 0.000000 +0.38 0.68 0.000000 0.000000 0.000000 +0.39 0.68 0.000000 0.000000 0.000000 +0.40 0.68 0.000000 0.000000 0.000000 +0.41 0.68 0.000000 0.000000 0.000000 +0.42 0.68 0.000000 0.000000 0.000000 +0.42 0.68 0.000000 0.000000 0.000000 +0.43 0.68 0.000000 0.000000 0.000000 +0.45 0.68 0.000000 0.000000 0.000000 +0.46 0.68 0.000000 0.000000 0.000000 +0.47 0.68 0.000000 0.000000 0.000000 +0.48 0.68 0.000000 0.000000 0.000000 +0.48 0.68 0.000000 0.000000 0.000000 +0.49 0.68 0.000000 0.000000 0.000000 +0.51 0.68 0.000000 0.000000 0.000000 +0.52 0.68 0.000000 0.000000 0.000000 +0.53 0.68 0.000000 0.000000 0.000000 +0.54 0.68 0.000000 0.000000 0.000000 +0.55 0.68 0.000000 0.000000 0.000000 +0.56 0.68 0.000000 0.000000 0.000000 +0.57 0.68 0.000000 0.000000 0.000000 +0.58 0.68 0.000000 0.000000 0.000000 +0.58 0.68 0.000000 0.000000 0.000000 +0.59 0.68 0.000000 0.000000 0.000000 +0.60 0.68 0.000000 0.000000 0.000000 +0.61 0.68 0.000000 0.000000 0.000000 +0.62 0.68 0.000000 0.000000 0.000000 +0.64 0.68 0.000000 0.000000 0.000000 +0.65 0.68 0.000000 0.000000 0.000000 +0.66 0.68 0.000000 0.000000 0.000000 +0.67 0.68 0.000000 0.000000 0.000000 +0.68 0.68 0.000000 0.000000 0.000000 +0.69 0.68 0.000000 0.000000 0.000000 +0.70 0.68 0.000000 0.000000 0.000000 +0.70 0.68 0.000000 0.000000 0.000000 +0.71 0.68 0.000000 0.000000 0.000000 +0.72 0.68 0.000000 0.000000 0.000000 +0.73 0.68 0.000000 0.000000 0.000000 +0.74 0.68 0.000000 0.000000 0.000000 +0.76 0.68 0.000000 0.000000 0.000000 +0.77 0.68 0.000000 0.000000 0.000000 +0.78 0.68 0.000000 0.000000 0.000000 +0.79 0.68 0.000000 0.000000 0.000000 +0.80 0.68 0.000000 0.000000 0.000000 +0.81 0.68 0.000000 0.000000 0.000000 +0.82 0.68 0.000000 0.000000 0.000000 +0.83 0.68 0.000000 0.000000 0.000000 +0.83 0.68 0.000000 0.000000 0.000000 +0.84 0.68 0.000000 0.000000 0.000000 +0.85 0.68 0.000000 0.000000 0.000000 +0.86 0.68 0.000000 0.000000 0.000000 +0.88 0.68 0.000000 0.000000 0.000000 +0.89 0.68 0.000000 0.000000 0.000000 +0.90 0.68 0.000000 0.000000 0.000000 +0.91 0.68 0.000000 0.000000 0.000000 +0.92 0.68 0.000000 0.000000 0.000000 +0.93 0.68 0.000000 0.000000 0.000000 +0.94 0.68 0.000000 0.000000 0.000000 +0.95 0.68 0.000000 0.000000 0.000000 +0.96 0.68 0.000000 0.000000 0.000000 +0.96 0.68 0.000000 0.000000 0.000000 +0.97 0.68 0.000000 0.000000 0.000000 +0.98 0.68 0.000000 0.000000 0.000000 +0.01 0.69 0.000000 0.000000 0.000000 +0.01 0.69 0.000000 0.000000 0.000000 +0.03 0.69 0.000000 0.000000 0.000000 +0.04 0.69 0.000000 0.000000 0.000000 +0.04 0.69 0.000000 0.000000 0.000000 +0.06 0.69 0.000000 0.000000 0.000000 +0.07 0.69 0.000000 0.000000 0.000000 +0.07 0.69 0.000000 0.000000 0.000000 +0.09 0.69 0.000000 0.000000 0.000000 +0.10 0.69 0.000000 0.000000 0.000000 +0.10 0.69 0.000000 0.000000 0.000000 +0.12 0.69 0.000000 0.000000 0.000000 +0.12 0.69 0.000000 0.000000 0.000000 +0.14 0.69 0.000000 0.000000 0.000000 +0.14 0.69 0.000000 0.000000 0.000000 +0.15 0.69 0.000000 0.000000 0.000000 +0.17 0.69 0.000000 0.000000 0.000000 +0.18 0.69 0.000000 0.000000 0.000000 +0.18 0.69 0.000000 0.000000 0.000000 +0.20 0.69 0.000000 0.000000 0.000000 +0.21 0.69 0.000000 0.000000 0.000000 +0.21 0.69 0.000000 0.000000 0.000000 +0.23 0.69 0.000000 0.000000 0.000000 +0.24 0.69 0.000000 0.000000 0.000000 +0.24 0.69 0.000000 0.000000 0.000000 +0.26 0.69 0.000000 0.000000 0.000000 +0.27 0.69 0.000000 0.000000 0.000000 +0.28 0.69 0.000000 0.000000 0.000000 +0.29 0.69 0.000000 0.000000 0.000000 +0.29 0.69 0.000000 0.000000 0.000000 +0.30 0.69 0.000000 0.000000 0.000000 +0.32 0.69 0.000000 0.000000 0.000000 +0.33 0.69 0.000000 0.000000 0.000000 +0.34 0.69 0.000000 0.000000 0.000000 +0.35 0.69 0.000000 0.000000 0.000000 +0.35 0.69 0.000000 0.000000 0.000000 +0.36 0.69 0.000000 0.000000 0.000000 +0.38 0.69 0.000000 0.000000 0.000000 +0.39 0.69 0.000000 0.000000 0.000000 +0.40 0.69 0.000000 0.000000 0.000000 +0.41 0.69 0.000000 0.000000 0.000000 +0.42 0.69 0.000000 0.000000 0.000000 +0.42 0.69 0.000000 0.000000 0.000000 +0.43 0.69 0.000000 0.000000 0.000000 +0.45 0.69 0.000000 0.000000 0.000000 +0.46 0.69 0.000000 0.000000 0.000000 +0.47 0.69 0.000000 0.000000 0.000000 +0.48 0.69 0.000000 0.000000 0.000000 +0.48 0.69 0.000000 0.000000 0.000000 +0.49 0.69 0.000000 0.000000 0.000000 +0.51 0.69 0.000000 0.000000 0.000000 +0.52 0.69 0.000000 0.000000 0.000000 +0.53 0.69 0.000000 0.000000 0.000000 +0.54 0.69 0.000000 0.000000 0.000000 +0.55 0.69 0.000000 0.000000 0.000000 +0.56 0.69 0.000000 0.000000 0.000000 +0.57 0.69 0.000000 0.000000 0.000000 +0.58 0.69 0.000000 0.000000 0.000000 +0.58 0.69 0.000000 0.000000 0.000000 +0.59 0.69 0.000000 0.000000 0.000000 +0.60 0.69 0.000000 0.000000 0.000000 +0.61 0.69 0.000000 0.000000 0.000000 +0.62 0.69 0.000000 0.000000 0.000000 +0.64 0.69 0.000000 0.000000 0.000000 +0.65 0.69 0.000000 0.000000 0.000000 +0.66 0.69 0.000000 0.000000 0.000000 +0.67 0.69 0.000000 0.000000 0.000000 +0.68 0.69 0.000000 0.000000 0.000000 +0.69 0.69 0.000000 0.000000 0.000000 +0.70 0.69 0.000000 0.000000 0.000000 +0.70 0.69 0.000000 0.000000 0.000000 +0.71 0.69 0.000000 0.000000 0.000000 +0.72 0.69 0.000000 0.000000 0.000000 +0.73 0.69 0.000000 0.000000 0.000000 +0.74 0.69 0.000000 0.000000 0.000000 +0.76 0.69 0.000000 0.000000 0.000000 +0.77 0.69 0.000000 0.000000 0.000000 +0.78 0.69 0.000000 0.000000 0.000000 +0.79 0.69 0.000000 0.000000 0.000000 +0.80 0.69 0.000000 0.000000 0.000000 +0.81 0.69 0.000000 0.000000 0.000000 +0.82 0.69 0.000000 0.000000 0.000000 +0.83 0.69 0.000000 0.000000 0.000000 +0.83 0.69 0.000000 0.000000 0.000000 +0.84 0.69 0.000000 0.000000 0.000000 +0.85 0.69 0.000000 0.000000 0.000000 +0.86 0.69 0.000000 0.000000 0.000000 +0.88 0.69 0.000000 0.000000 0.000000 +0.89 0.69 0.000000 0.000000 0.000000 +0.90 0.69 0.000000 0.000000 0.000000 +0.91 0.69 0.000000 0.000000 0.000000 +0.92 0.69 0.000000 0.000000 0.000000 +0.93 0.69 0.000000 0.000000 0.000000 +0.94 0.69 0.000000 0.000000 0.000000 +0.95 0.69 0.000000 0.000000 0.000000 +0.96 0.69 0.000000 0.000000 0.000000 +0.96 0.69 0.000000 0.000000 0.000000 +0.97 0.69 0.000000 0.000000 0.000000 +0.98 0.69 0.000000 0.000000 0.000000 +0.01 0.70 0.000000 0.000000 0.000000 +0.01 0.70 0.000000 0.000000 0.000000 +0.03 0.70 0.000000 0.000000 0.000000 +0.04 0.70 0.000000 0.000000 0.000000 +0.04 0.70 0.000000 0.000000 0.000000 +0.06 0.70 0.000000 0.000000 0.000000 +0.07 0.70 0.000000 0.000000 0.000000 +0.07 0.70 0.000000 0.000000 0.000000 +0.09 0.70 0.000000 0.000000 0.000000 +0.10 0.70 0.000000 0.000000 0.000000 +0.10 0.70 0.000000 0.000000 0.000000 +0.12 0.70 0.000000 0.000000 0.000000 +0.12 0.70 0.000000 0.000000 0.000000 +0.14 0.70 0.000000 0.000000 0.000000 +0.14 0.70 0.000000 0.000000 0.000000 +0.15 0.70 0.000000 0.000000 0.000000 +0.17 0.70 0.000000 0.000000 0.000000 +0.18 0.70 0.000000 0.000000 0.000000 +0.18 0.70 0.000000 0.000000 0.000000 +0.20 0.70 0.000000 0.000000 0.000000 +0.21 0.70 0.000000 0.000000 0.000000 +0.21 0.70 0.000000 0.000000 0.000000 +0.23 0.70 0.000000 0.000000 0.000000 +0.24 0.70 0.000000 0.000000 0.000000 +0.24 0.70 0.000000 0.000000 0.000000 +0.26 0.70 0.000000 0.000000 0.000000 +0.27 0.70 0.000000 0.000000 0.000000 +0.28 0.70 0.000000 0.000000 0.000000 +0.29 0.70 0.000000 0.000000 0.000000 +0.29 0.70 0.000000 0.000000 0.000000 +0.30 0.70 0.000000 0.000000 0.000000 +0.32 0.70 0.000000 0.000000 0.000000 +0.33 0.70 0.000000 0.000000 0.000000 +0.34 0.70 0.000000 0.000000 0.000000 +0.35 0.70 0.000000 0.000000 0.000000 +0.35 0.70 0.000000 0.000000 0.000000 +0.36 0.70 0.000000 0.000000 0.000000 +0.38 0.70 0.000000 0.000000 0.000000 +0.39 0.70 0.000000 0.000000 0.000000 +0.40 0.70 0.000000 0.000000 0.000000 +0.41 0.70 0.000000 0.000000 0.000000 +0.42 0.70 0.000000 0.000000 0.000000 +0.42 0.70 0.000000 0.000000 0.000000 +0.43 0.70 0.000000 0.000000 0.000000 +0.45 0.70 0.000000 0.000000 0.000000 +0.46 0.70 0.000000 0.000000 0.000000 +0.47 0.70 0.000000 0.000000 0.000000 +0.48 0.70 0.000000 0.000000 0.000000 +0.48 0.70 0.000000 0.000000 0.000000 +0.49 0.70 0.000000 0.000000 0.000000 +0.51 0.70 0.000000 0.000000 0.000000 +0.52 0.70 0.000000 0.000000 0.000000 +0.53 0.70 0.000000 0.000000 0.000000 +0.54 0.70 0.000000 0.000000 0.000000 +0.55 0.70 0.000000 0.000000 0.000000 +0.56 0.70 0.000000 0.000000 0.000000 +0.57 0.70 0.000000 0.000000 0.000000 +0.58 0.70 0.000000 0.000000 0.000000 +0.58 0.70 0.000000 0.000000 0.000000 +0.59 0.70 0.000000 0.000000 0.000000 +0.60 0.70 0.000000 0.000000 0.000000 +0.61 0.70 0.000000 0.000000 0.000000 +0.62 0.70 0.000000 0.000000 0.000000 +0.64 0.70 0.000000 0.000000 0.000000 +0.65 0.70 0.000000 0.000000 0.000000 +0.66 0.70 0.000000 0.000000 0.000000 +0.67 0.70 0.000000 0.000000 0.000000 +0.68 0.70 0.000000 0.000000 0.000000 +0.69 0.70 0.000000 0.000000 0.000000 +0.70 0.70 0.000000 0.000000 0.000000 +0.70 0.70 0.000000 0.000000 0.000000 +0.71 0.70 0.000000 0.000000 0.000000 +0.72 0.70 0.000000 0.000000 0.000000 +0.73 0.70 0.000000 0.000000 0.000000 +0.74 0.70 0.000000 0.000000 0.000000 +0.76 0.70 0.000000 0.000000 0.000000 +0.77 0.70 0.000000 0.000000 0.000000 +0.78 0.70 0.000000 0.000000 0.000000 +0.79 0.70 0.000000 0.000000 0.000000 +0.80 0.70 0.000000 0.000000 0.000000 +0.81 0.70 0.000000 0.000000 0.000000 +0.82 0.70 0.000000 0.000000 0.000000 +0.83 0.70 0.000000 0.000000 0.000000 +0.83 0.70 0.000000 0.000000 0.000000 +0.84 0.70 0.000000 0.000000 0.000000 +0.85 0.70 0.000000 0.000000 0.000000 +0.86 0.70 0.000000 0.000000 0.000000 +0.88 0.70 0.000000 0.000000 0.000000 +0.89 0.70 0.000000 0.000000 0.000000 +0.90 0.70 0.000000 0.000000 0.000000 +0.91 0.70 0.000000 0.000000 0.000000 +0.92 0.70 0.000000 0.000000 0.000000 +0.93 0.70 0.000000 0.000000 0.000000 +0.94 0.70 0.000000 0.000000 0.000000 +0.95 0.70 0.000000 0.000000 0.000000 +0.96 0.70 0.000000 0.000000 0.000000 +0.96 0.70 0.000000 0.000000 0.000000 +0.97 0.70 0.000000 0.000000 0.000000 +0.98 0.70 0.000000 0.000000 0.000000 +0.01 0.70 0.000000 0.000000 0.000000 +0.01 0.70 0.000000 0.000000 0.000000 +0.03 0.70 0.000000 0.000000 0.000000 +0.04 0.70 0.000000 0.000000 0.000000 +0.04 0.70 0.000000 0.000000 0.000000 +0.06 0.70 0.000000 0.000000 0.000000 +0.07 0.70 0.000000 0.000000 0.000000 +0.07 0.70 0.000000 0.000000 0.000000 +0.09 0.70 0.000000 0.000000 0.000000 +0.10 0.70 0.000000 0.000000 0.000000 +0.10 0.70 0.000000 0.000000 0.000000 +0.12 0.70 0.000000 0.000000 0.000000 +0.12 0.70 0.000000 0.000000 0.000000 +0.14 0.70 0.000000 0.000000 0.000000 +0.14 0.70 0.000000 0.000000 0.000000 +0.15 0.70 0.000000 0.000000 0.000000 +0.17 0.70 0.000000 0.000000 0.000000 +0.18 0.70 0.000000 0.000000 0.000000 +0.18 0.70 0.000000 0.000000 0.000000 +0.20 0.70 0.000000 0.000000 0.000000 +0.21 0.70 0.000000 0.000000 0.000000 +0.21 0.70 0.000000 0.000000 0.000000 +0.23 0.70 0.000000 0.000000 0.000000 +0.24 0.70 0.000000 0.000000 0.000000 +0.24 0.70 0.000000 0.000000 0.000000 +0.26 0.70 0.000000 0.000000 0.000000 +0.27 0.70 0.000000 0.000000 0.000000 +0.28 0.70 0.000000 0.000000 0.000000 +0.29 0.70 0.000000 0.000000 0.000000 +0.29 0.70 0.000000 0.000000 0.000000 +0.30 0.70 0.000000 0.000000 0.000000 +0.32 0.70 0.000000 0.000000 0.000000 +0.33 0.70 0.000000 0.000000 0.000000 +0.34 0.70 0.000000 0.000000 0.000000 +0.35 0.70 0.000000 0.000000 0.000000 +0.35 0.70 0.000000 0.000000 0.000000 +0.36 0.70 0.000000 0.000000 0.000000 +0.38 0.70 0.000000 0.000000 0.000000 +0.39 0.70 0.000000 0.000000 0.000000 +0.40 0.70 0.000000 0.000000 0.000000 +0.41 0.70 0.000000 0.000000 0.000000 +0.42 0.70 0.000000 0.000000 0.000000 +0.42 0.70 0.000000 0.000000 0.000000 +0.43 0.70 0.000000 0.000000 0.000000 +0.45 0.70 0.000000 0.000000 0.000000 +0.46 0.70 0.000000 0.000000 0.000000 +0.47 0.70 0.000000 0.000000 0.000000 +0.48 0.70 0.000000 0.000000 0.000000 +0.48 0.70 0.000000 0.000000 0.000000 +0.49 0.70 0.000000 0.000000 0.000000 +0.51 0.70 0.000000 0.000000 0.000000 +0.52 0.70 0.000000 0.000000 0.000000 +0.53 0.70 0.000000 0.000000 0.000000 +0.54 0.70 0.000000 0.000000 0.000000 +0.55 0.70 0.000000 0.000000 0.000000 +0.56 0.70 0.000000 0.000000 0.000000 +0.57 0.70 0.000000 0.000000 0.000000 +0.58 0.70 0.000000 0.000000 0.000000 +0.58 0.70 0.000000 0.000000 0.000000 +0.59 0.70 0.000000 0.000000 0.000000 +0.60 0.70 0.000000 0.000000 0.000000 +0.61 0.70 0.000000 0.000000 0.000000 +0.62 0.70 0.000000 0.000000 0.000000 +0.64 0.70 0.000000 0.000000 0.000000 +0.65 0.70 0.000000 0.000000 0.000000 +0.66 0.70 0.000000 0.000000 0.000000 +0.67 0.70 0.000000 0.000000 0.000000 +0.68 0.70 0.000000 0.000000 0.000000 +0.69 0.70 0.000000 0.000000 0.000000 +0.70 0.70 0.000000 0.000000 0.000000 +0.70 0.70 0.000000 0.000000 0.000000 +0.71 0.70 0.000000 0.000000 0.000000 +0.72 0.70 0.000000 0.000000 0.000000 +0.73 0.70 0.000000 0.000000 0.000000 +0.74 0.70 0.000000 0.000000 0.000000 +0.76 0.70 0.000000 0.000000 0.000000 +0.77 0.70 0.000000 0.000000 0.000000 +0.78 0.70 0.000000 0.000000 0.000000 +0.79 0.70 0.000000 0.000000 0.000000 +0.80 0.70 0.000000 0.000000 0.000000 +0.81 0.70 0.000000 0.000000 0.000000 +0.82 0.70 0.000000 0.000000 0.000000 +0.83 0.70 0.000000 0.000000 0.000000 +0.83 0.70 0.000000 0.000000 0.000000 +0.84 0.70 0.000000 0.000000 0.000000 +0.85 0.70 0.000000 0.000000 0.000000 +0.86 0.70 0.000000 0.000000 0.000000 +0.88 0.70 0.000000 0.000000 0.000000 +0.89 0.70 0.000000 0.000000 0.000000 +0.90 0.70 0.000000 0.000000 0.000000 +0.91 0.70 0.000000 0.000000 0.000000 +0.92 0.70 0.000000 0.000000 0.000000 +0.93 0.70 0.000000 0.000000 0.000000 +0.94 0.70 0.000000 0.000000 0.000000 +0.95 0.70 0.000000 0.000000 0.000000 +0.96 0.70 0.000000 0.000000 0.000000 +0.96 0.70 0.000000 0.000000 0.000000 +0.97 0.70 0.000000 0.000000 0.000000 +0.98 0.70 0.000000 0.000000 0.000000 +0.01 0.71 0.000000 0.000000 0.000000 +0.01 0.71 0.000000 0.000000 0.000000 +0.03 0.71 0.000000 0.000000 0.000000 +0.04 0.71 0.000000 0.000000 0.000000 +0.04 0.71 0.000000 0.000000 0.000000 +0.06 0.71 0.000000 0.000000 0.000000 +0.07 0.71 0.000000 0.000000 0.000000 +0.07 0.71 0.000000 0.000000 0.000000 +0.09 0.71 0.000000 0.000000 0.000000 +0.10 0.71 0.000000 0.000000 0.000000 +0.10 0.71 0.000000 0.000000 0.000000 +0.12 0.71 0.000000 0.000000 0.000000 +0.12 0.71 0.000000 0.000000 0.000000 +0.14 0.71 0.000000 0.000000 0.000000 +0.14 0.71 0.000000 0.000000 0.000000 +0.15 0.71 0.000000 0.000000 0.000000 +0.17 0.71 0.000000 0.000000 0.000000 +0.18 0.71 0.000000 0.000000 0.000000 +0.18 0.71 0.000000 0.000000 0.000000 +0.20 0.71 0.000000 0.000000 0.000000 +0.21 0.71 0.000000 0.000000 0.000000 +0.21 0.71 0.000000 0.000000 0.000000 +0.23 0.71 0.000000 0.000000 0.000000 +0.24 0.71 0.000000 0.000000 0.000000 +0.24 0.71 0.000000 0.000000 0.000000 +0.26 0.71 0.000000 0.000000 0.000000 +0.27 0.71 0.000000 0.000000 0.000000 +0.28 0.71 0.000000 0.000000 0.000000 +0.29 0.71 0.000000 0.000000 0.000000 +0.29 0.71 0.000000 0.000000 0.000000 +0.30 0.71 0.000000 0.000000 0.000000 +0.32 0.71 0.000000 0.000000 0.000000 +0.33 0.71 0.000000 0.000000 0.000000 +0.34 0.71 0.000000 0.000000 0.000000 +0.35 0.71 0.000000 0.000000 0.000000 +0.35 0.71 0.000000 0.000000 0.000000 +0.36 0.71 0.000000 0.000000 0.000000 +0.38 0.71 0.000000 0.000000 0.000000 +0.39 0.71 0.000000 0.000000 0.000000 +0.40 0.71 0.000000 0.000000 0.000000 +0.41 0.71 0.000000 0.000000 0.000000 +0.42 0.71 0.000000 0.000000 0.000000 +0.42 0.71 0.000000 0.000000 0.000000 +0.43 0.71 0.000000 0.000000 0.000000 +0.45 0.71 0.000000 0.000000 0.000000 +0.46 0.71 0.000000 0.000000 0.000000 +0.47 0.71 0.000000 0.000000 0.000000 +0.48 0.71 0.000000 0.000000 0.000000 +0.48 0.71 0.000000 0.000000 0.000000 +0.49 0.71 0.000000 0.000000 0.000000 +0.51 0.71 0.000000 0.000000 0.000000 +0.52 0.71 0.000000 0.000000 0.000000 +0.53 0.71 0.000000 0.000000 0.000000 +0.54 0.71 0.000000 0.000000 0.000000 +0.55 0.71 0.000000 0.000000 0.000000 +0.56 0.71 0.000000 0.000000 0.000000 +0.57 0.71 0.000000 0.000000 0.000000 +0.58 0.71 0.000000 0.000000 0.000000 +0.58 0.71 0.000000 0.000000 0.000000 +0.59 0.71 0.000000 0.000000 0.000000 +0.60 0.71 0.000000 0.000000 0.000000 +0.61 0.71 0.000000 0.000000 0.000000 +0.62 0.71 0.000000 0.000000 0.000000 +0.64 0.71 0.000000 0.000000 0.000000 +0.65 0.71 0.000000 0.000000 0.000000 +0.66 0.71 0.000000 0.000000 0.000000 +0.67 0.71 0.000000 0.000000 0.000000 +0.68 0.71 0.000000 0.000000 0.000000 +0.69 0.71 0.000000 0.000000 0.000000 +0.70 0.71 0.000000 0.000000 0.000000 +0.70 0.71 0.000000 0.000000 0.000000 +0.71 0.71 0.000000 0.000000 0.000000 +0.72 0.71 0.000000 0.000000 0.000000 +0.73 0.71 0.000000 0.000000 0.000000 +0.74 0.71 0.000000 0.000000 0.000000 +0.76 0.71 0.000000 0.000000 0.000000 +0.77 0.71 0.000000 0.000000 0.000000 +0.78 0.71 0.000000 0.000000 0.000000 +0.79 0.71 0.000000 0.000000 0.000000 +0.80 0.71 0.000000 0.000000 0.000000 +0.81 0.71 0.000000 0.000000 0.000000 +0.82 0.71 0.000000 0.000000 0.000000 +0.83 0.71 0.000000 0.000000 0.000000 +0.83 0.71 0.000000 0.000000 0.000000 +0.84 0.71 0.000000 0.000000 0.000000 +0.85 0.71 0.000000 0.000000 0.000000 +0.86 0.71 0.000000 0.000000 0.000000 +0.88 0.71 0.000000 0.000000 0.000000 +0.89 0.71 0.000000 0.000000 0.000000 +0.90 0.71 0.000000 0.000000 0.000000 +0.91 0.71 0.000000 0.000000 0.000000 +0.92 0.71 0.000000 0.000000 0.000000 +0.93 0.71 0.000000 0.000000 0.000000 +0.94 0.71 0.000000 0.000000 0.000000 +0.95 0.71 0.000000 0.000000 0.000000 +0.96 0.71 0.000000 0.000000 0.000000 +0.96 0.71 0.000000 0.000000 0.000000 +0.97 0.71 0.000000 0.000000 0.000000 +0.98 0.71 0.000000 0.000000 0.000000 +0.01 0.72 0.000000 0.000000 0.000000 +0.01 0.72 0.000000 0.000000 0.000000 +0.03 0.72 0.000000 0.000000 0.000000 +0.04 0.72 0.000000 0.000000 0.000000 +0.04 0.72 0.000000 0.000000 0.000000 +0.06 0.72 0.000000 0.000000 0.000000 +0.07 0.72 0.000000 0.000000 0.000000 +0.07 0.72 0.000000 0.000000 0.000000 +0.09 0.72 0.000000 0.000000 0.000000 +0.10 0.72 0.000000 0.000000 0.000000 +0.10 0.72 0.000000 0.000000 0.000000 +0.12 0.72 0.000000 0.000000 0.000000 +0.12 0.72 0.000000 0.000000 0.000000 +0.14 0.72 0.000000 0.000000 0.000000 +0.14 0.72 0.000000 0.000000 0.000000 +0.15 0.72 0.000000 0.000000 0.000000 +0.17 0.72 0.000000 0.000000 0.000000 +0.18 0.72 0.000000 0.000000 0.000000 +0.18 0.72 0.000000 0.000000 0.000000 +0.20 0.72 0.000000 0.000000 0.000000 +0.21 0.72 0.000000 0.000000 0.000000 +0.21 0.72 0.000000 0.000000 0.000000 +0.23 0.72 0.000000 0.000000 0.000000 +0.24 0.72 0.000000 0.000000 0.000000 +0.24 0.72 0.000000 0.000000 0.000000 +0.26 0.72 0.000000 0.000000 0.000000 +0.27 0.72 0.000000 0.000000 0.000000 +0.28 0.72 0.000000 0.000000 0.000000 +0.29 0.72 0.000000 0.000000 0.000000 +0.29 0.72 0.000000 0.000000 0.000000 +0.30 0.72 0.000000 0.000000 0.000000 +0.32 0.72 0.000000 0.000000 0.000000 +0.33 0.72 0.000000 0.000000 0.000000 +0.34 0.72 0.000000 0.000000 0.000000 +0.35 0.72 0.000000 0.000000 0.000000 +0.35 0.72 0.000000 0.000000 0.000000 +0.36 0.72 0.000000 0.000000 0.000000 +0.38 0.72 0.000000 0.000000 0.000000 +0.39 0.72 0.000000 0.000000 0.000000 +0.40 0.72 0.000000 0.000000 0.000000 +0.41 0.72 0.000000 0.000000 0.000000 +0.42 0.72 0.000000 0.000000 0.000000 +0.42 0.72 0.000000 0.000000 0.000000 +0.43 0.72 0.000000 0.000000 0.000000 +0.45 0.72 0.000000 0.000000 0.000000 +0.46 0.72 0.000000 0.000000 0.000000 +0.47 0.72 0.000000 0.000000 0.000000 +0.48 0.72 0.000000 0.000000 0.000000 +0.48 0.72 0.000000 0.000000 0.000000 +0.49 0.72 0.000000 0.000000 0.000000 +0.51 0.72 0.000000 0.000000 0.000000 +0.52 0.72 0.000000 0.000000 0.000000 +0.53 0.72 0.000000 0.000000 0.000000 +0.54 0.72 0.000000 0.000000 0.000000 +0.55 0.72 0.000000 0.000000 0.000000 +0.56 0.72 0.000000 0.000000 0.000000 +0.57 0.72 0.000000 0.000000 0.000000 +0.58 0.72 0.000000 0.000000 0.000000 +0.58 0.72 0.000000 0.000000 0.000000 +0.59 0.72 0.000000 0.000000 0.000000 +0.60 0.72 0.000000 0.000000 0.000000 +0.61 0.72 0.000000 0.000000 0.000000 +0.62 0.72 0.000000 0.000000 0.000000 +0.64 0.72 0.000000 0.000000 0.000000 +0.65 0.72 0.000000 0.000000 0.000000 +0.66 0.72 0.000000 0.000000 0.000000 +0.67 0.72 0.000000 0.000000 0.000000 +0.68 0.72 0.000000 0.000000 0.000000 +0.69 0.72 0.000000 0.000000 0.000000 +0.70 0.72 0.000000 0.000000 0.000000 +0.70 0.72 0.000000 0.000000 0.000000 +0.71 0.72 0.000000 0.000000 0.000000 +0.72 0.72 0.000000 0.000000 0.000000 +0.73 0.72 0.000000 0.000000 0.000000 +0.74 0.72 0.000000 0.000000 0.000000 +0.76 0.72 0.000000 0.000000 0.000000 +0.77 0.72 0.000000 0.000000 0.000000 +0.78 0.72 0.000000 0.000000 0.000000 +0.79 0.72 0.000000 0.000000 0.000000 +0.80 0.72 0.000000 0.000000 0.000000 +0.81 0.72 0.000000 0.000000 0.000000 +0.82 0.72 0.000000 0.000000 0.000000 +0.83 0.72 0.000000 0.000000 0.000000 +0.83 0.72 0.000000 0.000000 0.000000 +0.84 0.72 0.000000 0.000000 0.000000 +0.85 0.72 0.000000 0.000000 0.000000 +0.86 0.72 0.000000 0.000000 0.000000 +0.88 0.72 0.000000 0.000000 0.000000 +0.89 0.72 0.000000 0.000000 0.000000 +0.90 0.72 0.000000 0.000000 0.000000 +0.91 0.72 0.000000 0.000000 0.000000 +0.92 0.72 0.000000 0.000000 0.000000 +0.93 0.72 0.000000 0.000000 0.000000 +0.94 0.72 0.000000 0.000000 0.000000 +0.95 0.72 0.000000 0.000000 0.000000 +0.96 0.72 0.000000 0.000000 0.000000 +0.96 0.72 0.000000 0.000000 0.000000 +0.97 0.72 0.000000 0.000000 0.000000 +0.98 0.72 0.000000 0.000000 0.000000 +0.01 0.73 0.000000 0.000000 0.000000 +0.01 0.73 0.000000 0.000000 0.000000 +0.03 0.73 0.000000 0.000000 0.000000 +0.04 0.73 0.000000 0.000000 0.000000 +0.04 0.73 0.000000 0.000000 0.000000 +0.06 0.73 0.000000 0.000000 0.000000 +0.07 0.73 0.000000 0.000000 0.000000 +0.07 0.73 0.000000 0.000000 0.000000 +0.09 0.73 0.000000 0.000000 0.000000 +0.10 0.73 0.000000 0.000000 0.000000 +0.10 0.73 0.000000 0.000000 0.000000 +0.12 0.73 0.000000 0.000000 0.000000 +0.12 0.73 0.000000 0.000000 0.000000 +0.14 0.73 0.000000 0.000000 0.000000 +0.14 0.73 0.000000 0.000000 0.000000 +0.15 0.73 0.000000 0.000000 0.000000 +0.17 0.73 0.000000 0.000000 0.000000 +0.18 0.73 0.000000 0.000000 0.000000 +0.18 0.73 0.000000 0.000000 0.000000 +0.20 0.73 0.000000 0.000000 0.000000 +0.21 0.73 0.000000 0.000000 0.000000 +0.21 0.73 0.000000 0.000000 0.000000 +0.23 0.73 0.000000 0.000000 0.000000 +0.24 0.73 0.000000 0.000000 0.000000 +0.24 0.73 0.000000 0.000000 0.000000 +0.26 0.73 0.000000 0.000000 0.000000 +0.27 0.73 0.000000 0.000000 0.000000 +0.28 0.73 0.000000 0.000000 0.000000 +0.29 0.73 0.000000 0.000000 0.000000 +0.29 0.73 0.000000 0.000000 0.000000 +0.30 0.73 0.000000 0.000000 0.000000 +0.32 0.73 0.000000 0.000000 0.000000 +0.33 0.73 0.000000 0.000000 0.000000 +0.34 0.73 0.000000 0.000000 0.000000 +0.35 0.73 0.000000 0.000000 0.000000 +0.35 0.73 0.000000 0.000000 0.000000 +0.36 0.73 0.000000 0.000000 0.000000 +0.38 0.73 0.000000 0.000000 0.000000 +0.39 0.73 0.000000 0.000000 0.000000 +0.40 0.73 0.000000 0.000000 0.000000 +0.41 0.73 0.000000 0.000000 0.000000 +0.42 0.73 0.000000 0.000000 0.000000 +0.42 0.73 0.000000 0.000000 0.000000 +0.43 0.73 0.000000 0.000000 0.000000 +0.45 0.73 0.000000 0.000000 0.000000 +0.46 0.73 0.000000 0.000000 0.000000 +0.47 0.73 0.000000 0.000000 0.000000 +0.48 0.73 0.000000 0.000000 0.000000 +0.48 0.73 0.000000 0.000000 0.000000 +0.49 0.73 0.000000 0.000000 0.000000 +0.51 0.73 0.000000 0.000000 0.000000 +0.52 0.73 0.000000 0.000000 0.000000 +0.53 0.73 0.000000 0.000000 0.000000 +0.54 0.73 0.000000 0.000000 0.000000 +0.55 0.73 0.000000 0.000000 0.000000 +0.56 0.73 0.000000 0.000000 0.000000 +0.57 0.73 0.000000 0.000000 0.000000 +0.58 0.73 0.000000 0.000000 0.000000 +0.58 0.73 0.000000 0.000000 0.000000 +0.59 0.73 0.000000 0.000000 0.000000 +0.60 0.73 0.000000 0.000000 0.000000 +0.61 0.73 0.000000 0.000000 0.000000 +0.62 0.73 0.000000 0.000000 0.000000 +0.64 0.73 0.000000 0.000000 0.000000 +0.65 0.73 0.000000 0.000000 0.000000 +0.66 0.73 0.000000 0.000000 0.000000 +0.67 0.73 0.000000 0.000000 0.000000 +0.68 0.73 0.000000 0.000000 0.000000 +0.69 0.73 0.000000 0.000000 0.000000 +0.70 0.73 0.000000 0.000000 0.000000 +0.70 0.73 0.000000 0.000000 0.000000 +0.71 0.73 0.000000 0.000000 0.000000 +0.72 0.73 0.000000 0.000000 0.000000 +0.73 0.73 0.000000 0.000000 0.000000 +0.74 0.73 0.000000 0.000000 0.000000 +0.76 0.73 0.000000 0.000000 0.000000 +0.77 0.73 0.000000 0.000000 0.000000 +0.78 0.73 0.000000 0.000000 0.000000 +0.79 0.73 0.000000 0.000000 0.000000 +0.80 0.73 0.000000 0.000000 0.000000 +0.81 0.73 0.000000 0.000000 0.000000 +0.82 0.73 0.000000 0.000000 0.000000 +0.83 0.73 0.000000 0.000000 0.000000 +0.83 0.73 0.000000 0.000000 0.000000 +0.84 0.73 0.000000 0.000000 0.000000 +0.85 0.73 0.000000 0.000000 0.000000 +0.86 0.73 0.000000 0.000000 0.000000 +0.88 0.73 0.000000 0.000000 0.000000 +0.89 0.73 0.000000 0.000000 0.000000 +0.90 0.73 0.000000 0.000000 0.000000 +0.91 0.73 0.000000 0.000000 0.000000 +0.92 0.73 0.000000 0.000000 0.000000 +0.93 0.73 0.000000 0.000000 0.000000 +0.94 0.73 0.000000 0.000000 0.000000 +0.95 0.73 0.000000 0.000000 0.000000 +0.96 0.73 0.000000 0.000000 0.000000 +0.96 0.73 0.000000 0.000000 0.000000 +0.97 0.73 0.000000 0.000000 0.000000 +0.98 0.73 0.000000 0.000000 0.000000 +0.01 0.74 0.000000 0.000000 0.000000 +0.01 0.74 0.000000 0.000000 0.000000 +0.03 0.74 0.000000 0.000000 0.000000 +0.04 0.74 0.000000 0.000000 0.000000 +0.04 0.74 0.000000 0.000000 0.000000 +0.06 0.74 0.000000 0.000000 0.000000 +0.07 0.74 0.000000 0.000000 0.000000 +0.07 0.74 0.000000 0.000000 0.000000 +0.09 0.74 0.000000 0.000000 0.000000 +0.10 0.74 0.000000 0.000000 0.000000 +0.10 0.74 0.000000 0.000000 0.000000 +0.12 0.74 0.000000 0.000000 0.000000 +0.12 0.74 0.000000 0.000000 0.000000 +0.14 0.74 0.000000 0.000000 0.000000 +0.14 0.74 0.000000 0.000000 0.000000 +0.15 0.74 0.000000 0.000000 0.000000 +0.17 0.74 0.000000 0.000000 0.000000 +0.18 0.74 0.000000 0.000000 0.000000 +0.18 0.74 0.000000 0.000000 0.000000 +0.20 0.74 0.000000 0.000000 0.000000 +0.21 0.74 0.000000 0.000000 0.000000 +0.21 0.74 0.000000 0.000000 0.000000 +0.23 0.74 0.000000 0.000000 0.000000 +0.24 0.74 0.000000 0.000000 0.000000 +0.24 0.74 0.000000 0.000000 0.000000 +0.26 0.74 0.000000 0.000000 0.000000 +0.27 0.74 0.000000 0.000000 0.000000 +0.28 0.74 0.000000 0.000000 0.000000 +0.29 0.74 0.000000 0.000000 0.000000 +0.29 0.74 0.000000 0.000000 0.000000 +0.30 0.74 0.000000 0.000000 0.000000 +0.32 0.74 0.000000 0.000000 0.000000 +0.33 0.74 0.000000 0.000000 0.000000 +0.34 0.74 0.000000 0.000000 0.000000 +0.35 0.74 0.000000 0.000000 0.000000 +0.35 0.74 0.000000 0.000000 0.000000 +0.36 0.74 0.000000 0.000000 0.000000 +0.38 0.74 0.000000 0.000000 0.000000 +0.39 0.74 0.000000 0.000000 0.000000 +0.40 0.74 0.000000 0.000000 0.000000 +0.41 0.74 0.000000 0.000000 0.000000 +0.42 0.74 0.000000 0.000000 0.000000 +0.42 0.74 0.000000 0.000000 0.000000 +0.43 0.74 0.000000 0.000000 0.000000 +0.45 0.74 0.000000 0.000000 0.000000 +0.46 0.74 0.000000 0.000000 0.000000 +0.47 0.74 0.000000 0.000000 0.000000 +0.48 0.74 0.000000 0.000000 0.000000 +0.48 0.74 0.000000 0.000000 0.000000 +0.49 0.74 0.000000 0.000000 0.000000 +0.51 0.74 0.000000 0.000000 0.000000 +0.52 0.74 0.000000 0.000000 0.000000 +0.53 0.74 0.000000 0.000000 0.000000 +0.54 0.74 0.000000 0.000000 0.000000 +0.55 0.74 0.000000 0.000000 0.000000 +0.56 0.74 0.000000 0.000000 0.000000 +0.57 0.74 0.000000 0.000000 0.000000 +0.58 0.74 0.000000 0.000000 0.000000 +0.58 0.74 0.000000 0.000000 0.000000 +0.59 0.74 0.000000 0.000000 0.000000 +0.60 0.74 0.000000 0.000000 0.000000 +0.61 0.74 0.000000 0.000000 0.000000 +0.62 0.74 0.000000 0.000000 0.000000 +0.64 0.74 0.000000 0.000000 0.000000 +0.65 0.74 0.000000 0.000000 0.000000 +0.66 0.74 0.000000 0.000000 0.000000 +0.67 0.74 0.000000 0.000000 0.000000 +0.68 0.74 0.000000 0.000000 0.000000 +0.69 0.74 0.000000 0.000000 0.000000 +0.70 0.74 0.000000 0.000000 0.000000 +0.70 0.74 0.000000 0.000000 0.000000 +0.71 0.74 0.000000 0.000000 0.000000 +0.72 0.74 0.000000 0.000000 0.000000 +0.73 0.74 0.000000 0.000000 0.000000 +0.74 0.74 0.000000 0.000000 0.000000 +0.76 0.74 0.000000 0.000000 0.000000 +0.77 0.74 0.000000 0.000000 0.000000 +0.78 0.74 0.000000 0.000000 0.000000 +0.79 0.74 0.000000 0.000000 0.000000 +0.80 0.74 0.000000 0.000000 0.000000 +0.81 0.74 0.000000 0.000000 0.000000 +0.82 0.74 0.000000 0.000000 0.000000 +0.83 0.74 0.000000 0.000000 0.000000 +0.83 0.74 0.000000 0.000000 0.000000 +0.84 0.74 0.000000 0.000000 0.000000 +0.85 0.74 0.000000 0.000000 0.000000 +0.86 0.74 0.000000 0.000000 0.000000 +0.88 0.74 0.000000 0.000000 0.000000 +0.89 0.74 0.000000 0.000000 0.000000 +0.90 0.74 0.000000 0.000000 0.000000 +0.91 0.74 0.000000 0.000000 0.000000 +0.92 0.74 0.000000 0.000000 0.000000 +0.93 0.74 0.000000 0.000000 0.000000 +0.94 0.74 0.000000 0.000000 0.000000 +0.95 0.74 0.000000 0.000000 0.000000 +0.96 0.74 0.000000 0.000000 0.000000 +0.96 0.74 0.000000 0.000000 0.000000 +0.97 0.74 0.000000 0.000000 0.000000 +0.98 0.74 0.000000 0.000000 0.000000 +0.01 0.76 0.000000 0.000000 0.000000 +0.01 0.76 0.000000 0.000000 0.000000 +0.03 0.76 0.000000 0.000000 0.000000 +0.04 0.76 0.000000 0.000000 0.000000 +0.04 0.76 0.000000 0.000000 0.000000 +0.06 0.76 0.000000 0.000000 0.000000 +0.07 0.76 0.000000 0.000000 0.000000 +0.07 0.76 0.000000 0.000000 0.000000 +0.09 0.76 0.000000 0.000000 0.000000 +0.10 0.76 0.000000 0.000000 0.000000 +0.10 0.76 0.000000 0.000000 0.000000 +0.12 0.76 0.000000 0.000000 0.000000 +0.12 0.76 0.000000 0.000000 0.000000 +0.14 0.76 0.000000 0.000000 0.000000 +0.14 0.76 0.000000 0.000000 0.000000 +0.15 0.76 0.000000 0.000000 0.000000 +0.17 0.76 0.000000 0.000000 0.000000 +0.18 0.76 0.000000 0.000000 0.000000 +0.18 0.76 0.000000 0.000000 0.000000 +0.20 0.76 0.000000 0.000000 0.000000 +0.21 0.76 0.000000 0.000000 0.000000 +0.21 0.76 0.000000 0.000000 0.000000 +0.23 0.76 0.000000 0.000000 0.000000 +0.24 0.76 0.000000 0.000000 0.000000 +0.24 0.76 0.000000 0.000000 0.000000 +0.26 0.76 0.000000 0.000000 0.000000 +0.27 0.76 0.000000 0.000000 0.000000 +0.28 0.76 0.000000 0.000000 0.000000 +0.29 0.76 0.000000 0.000000 0.000000 +0.29 0.76 0.000000 0.000000 0.000000 +0.30 0.76 0.000000 0.000000 0.000000 +0.32 0.76 0.000000 0.000000 0.000000 +0.33 0.76 0.000000 0.000000 0.000000 +0.34 0.76 0.000000 0.000000 0.000000 +0.35 0.76 0.000000 0.000000 0.000000 +0.35 0.76 0.000000 0.000000 0.000000 +0.36 0.76 0.000000 0.000000 0.000000 +0.38 0.76 0.000000 0.000000 0.000000 +0.39 0.76 0.000000 0.000000 0.000000 +0.40 0.76 0.000000 0.000000 0.000000 +0.41 0.76 0.000000 0.000000 0.000000 +0.42 0.76 0.000000 0.000000 0.000000 +0.42 0.76 0.000000 0.000000 0.000000 +0.43 0.76 0.000000 0.000000 0.000000 +0.45 0.76 0.000000 0.000000 0.000000 +0.46 0.76 0.000000 0.000000 0.000000 +0.47 0.76 0.000000 0.000000 0.000000 +0.48 0.76 0.000000 0.000000 0.000000 +0.48 0.76 0.000000 0.000000 0.000000 +0.49 0.76 0.000000 0.000000 0.000000 +0.51 0.76 0.000000 0.000000 0.000000 +0.52 0.76 0.000000 0.000000 0.000000 +0.53 0.76 0.000000 0.000000 0.000000 +0.54 0.76 0.000000 0.000000 0.000000 +0.55 0.76 0.000000 0.000000 0.000000 +0.56 0.76 0.000000 0.000000 0.000000 +0.57 0.76 0.000000 0.000000 0.000000 +0.58 0.76 0.000000 0.000000 0.000000 +0.58 0.76 0.000000 0.000000 0.000000 +0.59 0.76 0.000000 0.000000 0.000000 +0.60 0.76 0.000000 0.000000 0.000000 +0.61 0.76 0.000000 0.000000 0.000000 +0.62 0.76 0.000000 0.000000 0.000000 +0.64 0.76 0.000000 0.000000 0.000000 +0.65 0.76 0.000000 0.000000 0.000000 +0.66 0.76 0.000000 0.000000 0.000000 +0.67 0.76 0.000000 0.000000 0.000000 +0.68 0.76 0.000000 0.000000 0.000000 +0.69 0.76 0.000000 0.000000 0.000000 +0.70 0.76 0.000000 0.000000 0.000000 +0.70 0.76 0.000000 0.000000 0.000000 +0.71 0.76 0.000000 0.000000 0.000000 +0.72 0.76 0.000000 0.000000 0.000000 +0.73 0.76 0.000000 0.000000 0.000000 +0.74 0.76 0.000000 0.000000 0.000000 +0.76 0.76 0.000000 0.000000 0.000000 +0.77 0.76 0.000000 0.000000 0.000000 +0.78 0.76 0.000000 0.000000 0.000000 +0.79 0.76 0.000000 0.000000 0.000000 +0.80 0.76 0.000000 0.000000 0.000000 +0.81 0.76 0.000000 0.000000 0.000000 +0.82 0.76 0.000000 0.000000 0.000000 +0.83 0.76 0.000000 0.000000 0.000000 +0.83 0.76 0.000000 0.000000 0.000000 +0.84 0.76 0.000000 0.000000 0.000000 +0.85 0.76 0.000000 0.000000 0.000000 +0.86 0.76 0.000000 0.000000 0.000000 +0.88 0.76 0.000000 0.000000 0.000000 +0.89 0.76 0.000000 0.000000 0.000000 +0.90 0.76 0.000000 0.000000 0.000000 +0.91 0.76 0.000000 0.000000 0.000000 +0.92 0.76 0.000000 0.000000 0.000000 +0.93 0.76 0.000000 0.000000 0.000000 +0.94 0.76 0.000000 0.000000 0.000000 +0.95 0.76 0.000000 0.000000 0.000000 +0.96 0.76 0.000000 0.000000 0.000000 +0.96 0.76 0.000000 0.000000 0.000000 +0.97 0.76 0.000000 0.000000 0.000000 +0.98 0.76 0.000000 0.000000 0.000000 +0.01 0.77 0.000000 0.000000 0.000000 +0.01 0.77 0.000000 0.000000 0.000000 +0.03 0.77 0.000000 0.000000 0.000000 +0.04 0.77 0.000000 0.000000 0.000000 +0.04 0.77 0.000000 0.000000 0.000000 +0.06 0.77 0.000000 0.000000 0.000000 +0.07 0.77 0.000000 0.000000 0.000000 +0.07 0.77 0.000000 0.000000 0.000000 +0.09 0.77 0.000000 0.000000 0.000000 +0.10 0.77 0.000000 0.000000 0.000000 +0.10 0.77 0.000000 0.000000 0.000000 +0.12 0.77 0.000000 0.000000 0.000000 +0.12 0.77 0.000000 0.000000 0.000000 +0.14 0.77 0.000000 0.000000 0.000000 +0.14 0.77 0.000000 0.000000 0.000000 +0.15 0.77 0.000000 0.000000 0.000000 +0.17 0.77 0.000000 0.000000 0.000000 +0.18 0.77 0.000000 0.000000 0.000000 +0.18 0.77 0.000000 0.000000 0.000000 +0.20 0.77 0.000000 0.000000 0.000000 +0.21 0.77 0.000000 0.000000 0.000000 +0.21 0.77 0.000000 0.000000 0.000000 +0.23 0.77 0.000000 0.000000 0.000000 +0.24 0.77 0.000000 0.000000 0.000000 +0.24 0.77 0.000000 0.000000 0.000000 +0.26 0.77 0.000000 0.000000 0.000000 +0.27 0.77 0.000000 0.000000 0.000000 +0.28 0.77 0.000000 0.000000 0.000000 +0.29 0.77 0.000000 0.000000 0.000000 +0.29 0.77 0.000000 0.000000 0.000000 +0.30 0.77 0.000000 0.000000 0.000000 +0.32 0.77 0.000000 0.000000 0.000000 +0.33 0.77 0.000000 0.000000 0.000000 +0.34 0.77 0.000000 0.000000 0.000000 +0.35 0.77 0.000000 0.000000 0.000000 +0.35 0.77 0.000000 0.000000 0.000000 +0.36 0.77 0.000000 0.000000 0.000000 +0.38 0.77 0.000000 0.000000 0.000000 +0.39 0.77 0.000000 0.000000 0.000000 +0.40 0.77 0.000000 0.000000 0.000000 +0.41 0.77 0.000000 0.000000 0.000000 +0.42 0.77 0.000000 0.000000 0.000000 +0.42 0.77 0.000000 0.000000 0.000000 +0.43 0.77 0.000000 0.000000 0.000000 +0.45 0.77 0.000000 0.000000 0.000000 +0.46 0.77 0.000000 0.000000 0.000000 +0.47 0.77 0.000000 0.000000 0.000000 +0.48 0.77 0.000000 0.000000 0.000000 +0.48 0.77 0.000000 0.000000 0.000000 +0.49 0.77 0.000000 0.000000 0.000000 +0.51 0.77 0.000000 0.000000 0.000000 +0.52 0.77 0.000000 0.000000 0.000000 +0.53 0.77 0.000000 0.000000 0.000000 +0.54 0.77 0.000000 0.000000 0.000000 +0.55 0.77 0.000000 0.000000 0.000000 +0.56 0.77 0.000000 0.000000 0.000000 +0.57 0.77 0.000000 0.000000 0.000000 +0.58 0.77 0.000000 0.000000 0.000000 +0.58 0.77 0.000000 0.000000 0.000000 +0.59 0.77 0.000000 0.000000 0.000000 +0.60 0.77 0.000000 0.000000 0.000000 +0.61 0.77 0.000000 0.000000 0.000000 +0.62 0.77 0.000000 0.000000 0.000000 +0.64 0.77 0.000000 0.000000 0.000000 +0.65 0.77 0.000000 0.000000 0.000000 +0.66 0.77 0.000000 0.000000 0.000000 +0.67 0.77 0.000000 0.000000 0.000000 +0.68 0.77 0.000000 0.000000 0.000000 +0.69 0.77 0.000000 0.000000 0.000000 +0.70 0.77 0.000000 0.000000 0.000000 +0.70 0.77 0.000000 0.000000 0.000000 +0.71 0.77 0.000000 0.000000 0.000000 +0.72 0.77 0.000000 0.000000 0.000000 +0.73 0.77 0.000000 0.000000 0.000000 +0.74 0.77 0.000000 0.000000 0.000000 +0.76 0.77 0.000000 0.000000 0.000000 +0.77 0.77 0.000000 0.000000 0.000000 +0.78 0.77 0.000000 0.000000 0.000000 +0.79 0.77 0.000000 0.000000 0.000000 +0.80 0.77 0.000000 0.000000 0.000000 +0.81 0.77 0.000000 0.000000 0.000000 +0.82 0.77 0.000000 0.000000 0.000000 +0.83 0.77 0.000000 0.000000 0.000000 +0.83 0.77 0.000000 0.000000 0.000000 +0.84 0.77 0.000000 0.000000 0.000000 +0.85 0.77 0.000000 0.000000 0.000000 +0.86 0.77 0.000000 0.000000 0.000000 +0.88 0.77 0.000000 0.000000 0.000000 +0.89 0.77 0.000000 0.000000 0.000000 +0.90 0.77 0.000000 0.000000 0.000000 +0.91 0.77 0.000000 0.000000 0.000000 +0.92 0.77 0.000000 0.000000 0.000000 +0.93 0.77 0.000000 0.000000 0.000000 +0.94 0.77 0.000000 0.000000 0.000000 +0.95 0.77 0.000000 0.000000 0.000000 +0.96 0.77 0.000000 0.000000 0.000000 +0.96 0.77 0.000000 0.000000 0.000000 +0.97 0.77 0.000000 0.000000 0.000000 +0.98 0.77 0.000000 0.000000 0.000000 +0.01 0.78 0.000000 0.000000 0.000000 +0.01 0.78 0.000000 0.000000 0.000000 +0.03 0.78 0.000000 0.000000 0.000000 +0.04 0.78 0.000000 0.000000 0.000000 +0.04 0.78 0.000000 0.000000 0.000000 +0.06 0.78 0.000000 0.000000 0.000000 +0.07 0.78 0.000000 0.000000 0.000000 +0.07 0.78 0.000000 0.000000 0.000000 +0.09 0.78 0.000000 0.000000 0.000000 +0.10 0.78 0.000000 0.000000 0.000000 +0.10 0.78 0.000000 0.000000 0.000000 +0.12 0.78 0.000000 0.000000 0.000000 +0.12 0.78 0.000000 0.000000 0.000000 +0.14 0.78 0.000000 0.000000 0.000000 +0.14 0.78 0.000000 0.000000 0.000000 +0.15 0.78 0.000000 0.000000 0.000000 +0.17 0.78 0.000000 0.000000 0.000000 +0.18 0.78 0.000000 0.000000 0.000000 +0.18 0.78 0.000000 0.000000 0.000000 +0.20 0.78 0.000000 0.000000 0.000000 +0.21 0.78 0.000000 0.000000 0.000000 +0.21 0.78 0.000000 0.000000 0.000000 +0.23 0.78 0.000000 0.000000 0.000000 +0.24 0.78 0.000000 0.000000 0.000000 +0.24 0.78 0.000000 0.000000 0.000000 +0.26 0.78 0.000000 0.000000 0.000000 +0.27 0.78 0.000000 0.000000 0.000000 +0.28 0.78 0.000000 0.000000 0.000000 +0.29 0.78 0.000000 0.000000 0.000000 +0.29 0.78 0.000000 0.000000 0.000000 +0.30 0.78 0.000000 0.000000 0.000000 +0.32 0.78 0.000000 0.000000 0.000000 +0.33 0.78 0.000000 0.000000 0.000000 +0.34 0.78 0.000000 0.000000 0.000000 +0.35 0.78 0.000000 0.000000 0.000000 +0.35 0.78 0.000000 0.000000 0.000000 +0.36 0.78 0.000000 0.000000 0.000000 +0.38 0.78 0.000000 0.000000 0.000000 +0.39 0.78 0.000000 0.000000 0.000000 +0.40 0.78 0.000000 0.000000 0.000000 +0.41 0.78 0.000000 0.000000 0.000000 +0.42 0.78 0.000000 0.000000 0.000000 +0.42 0.78 0.000000 0.000000 0.000000 +0.43 0.78 0.000000 0.000000 0.000000 +0.45 0.78 0.000000 0.000000 0.000000 +0.46 0.78 0.000000 0.000000 0.000000 +0.47 0.78 0.000000 0.000000 0.000000 +0.48 0.78 0.000000 0.000000 0.000000 +0.48 0.78 0.000000 0.000000 0.000000 +0.49 0.78 0.000000 0.000000 0.000000 +0.51 0.78 0.000000 0.000000 0.000000 +0.52 0.78 0.000000 0.000000 0.000000 +0.53 0.78 0.000000 0.000000 0.000000 +0.54 0.78 0.000000 0.000000 0.000000 +0.55 0.78 0.000000 0.000000 0.000000 +0.56 0.78 0.000000 0.000000 0.000000 +0.57 0.78 0.000000 0.000000 0.000000 +0.58 0.78 0.000000 0.000000 0.000000 +0.58 0.78 0.000000 0.000000 0.000000 +0.59 0.78 0.000000 0.000000 0.000000 +0.60 0.78 0.000000 0.000000 0.000000 +0.61 0.78 0.000000 0.000000 0.000000 +0.62 0.78 0.000000 0.000000 0.000000 +0.64 0.78 0.000000 0.000000 0.000000 +0.65 0.78 0.000000 0.000000 0.000000 +0.66 0.78 0.000000 0.000000 0.000000 +0.67 0.78 0.000000 0.000000 0.000000 +0.68 0.78 0.000000 0.000000 0.000000 +0.69 0.78 0.000000 0.000000 0.000000 +0.70 0.78 0.000000 0.000000 0.000000 +0.70 0.78 0.000000 0.000000 0.000000 +0.71 0.78 0.000000 0.000000 0.000000 +0.72 0.78 0.000000 0.000000 0.000000 +0.73 0.78 0.000000 0.000000 0.000000 +0.74 0.78 0.000000 0.000000 0.000000 +0.76 0.78 0.000000 0.000000 0.000000 +0.77 0.78 0.000000 0.000000 0.000000 +0.78 0.78 0.000000 0.000000 0.000000 +0.79 0.78 0.000000 0.000000 0.000000 +0.80 0.78 0.000000 0.000000 0.000000 +0.81 0.78 0.000000 0.000000 0.000000 +0.82 0.78 0.000000 0.000000 0.000000 +0.83 0.78 0.000000 0.000000 0.000000 +0.83 0.78 0.000000 0.000000 0.000000 +0.84 0.78 0.000000 0.000000 0.000000 +0.85 0.78 0.000000 0.000000 0.000000 +0.86 0.78 0.000000 0.000000 0.000000 +0.88 0.78 0.000000 0.000000 0.000000 +0.89 0.78 0.000000 0.000000 0.000000 +0.90 0.78 0.000000 0.000000 0.000000 +0.91 0.78 0.000000 0.000000 0.000000 +0.92 0.78 0.000000 0.000000 0.000000 +0.93 0.78 0.000000 0.000000 0.000000 +0.94 0.78 0.000000 0.000000 0.000000 +0.95 0.78 0.000000 0.000000 0.000000 +0.96 0.78 0.000000 0.000000 0.000000 +0.96 0.78 0.000000 0.000000 0.000000 +0.97 0.78 0.000000 0.000000 0.000000 +0.98 0.78 0.000000 0.000000 0.000000 +0.01 0.79 0.000000 0.000000 0.000000 +0.01 0.79 0.000000 0.000000 0.000000 +0.03 0.79 0.000000 0.000000 0.000000 +0.04 0.79 0.000000 0.000000 0.000000 +0.04 0.79 0.000000 0.000000 0.000000 +0.06 0.79 0.000000 0.000000 0.000000 +0.07 0.79 0.000000 0.000000 0.000000 +0.07 0.79 0.000000 0.000000 0.000000 +0.09 0.79 0.000000 0.000000 0.000000 +0.10 0.79 0.000000 0.000000 0.000000 +0.10 0.79 0.000000 0.000000 0.000000 +0.12 0.79 0.000000 0.000000 0.000000 +0.12 0.79 0.000000 0.000000 0.000000 +0.14 0.79 0.000000 0.000000 0.000000 +0.14 0.79 0.000000 0.000000 0.000000 +0.15 0.79 0.000000 0.000000 0.000000 +0.17 0.79 0.000000 0.000000 0.000000 +0.18 0.79 0.000000 0.000000 0.000000 +0.18 0.79 0.000000 0.000000 0.000000 +0.20 0.79 0.000000 0.000000 0.000000 +0.21 0.79 0.000000 0.000000 0.000000 +0.21 0.79 0.000000 0.000000 0.000000 +0.23 0.79 0.000000 0.000000 0.000000 +0.24 0.79 0.000000 0.000000 0.000000 +0.24 0.79 0.000000 0.000000 0.000000 +0.26 0.79 0.000000 0.000000 0.000000 +0.27 0.79 0.000000 0.000000 0.000000 +0.28 0.79 0.000000 0.000000 0.000000 +0.29 0.79 0.000000 0.000000 0.000000 +0.29 0.79 0.000000 0.000000 0.000000 +0.30 0.79 0.000000 0.000000 0.000000 +0.32 0.79 0.000000 0.000000 0.000000 +0.33 0.79 0.000000 0.000000 0.000000 +0.34 0.79 0.000000 0.000000 0.000000 +0.35 0.79 0.000000 0.000000 0.000000 +0.35 0.79 0.000000 0.000000 0.000000 +0.36 0.79 0.000000 0.000000 0.000000 +0.38 0.79 0.000000 0.000000 0.000000 +0.39 0.79 0.000000 0.000000 0.000000 +0.40 0.79 0.000000 0.000000 0.000000 +0.41 0.79 0.000000 0.000000 0.000000 +0.42 0.79 0.000000 0.000000 0.000000 +0.42 0.79 0.000000 0.000000 0.000000 +0.43 0.79 0.000000 0.000000 0.000000 +0.45 0.79 0.000000 0.000000 0.000000 +0.46 0.79 0.000000 0.000000 0.000000 +0.47 0.79 0.000000 0.000000 0.000000 +0.48 0.79 0.000000 0.000000 0.000000 +0.48 0.79 0.000000 0.000000 0.000000 +0.49 0.79 0.000000 0.000000 0.000000 +0.51 0.79 0.000000 0.000000 0.000000 +0.52 0.79 0.000000 0.000000 0.000000 +0.53 0.79 0.000000 0.000000 0.000000 +0.54 0.79 0.000000 0.000000 0.000000 +0.55 0.79 0.000000 0.000000 0.000000 +0.56 0.79 0.000000 0.000000 0.000000 +0.57 0.79 0.000000 0.000000 0.000000 +0.58 0.79 0.000000 0.000000 0.000000 +0.58 0.79 0.000000 0.000000 0.000000 +0.59 0.79 0.000000 0.000000 0.000000 +0.60 0.79 0.000000 0.000000 0.000000 +0.61 0.79 0.000000 0.000000 0.000000 +0.62 0.79 0.000000 0.000000 0.000000 +0.64 0.79 0.000000 0.000000 0.000000 +0.65 0.79 0.000000 0.000000 0.000000 +0.66 0.79 0.000000 0.000000 0.000000 +0.67 0.79 0.000000 0.000000 0.000000 +0.68 0.79 0.000000 0.000000 0.000000 +0.69 0.79 0.000000 0.000000 0.000000 +0.70 0.79 0.000000 0.000000 0.000000 +0.70 0.79 0.000000 0.000000 0.000000 +0.71 0.79 0.000000 0.000000 0.000000 +0.72 0.79 0.000000 0.000000 0.000000 +0.73 0.79 0.000000 0.000000 0.000000 +0.74 0.79 0.000000 0.000000 0.000000 +0.76 0.79 0.000000 0.000000 0.000000 +0.77 0.79 0.000000 0.000000 0.000000 +0.78 0.79 0.000000 0.000000 0.000000 +0.79 0.79 0.000000 0.000000 0.000000 +0.80 0.79 0.000000 0.000000 0.000000 +0.81 0.79 0.000000 0.000000 0.000000 +0.82 0.79 0.000000 0.000000 0.000000 +0.83 0.79 0.000000 0.000000 0.000000 +0.83 0.79 0.000000 0.000000 0.000000 +0.84 0.79 0.000000 0.000000 0.000000 +0.85 0.79 0.000000 0.000000 0.000000 +0.86 0.79 0.000000 0.000000 0.000000 +0.88 0.79 0.000000 0.000000 0.000000 +0.89 0.79 0.000000 0.000000 0.000000 +0.90 0.79 0.000000 0.000000 0.000000 +0.91 0.79 0.000000 0.000000 0.000000 +0.92 0.79 0.000000 0.000000 0.000000 +0.93 0.79 0.000000 0.000000 0.000000 +0.94 0.79 0.000000 0.000000 0.000000 +0.95 0.79 0.000000 0.000000 0.000000 +0.96 0.79 0.000000 0.000000 0.000000 +0.96 0.79 0.000000 0.000000 0.000000 +0.97 0.79 0.000000 0.000000 0.000000 +0.98 0.79 0.000000 0.000000 0.000000 +0.01 0.80 0.000000 0.000000 0.000000 +0.01 0.80 0.000000 0.000000 0.000000 +0.03 0.80 0.000000 0.000000 0.000000 +0.04 0.80 0.000000 0.000000 0.000000 +0.04 0.80 0.000000 0.000000 0.000000 +0.06 0.80 0.000000 0.000000 0.000000 +0.07 0.80 0.000000 0.000000 0.000000 +0.07 0.80 0.000000 0.000000 0.000000 +0.09 0.80 0.000000 0.000000 0.000000 +0.10 0.80 0.000000 0.000000 0.000000 +0.10 0.80 0.000000 0.000000 0.000000 +0.12 0.80 0.000000 0.000000 0.000000 +0.12 0.80 0.000000 0.000000 0.000000 +0.14 0.80 0.000000 0.000000 0.000000 +0.14 0.80 0.000000 0.000000 0.000000 +0.15 0.80 0.000000 0.000000 0.000000 +0.17 0.80 0.000000 0.000000 0.000000 +0.18 0.80 0.000000 0.000000 0.000000 +0.18 0.80 0.000000 0.000000 0.000000 +0.20 0.80 0.000000 0.000000 0.000000 +0.21 0.80 0.000000 0.000000 0.000000 +0.21 0.80 0.000000 0.000000 0.000000 +0.23 0.80 0.000000 0.000000 0.000000 +0.24 0.80 0.000000 0.000000 0.000000 +0.24 0.80 0.000000 0.000000 0.000000 +0.26 0.80 0.000000 0.000000 0.000000 +0.27 0.80 0.000000 0.000000 0.000000 +0.28 0.80 0.000000 0.000000 0.000000 +0.29 0.80 0.000000 0.000000 0.000000 +0.29 0.80 0.000000 0.000000 0.000000 +0.30 0.80 0.000000 0.000000 0.000000 +0.32 0.80 0.000000 0.000000 0.000000 +0.33 0.80 0.000000 0.000000 0.000000 +0.34 0.80 0.000000 0.000000 0.000000 +0.35 0.80 0.000000 0.000000 0.000000 +0.35 0.80 0.000000 0.000000 0.000000 +0.36 0.80 0.000000 0.000000 0.000000 +0.38 0.80 0.000000 0.000000 0.000000 +0.39 0.80 0.000000 0.000000 0.000000 +0.40 0.80 0.000000 0.000000 0.000000 +0.41 0.80 0.000000 0.000000 0.000000 +0.42 0.80 0.000000 0.000000 0.000000 +0.42 0.80 0.000000 0.000000 0.000000 +0.43 0.80 0.000000 0.000000 0.000000 +0.45 0.80 0.000000 0.000000 0.000000 +0.46 0.80 0.000000 0.000000 0.000000 +0.47 0.80 0.000000 0.000000 0.000000 +0.48 0.80 0.000000 0.000000 0.000000 +0.48 0.80 0.000000 0.000000 0.000000 +0.49 0.80 0.000000 0.000000 0.000000 +0.51 0.80 0.000000 0.000000 0.000000 +0.52 0.80 0.000000 0.000000 0.000000 +0.53 0.80 0.000000 0.000000 0.000000 +0.54 0.80 0.000000 0.000000 0.000000 +0.55 0.80 0.000000 0.000000 0.000000 +0.56 0.80 0.000000 0.000000 0.000000 +0.57 0.80 0.000000 0.000000 0.000000 +0.58 0.80 0.000000 0.000000 0.000000 +0.58 0.80 0.000000 0.000000 0.000000 +0.59 0.80 0.000000 0.000000 0.000000 +0.60 0.80 0.000000 0.000000 0.000000 +0.61 0.80 0.000000 0.000000 0.000000 +0.62 0.80 0.000000 0.000000 0.000000 +0.64 0.80 0.000000 0.000000 0.000000 +0.65 0.80 0.000000 0.000000 0.000000 +0.66 0.80 0.000000 0.000000 0.000000 +0.67 0.80 0.000000 0.000000 0.000000 +0.68 0.80 0.000000 0.000000 0.000000 +0.69 0.80 0.000000 0.000000 0.000000 +0.70 0.80 0.000000 0.000000 0.000000 +0.70 0.80 0.000000 0.000000 0.000000 +0.71 0.80 0.000000 0.000000 0.000000 +0.72 0.80 0.000000 0.000000 0.000000 +0.73 0.80 0.000000 0.000000 0.000000 +0.74 0.80 0.000000 0.000000 0.000000 +0.76 0.80 0.000000 0.000000 0.000000 +0.77 0.80 0.000000 0.000000 0.000000 +0.78 0.80 0.000000 0.000000 0.000000 +0.79 0.80 0.000000 0.000000 0.000000 +0.80 0.80 0.000000 0.000000 0.000000 +0.81 0.80 0.000000 0.000000 0.000000 +0.82 0.80 0.000000 0.000000 0.000000 +0.83 0.80 0.000000 0.000000 0.000000 +0.83 0.80 0.000000 0.000000 0.000000 +0.84 0.80 0.000000 0.000000 0.000000 +0.85 0.80 0.000000 0.000000 0.000000 +0.86 0.80 0.000000 0.000000 0.000000 +0.88 0.80 0.000000 0.000000 0.000000 +0.89 0.80 0.000000 0.000000 0.000000 +0.90 0.80 0.000000 0.000000 0.000000 +0.91 0.80 0.000000 0.000000 0.000000 +0.92 0.80 0.000000 0.000000 0.000000 +0.93 0.80 0.000000 0.000000 0.000000 +0.94 0.80 0.000000 0.000000 0.000000 +0.95 0.80 0.000000 0.000000 0.000000 +0.96 0.80 0.000000 0.000000 0.000000 +0.96 0.80 0.000000 0.000000 0.000000 +0.97 0.80 0.000000 0.000000 0.000000 +0.98 0.80 0.000000 0.000000 0.000000 +0.01 0.81 0.000000 0.000000 0.000000 +0.01 0.81 0.000000 0.000000 0.000000 +0.03 0.81 0.000000 0.000000 0.000000 +0.04 0.81 0.000000 0.000000 0.000000 +0.04 0.81 0.000000 0.000000 0.000000 +0.06 0.81 0.000000 0.000000 0.000000 +0.07 0.81 0.000000 0.000000 0.000000 +0.07 0.81 0.000000 0.000000 0.000000 +0.09 0.81 0.000000 0.000000 0.000000 +0.10 0.81 0.000000 0.000000 0.000000 +0.10 0.81 0.000000 0.000000 0.000000 +0.12 0.81 0.000000 0.000000 0.000000 +0.12 0.81 0.000000 0.000000 0.000000 +0.14 0.81 0.000000 0.000000 0.000000 +0.14 0.81 0.000000 0.000000 0.000000 +0.15 0.81 0.000000 0.000000 0.000000 +0.17 0.81 0.000000 0.000000 0.000000 +0.18 0.81 0.000000 0.000000 0.000000 +0.18 0.81 0.000000 0.000000 0.000000 +0.20 0.81 0.000000 0.000000 0.000000 +0.21 0.81 0.000000 0.000000 0.000000 +0.21 0.81 0.000000 0.000000 0.000000 +0.23 0.81 0.000000 0.000000 0.000000 +0.24 0.81 0.000000 0.000000 0.000000 +0.24 0.81 0.000000 0.000000 0.000000 +0.26 0.81 0.000000 0.000000 0.000000 +0.27 0.81 0.000000 0.000000 0.000000 +0.28 0.81 0.000000 0.000000 0.000000 +0.29 0.81 0.000000 0.000000 0.000000 +0.29 0.81 0.000000 0.000000 0.000000 +0.30 0.81 0.000000 0.000000 0.000000 +0.32 0.81 0.000000 0.000000 0.000000 +0.33 0.81 0.000000 0.000000 0.000000 +0.34 0.81 0.000000 0.000000 0.000000 +0.35 0.81 0.000000 0.000000 0.000000 +0.35 0.81 0.000000 0.000000 0.000000 +0.36 0.81 0.000000 0.000000 0.000000 +0.38 0.81 0.000000 0.000000 0.000000 +0.39 0.81 0.000000 0.000000 0.000000 +0.40 0.81 0.000000 0.000000 0.000000 +0.41 0.81 0.000000 0.000000 0.000000 +0.42 0.81 0.000000 0.000000 0.000000 +0.42 0.81 0.000000 0.000000 0.000000 +0.43 0.81 0.000000 0.000000 0.000000 +0.45 0.81 0.000000 0.000000 0.000000 +0.46 0.81 0.000000 0.000000 0.000000 +0.47 0.81 0.000000 0.000000 0.000000 +0.48 0.81 0.000000 0.000000 0.000000 +0.48 0.81 0.000000 0.000000 0.000000 +0.49 0.81 0.000000 0.000000 0.000000 +0.51 0.81 0.000000 0.000000 0.000000 +0.52 0.81 0.000000 0.000000 0.000000 +0.53 0.81 0.000000 0.000000 0.000000 +0.54 0.81 0.000000 0.000000 0.000000 +0.55 0.81 0.000000 0.000000 0.000000 +0.56 0.81 0.000000 0.000000 0.000000 +0.57 0.81 0.000000 0.000000 0.000000 +0.58 0.81 0.000000 0.000000 0.000000 +0.58 0.81 0.000000 0.000000 0.000000 +0.59 0.81 0.000000 0.000000 0.000000 +0.60 0.81 0.000000 0.000000 0.000000 +0.61 0.81 0.000000 0.000000 0.000000 +0.62 0.81 0.000000 0.000000 0.000000 +0.64 0.81 0.000000 0.000000 0.000000 +0.65 0.81 0.000000 0.000000 0.000000 +0.66 0.81 0.000000 0.000000 0.000000 +0.67 0.81 0.000000 0.000000 0.000000 +0.68 0.81 0.000000 0.000000 0.000000 +0.69 0.81 0.000000 0.000000 0.000000 +0.70 0.81 0.000000 0.000000 0.000000 +0.70 0.81 0.000000 0.000000 0.000000 +0.71 0.81 0.000000 0.000000 0.000000 +0.72 0.81 0.000000 0.000000 0.000000 +0.73 0.81 0.000000 0.000000 0.000000 +0.74 0.81 0.000000 0.000000 0.000000 +0.76 0.81 0.000000 0.000000 0.000000 +0.77 0.81 0.000000 0.000000 0.000000 +0.78 0.81 0.000000 0.000000 0.000000 +0.79 0.81 0.000000 0.000000 0.000000 +0.80 0.81 0.000000 0.000000 0.000000 +0.81 0.81 0.000000 0.000000 0.000000 +0.82 0.81 0.000000 0.000000 0.000000 +0.83 0.81 0.000000 0.000000 0.000000 +0.83 0.81 0.000000 0.000000 0.000000 +0.84 0.81 0.000000 0.000000 0.000000 +0.85 0.81 0.000000 0.000000 0.000000 +0.86 0.81 0.000000 0.000000 0.000000 +0.88 0.81 0.000000 0.000000 0.000000 +0.89 0.81 0.000000 0.000000 0.000000 +0.90 0.81 0.000000 0.000000 0.000000 +0.91 0.81 0.000000 0.000000 0.000000 +0.92 0.81 0.000000 0.000000 0.000000 +0.93 0.81 0.000000 0.000000 0.000000 +0.94 0.81 0.000000 0.000000 0.000000 +0.95 0.81 0.000000 0.000000 0.000000 +0.96 0.81 0.000000 0.000000 0.000000 +0.96 0.81 0.000000 0.000000 0.000000 +0.97 0.81 0.000000 0.000000 0.000000 +0.98 0.81 0.000000 0.000000 0.000000 +0.01 0.82 0.000000 0.000000 0.000000 +0.01 0.82 0.000000 0.000000 0.000000 +0.03 0.82 0.000000 0.000000 0.000000 +0.04 0.82 0.000000 0.000000 0.000000 +0.04 0.82 0.000000 0.000000 0.000000 +0.06 0.82 0.000000 0.000000 0.000000 +0.07 0.82 0.000000 0.000000 0.000000 +0.07 0.82 0.000000 0.000000 0.000000 +0.09 0.82 0.000000 0.000000 0.000000 +0.10 0.82 0.000000 0.000000 0.000000 +0.10 0.82 0.000000 0.000000 0.000000 +0.12 0.82 0.000000 0.000000 0.000000 +0.12 0.82 0.000000 0.000000 0.000000 +0.14 0.82 0.000000 0.000000 0.000000 +0.14 0.82 0.000000 0.000000 0.000000 +0.15 0.82 0.000000 0.000000 0.000000 +0.17 0.82 0.000000 0.000000 0.000000 +0.18 0.82 0.000000 0.000000 0.000000 +0.18 0.82 0.000000 0.000000 0.000000 +0.20 0.82 0.000000 0.000000 0.000000 +0.21 0.82 0.000000 0.000000 0.000000 +0.21 0.82 0.000000 0.000000 0.000000 +0.23 0.82 0.000000 0.000000 0.000000 +0.24 0.82 0.000000 0.000000 0.000000 +0.24 0.82 0.000000 0.000000 0.000000 +0.26 0.82 0.000000 0.000000 0.000000 +0.27 0.82 0.000000 0.000000 0.000000 +0.28 0.82 0.000000 0.000000 0.000000 +0.29 0.82 0.000000 0.000000 0.000000 +0.29 0.82 0.000000 0.000000 0.000000 +0.30 0.82 0.000000 0.000000 0.000000 +0.32 0.82 0.000000 0.000000 0.000000 +0.33 0.82 0.000000 0.000000 0.000000 +0.34 0.82 0.000000 0.000000 0.000000 +0.35 0.82 0.000000 0.000000 0.000000 +0.35 0.82 0.000000 0.000000 0.000000 +0.36 0.82 0.000000 0.000000 0.000000 +0.38 0.82 0.000000 0.000000 0.000000 +0.39 0.82 0.000000 0.000000 0.000000 +0.40 0.82 0.000000 0.000000 0.000000 +0.41 0.82 0.000000 0.000000 0.000000 +0.42 0.82 0.000000 0.000000 0.000000 +0.42 0.82 0.000000 0.000000 0.000000 +0.43 0.82 0.000000 0.000000 0.000000 +0.45 0.82 0.000000 0.000000 0.000000 +0.46 0.82 0.000000 0.000000 0.000000 +0.47 0.82 0.000000 0.000000 0.000000 +0.48 0.82 0.000000 0.000000 0.000000 +0.48 0.82 0.000000 0.000000 0.000000 +0.49 0.82 0.000000 0.000000 0.000000 +0.51 0.82 0.000000 0.000000 0.000000 +0.52 0.82 0.000000 0.000000 0.000000 +0.53 0.82 0.000000 0.000000 0.000000 +0.54 0.82 0.000000 0.000000 0.000000 +0.55 0.82 0.000000 0.000000 0.000000 +0.56 0.82 0.000000 0.000000 0.000000 +0.57 0.82 0.000000 0.000000 0.000000 +0.58 0.82 0.000000 0.000000 0.000000 +0.58 0.82 0.000000 0.000000 0.000000 +0.59 0.82 0.000000 0.000000 0.000000 +0.60 0.82 0.000000 0.000000 0.000000 +0.61 0.82 0.000000 0.000000 0.000000 +0.62 0.82 0.000000 0.000000 0.000000 +0.64 0.82 0.000000 0.000000 0.000000 +0.65 0.82 0.000000 0.000000 0.000000 +0.66 0.82 0.000000 0.000000 0.000000 +0.67 0.82 0.000000 0.000000 0.000000 +0.68 0.82 0.000000 0.000000 0.000000 +0.69 0.82 0.000000 0.000000 0.000000 +0.70 0.82 0.000000 0.000000 0.000000 +0.70 0.82 0.000000 0.000000 0.000000 +0.71 0.82 0.000000 0.000000 0.000000 +0.72 0.82 0.000000 0.000000 0.000000 +0.73 0.82 0.000000 0.000000 0.000000 +0.74 0.82 0.000000 0.000000 0.000000 +0.76 0.82 0.000000 0.000000 0.000000 +0.77 0.82 0.000000 0.000000 0.000000 +0.78 0.82 0.000000 0.000000 0.000000 +0.79 0.82 0.000000 0.000000 0.000000 +0.80 0.82 0.000000 0.000000 0.000000 +0.81 0.82 0.000000 0.000000 0.000000 +0.82 0.82 0.000000 0.000000 0.000000 +0.83 0.82 0.000000 0.000000 0.000000 +0.83 0.82 0.000000 0.000000 0.000000 +0.84 0.82 0.000000 0.000000 0.000000 +0.85 0.82 0.000000 0.000000 0.000000 +0.86 0.82 0.000000 0.000000 0.000000 +0.88 0.82 0.000000 0.000000 0.000000 +0.89 0.82 0.000000 0.000000 0.000000 +0.90 0.82 0.000000 0.000000 0.000000 +0.91 0.82 0.000000 0.000000 0.000000 +0.92 0.82 0.000000 0.000000 0.000000 +0.93 0.82 0.000000 0.000000 0.000000 +0.94 0.82 0.000000 0.000000 0.000000 +0.95 0.82 0.000000 0.000000 0.000000 +0.96 0.82 0.000000 0.000000 0.000000 +0.96 0.82 0.000000 0.000000 0.000000 +0.97 0.82 0.000000 0.000000 0.000000 +0.98 0.82 0.000000 0.000000 0.000000 +0.01 0.83 0.000000 0.000000 0.000000 +0.01 0.83 0.000000 0.000000 0.000000 +0.03 0.83 0.000000 0.000000 0.000000 +0.04 0.83 0.000000 0.000000 0.000000 +0.04 0.83 0.000000 0.000000 0.000000 +0.06 0.83 0.000000 0.000000 0.000000 +0.07 0.83 0.000000 0.000000 0.000000 +0.07 0.83 0.000000 0.000000 0.000000 +0.09 0.83 0.000000 0.000000 0.000000 +0.10 0.83 0.000000 0.000000 0.000000 +0.10 0.83 0.000000 0.000000 0.000000 +0.12 0.83 0.000000 0.000000 0.000000 +0.12 0.83 0.000000 0.000000 0.000000 +0.14 0.83 0.000000 0.000000 0.000000 +0.14 0.83 0.000000 0.000000 0.000000 +0.15 0.83 0.000000 0.000000 0.000000 +0.17 0.83 0.000000 0.000000 0.000000 +0.18 0.83 0.000000 0.000000 0.000000 +0.18 0.83 0.000000 0.000000 0.000000 +0.20 0.83 0.000000 0.000000 0.000000 +0.21 0.83 0.000000 0.000000 0.000000 +0.21 0.83 0.000000 0.000000 0.000000 +0.23 0.83 0.000000 0.000000 0.000000 +0.24 0.83 0.000000 0.000000 0.000000 +0.24 0.83 0.000000 0.000000 0.000000 +0.26 0.83 0.000000 0.000000 0.000000 +0.27 0.83 0.000000 0.000000 0.000000 +0.28 0.83 0.000000 0.000000 0.000000 +0.29 0.83 0.000000 0.000000 0.000000 +0.29 0.83 0.000000 0.000000 0.000000 +0.30 0.83 0.000000 0.000000 0.000000 +0.32 0.83 0.000000 0.000000 0.000000 +0.33 0.83 0.000000 0.000000 0.000000 +0.34 0.83 0.000000 0.000000 0.000000 +0.35 0.83 0.000000 0.000000 0.000000 +0.35 0.83 0.000000 0.000000 0.000000 +0.36 0.83 0.000000 0.000000 0.000000 +0.38 0.83 0.000000 0.000000 0.000000 +0.39 0.83 0.000000 0.000000 0.000000 +0.40 0.83 0.000000 0.000000 0.000000 +0.41 0.83 0.000000 0.000000 0.000000 +0.42 0.83 0.000000 0.000000 0.000000 +0.42 0.83 0.000000 0.000000 0.000000 +0.43 0.83 0.000000 0.000000 0.000000 +0.45 0.83 0.000000 0.000000 0.000000 +0.46 0.83 0.000000 0.000000 0.000000 +0.47 0.83 0.000000 0.000000 0.000000 +0.48 0.83 0.000000 0.000000 0.000000 +0.48 0.83 0.000000 0.000000 0.000000 +0.49 0.83 0.000000 0.000000 0.000000 +0.51 0.83 0.000000 0.000000 0.000000 +0.52 0.83 0.000000 0.000000 0.000000 +0.53 0.83 0.000000 0.000000 0.000000 +0.54 0.83 0.000000 0.000000 0.000000 +0.55 0.83 0.000000 0.000000 0.000000 +0.56 0.83 0.000000 0.000000 0.000000 +0.57 0.83 0.000000 0.000000 0.000000 +0.58 0.83 0.000000 0.000000 0.000000 +0.58 0.83 0.000000 0.000000 0.000000 +0.59 0.83 0.000000 0.000000 0.000000 +0.60 0.83 0.000000 0.000000 0.000000 +0.61 0.83 0.000000 0.000000 0.000000 +0.62 0.83 0.000000 0.000000 0.000000 +0.64 0.83 0.000000 0.000000 0.000000 +0.65 0.83 0.000000 0.000000 0.000000 +0.66 0.83 0.000000 0.000000 0.000000 +0.67 0.83 0.000000 0.000000 0.000000 +0.68 0.83 0.000000 0.000000 0.000000 +0.69 0.83 0.000000 0.000000 0.000000 +0.70 0.83 0.000000 0.000000 0.000000 +0.70 0.83 0.000000 0.000000 0.000000 +0.71 0.83 0.000000 0.000000 0.000000 +0.72 0.83 0.000000 0.000000 0.000000 +0.73 0.83 0.000000 0.000000 0.000000 +0.74 0.83 0.000000 0.000000 0.000000 +0.76 0.83 0.000000 0.000000 0.000000 +0.77 0.83 0.000000 0.000000 0.000000 +0.78 0.83 0.000000 0.000000 0.000000 +0.79 0.83 0.000000 0.000000 0.000000 +0.80 0.83 0.000000 0.000000 0.000000 +0.81 0.83 0.000000 0.000000 0.000000 +0.82 0.83 0.000000 0.000000 0.000000 +0.83 0.83 0.000000 0.000000 0.000000 +0.83 0.83 0.000000 0.000000 0.000000 +0.84 0.83 0.000000 0.000000 0.000000 +0.85 0.83 0.000000 0.000000 0.000000 +0.86 0.83 0.000000 0.000000 0.000000 +0.88 0.83 0.000000 0.000000 0.000000 +0.89 0.83 0.000000 0.000000 0.000000 +0.90 0.83 0.000000 0.000000 0.000000 +0.91 0.83 0.000000 0.000000 0.000000 +0.92 0.83 0.000000 0.000000 0.000000 +0.93 0.83 0.000000 0.000000 0.000000 +0.94 0.83 0.000000 0.000000 0.000000 +0.95 0.83 0.000000 0.000000 0.000000 +0.96 0.83 0.000000 0.000000 0.000000 +0.96 0.83 0.000000 0.000000 0.000000 +0.97 0.83 0.000000 0.000000 0.000000 +0.98 0.83 0.000000 0.000000 0.000000 +0.01 0.83 0.000000 0.000000 0.000000 +0.01 0.83 0.000000 0.000000 0.000000 +0.03 0.83 0.000000 0.000000 0.000000 +0.04 0.83 0.000000 0.000000 0.000000 +0.04 0.83 0.000000 0.000000 0.000000 +0.06 0.83 0.000000 0.000000 0.000000 +0.07 0.83 0.000000 0.000000 0.000000 +0.07 0.83 0.000000 0.000000 0.000000 +0.09 0.83 0.000000 0.000000 0.000000 +0.10 0.83 0.000000 0.000000 0.000000 +0.10 0.83 0.000000 0.000000 0.000000 +0.12 0.83 0.000000 0.000000 0.000000 +0.12 0.83 0.000000 0.000000 0.000000 +0.14 0.83 0.000000 0.000000 0.000000 +0.14 0.83 0.000000 0.000000 0.000000 +0.15 0.83 0.000000 0.000000 0.000000 +0.17 0.83 0.000000 0.000000 0.000000 +0.18 0.83 0.000000 0.000000 0.000000 +0.18 0.83 0.000000 0.000000 0.000000 +0.20 0.83 0.000000 0.000000 0.000000 +0.21 0.83 0.000000 0.000000 0.000000 +0.21 0.83 0.000000 0.000000 0.000000 +0.23 0.83 0.000000 0.000000 0.000000 +0.24 0.83 0.000000 0.000000 0.000000 +0.24 0.83 0.000000 0.000000 0.000000 +0.26 0.83 0.000000 0.000000 0.000000 +0.27 0.83 0.000000 0.000000 0.000000 +0.28 0.83 0.000000 0.000000 0.000000 +0.29 0.83 0.000000 0.000000 0.000000 +0.29 0.83 0.000000 0.000000 0.000000 +0.30 0.83 0.000000 0.000000 0.000000 +0.32 0.83 0.000000 0.000000 0.000000 +0.33 0.83 0.000000 0.000000 0.000000 +0.34 0.83 0.000000 0.000000 0.000000 +0.35 0.83 0.000000 0.000000 0.000000 +0.35 0.83 0.000000 0.000000 0.000000 +0.36 0.83 0.000000 0.000000 0.000000 +0.38 0.83 0.000000 0.000000 0.000000 +0.39 0.83 0.000000 0.000000 0.000000 +0.40 0.83 0.000000 0.000000 0.000000 +0.41 0.83 0.000000 0.000000 0.000000 +0.42 0.83 0.000000 0.000000 0.000000 +0.42 0.83 0.000000 0.000000 0.000000 +0.43 0.83 0.000000 0.000000 0.000000 +0.45 0.83 0.000000 0.000000 0.000000 +0.46 0.83 0.000000 0.000000 0.000000 +0.47 0.83 0.000000 0.000000 0.000000 +0.48 0.83 0.000000 0.000000 0.000000 +0.48 0.83 0.000000 0.000000 0.000000 +0.49 0.83 0.000000 0.000000 0.000000 +0.51 0.83 0.000000 0.000000 0.000000 +0.52 0.83 0.000000 0.000000 0.000000 +0.53 0.83 0.000000 0.000000 0.000000 +0.54 0.83 0.000000 0.000000 0.000000 +0.55 0.83 0.000000 0.000000 0.000000 +0.56 0.83 0.000000 0.000000 0.000000 +0.57 0.83 0.000000 0.000000 0.000000 +0.58 0.83 0.000000 0.000000 0.000000 +0.58 0.83 0.000000 0.000000 0.000000 +0.59 0.83 0.000000 0.000000 0.000000 +0.60 0.83 0.000000 0.000000 0.000000 +0.61 0.83 0.000000 0.000000 0.000000 +0.62 0.83 0.000000 0.000000 0.000000 +0.64 0.83 0.000000 0.000000 0.000000 +0.65 0.83 0.000000 0.000000 0.000000 +0.66 0.83 0.000000 0.000000 0.000000 +0.67 0.83 0.000000 0.000000 0.000000 +0.68 0.83 0.000000 0.000000 0.000000 +0.69 0.83 0.000000 0.000000 0.000000 +0.70 0.83 0.000000 0.000000 0.000000 +0.70 0.83 0.000000 0.000000 0.000000 +0.71 0.83 0.000000 0.000000 0.000000 +0.72 0.83 0.000000 0.000000 0.000000 +0.73 0.83 0.000000 0.000000 0.000000 +0.74 0.83 0.000000 0.000000 0.000000 +0.76 0.83 0.000000 0.000000 0.000000 +0.77 0.83 0.000000 0.000000 0.000000 +0.78 0.83 0.000000 0.000000 0.000000 +0.79 0.83 0.000000 0.000000 0.000000 +0.80 0.83 0.000000 0.000000 0.000000 +0.81 0.83 0.000000 0.000000 0.000000 +0.82 0.83 0.000000 0.000000 0.000000 +0.83 0.83 0.000000 0.000000 0.000000 +0.83 0.83 0.000000 0.000000 0.000000 +0.84 0.83 0.000000 0.000000 0.000000 +0.85 0.83 0.000000 0.000000 0.000000 +0.86 0.83 0.000000 0.000000 0.000000 +0.88 0.83 0.000000 0.000000 0.000000 +0.89 0.83 0.000000 0.000000 0.000000 +0.90 0.83 0.000000 0.000000 0.000000 +0.91 0.83 0.000000 0.000000 0.000000 +0.92 0.83 0.000000 0.000000 0.000000 +0.93 0.83 0.000000 0.000000 0.000000 +0.94 0.83 0.000000 0.000000 0.000000 +0.95 0.83 0.000000 0.000000 0.000000 +0.96 0.83 0.000000 0.000000 0.000000 +0.96 0.83 0.000000 0.000000 0.000000 +0.97 0.83 0.000000 0.000000 0.000000 +0.98 0.83 0.000000 0.000000 0.000000 +0.01 0.84 0.000000 0.000000 0.000000 +0.01 0.84 0.000000 0.000000 0.000000 +0.03 0.84 0.000000 0.000000 0.000000 +0.04 0.84 0.000000 0.000000 0.000000 +0.04 0.84 0.000000 0.000000 0.000000 +0.06 0.84 0.000000 0.000000 0.000000 +0.07 0.84 0.000000 0.000000 0.000000 +0.07 0.84 0.000000 0.000000 0.000000 +0.09 0.84 0.000000 0.000000 0.000000 +0.10 0.84 0.000000 0.000000 0.000000 +0.10 0.84 0.000000 0.000000 0.000000 +0.12 0.84 0.000000 0.000000 0.000000 +0.12 0.84 0.000000 0.000000 0.000000 +0.14 0.84 0.000000 0.000000 0.000000 +0.14 0.84 0.000000 0.000000 0.000000 +0.15 0.84 0.000000 0.000000 0.000000 +0.17 0.84 0.000000 0.000000 0.000000 +0.18 0.84 0.000000 0.000000 0.000000 +0.18 0.84 0.000000 0.000000 0.000000 +0.20 0.84 0.000000 0.000000 0.000000 +0.21 0.84 0.000000 0.000000 0.000000 +0.21 0.84 0.000000 0.000000 0.000000 +0.23 0.84 0.000000 0.000000 0.000000 +0.24 0.84 0.000000 0.000000 0.000000 +0.24 0.84 0.000000 0.000000 0.000000 +0.26 0.84 0.000000 0.000000 0.000000 +0.27 0.84 0.000000 0.000000 0.000000 +0.28 0.84 0.000000 0.000000 0.000000 +0.29 0.84 0.000000 0.000000 0.000000 +0.29 0.84 0.000000 0.000000 0.000000 +0.30 0.84 0.000000 0.000000 0.000000 +0.32 0.84 0.000000 0.000000 0.000000 +0.33 0.84 0.000000 0.000000 0.000000 +0.34 0.84 0.000000 0.000000 0.000000 +0.35 0.84 0.000000 0.000000 0.000000 +0.35 0.84 0.000000 0.000000 0.000000 +0.36 0.84 0.000000 0.000000 0.000000 +0.38 0.84 0.000000 0.000000 0.000000 +0.39 0.84 0.000000 0.000000 0.000000 +0.40 0.84 0.000000 0.000000 0.000000 +0.41 0.84 0.000000 0.000000 0.000000 +0.42 0.84 0.000000 0.000000 0.000000 +0.42 0.84 0.000000 0.000000 0.000000 +0.43 0.84 0.000000 0.000000 0.000000 +0.45 0.84 0.000000 0.000000 0.000000 +0.46 0.84 0.000000 0.000000 0.000000 +0.47 0.84 0.000000 0.000000 0.000000 +0.48 0.84 0.000000 0.000000 0.000000 +0.48 0.84 0.000000 0.000000 0.000000 +0.49 0.84 0.000000 0.000000 0.000000 +0.51 0.84 0.000000 0.000000 0.000000 +0.52 0.84 0.000000 0.000000 0.000000 +0.53 0.84 0.000000 0.000000 0.000000 +0.54 0.84 0.000000 0.000000 0.000000 +0.55 0.84 0.000000 0.000000 0.000000 +0.56 0.84 0.000000 0.000000 0.000000 +0.57 0.84 0.000000 0.000000 0.000000 +0.58 0.84 0.000000 0.000000 0.000000 +0.58 0.84 0.000000 0.000000 0.000000 +0.59 0.84 0.000000 0.000000 0.000000 +0.60 0.84 0.000000 0.000000 0.000000 +0.61 0.84 0.000000 0.000000 0.000000 +0.62 0.84 0.000000 0.000000 0.000000 +0.64 0.84 0.000000 0.000000 0.000000 +0.65 0.84 0.000000 0.000000 0.000000 +0.66 0.84 0.000000 0.000000 0.000000 +0.67 0.84 0.000000 0.000000 0.000000 +0.68 0.84 0.000000 0.000000 0.000000 +0.69 0.84 0.000000 0.000000 0.000000 +0.70 0.84 0.000000 0.000000 0.000000 +0.70 0.84 0.000000 0.000000 0.000000 +0.71 0.84 0.000000 0.000000 0.000000 +0.72 0.84 0.000000 0.000000 0.000000 +0.73 0.84 0.000000 0.000000 0.000000 +0.74 0.84 0.000000 0.000000 0.000000 +0.76 0.84 0.000000 0.000000 0.000000 +0.77 0.84 0.000000 0.000000 0.000000 +0.78 0.84 0.000000 0.000000 0.000000 +0.79 0.84 0.000000 0.000000 0.000000 +0.80 0.84 0.000000 0.000000 0.000000 +0.81 0.84 0.000000 0.000000 0.000000 +0.82 0.84 0.000000 0.000000 0.000000 +0.83 0.84 0.000000 0.000000 0.000000 +0.83 0.84 0.000000 0.000000 0.000000 +0.84 0.84 0.000000 0.000000 0.000000 +0.85 0.84 0.000000 0.000000 0.000000 +0.86 0.84 0.000000 0.000000 0.000000 +0.88 0.84 0.000000 0.000000 0.000000 +0.89 0.84 0.000000 0.000000 0.000000 +0.90 0.84 0.000000 0.000000 0.000000 +0.91 0.84 0.000000 0.000000 0.000000 +0.92 0.84 0.000000 0.000000 0.000000 +0.93 0.84 0.000000 0.000000 0.000000 +0.94 0.84 0.000000 0.000000 0.000000 +0.95 0.84 0.000000 0.000000 0.000000 +0.96 0.84 0.000000 0.000000 0.000000 +0.96 0.84 0.000000 0.000000 0.000000 +0.97 0.84 0.000000 0.000000 0.000000 +0.98 0.84 0.000000 0.000000 0.000000 +0.01 0.85 0.000000 0.000000 0.000000 +0.01 0.85 0.000000 0.000000 0.000000 +0.03 0.85 0.000000 0.000000 0.000000 +0.04 0.85 0.000000 0.000000 0.000000 +0.04 0.85 0.000000 0.000000 0.000000 +0.06 0.85 0.000000 0.000000 0.000000 +0.07 0.85 0.000000 0.000000 0.000000 +0.07 0.85 0.000000 0.000000 0.000000 +0.09 0.85 0.000000 0.000000 0.000000 +0.10 0.85 0.000000 0.000000 0.000000 +0.10 0.85 0.000000 0.000000 0.000000 +0.12 0.85 0.000000 0.000000 0.000000 +0.12 0.85 0.000000 0.000000 0.000000 +0.14 0.85 0.000000 0.000000 0.000000 +0.14 0.85 0.000000 0.000000 0.000000 +0.15 0.85 0.000000 0.000000 0.000000 +0.17 0.85 0.000000 0.000000 0.000000 +0.18 0.85 0.000000 0.000000 0.000000 +0.18 0.85 0.000000 0.000000 0.000000 +0.20 0.85 0.000000 0.000000 0.000000 +0.21 0.85 0.000000 0.000000 0.000000 +0.21 0.85 0.000000 0.000000 0.000000 +0.23 0.85 0.000000 0.000000 0.000000 +0.24 0.85 0.000000 0.000000 0.000000 +0.24 0.85 0.000000 0.000000 0.000000 +0.26 0.85 0.000000 0.000000 0.000000 +0.27 0.85 0.000000 0.000000 0.000000 +0.28 0.85 0.000000 0.000000 0.000000 +0.29 0.85 0.000000 0.000000 0.000000 +0.29 0.85 0.000000 0.000000 0.000000 +0.30 0.85 0.000000 0.000000 0.000000 +0.32 0.85 0.000000 0.000000 0.000000 +0.33 0.85 0.000000 0.000000 0.000000 +0.34 0.85 0.000000 0.000000 0.000000 +0.35 0.85 0.000000 0.000000 0.000000 +0.35 0.85 0.000000 0.000000 0.000000 +0.36 0.85 0.000000 0.000000 0.000000 +0.38 0.85 0.000000 0.000000 0.000000 +0.39 0.85 0.000000 0.000000 0.000000 +0.40 0.85 0.000000 0.000000 0.000000 +0.41 0.85 0.000000 0.000000 0.000000 +0.42 0.85 0.000000 0.000000 0.000000 +0.42 0.85 0.000000 0.000000 0.000000 +0.43 0.85 0.000000 0.000000 0.000000 +0.45 0.85 0.000000 0.000000 0.000000 +0.46 0.85 0.000000 0.000000 0.000000 +0.47 0.85 0.000000 0.000000 0.000000 +0.48 0.85 0.000000 0.000000 0.000000 +0.48 0.85 0.000000 0.000000 0.000000 +0.49 0.85 0.000000 0.000000 0.000000 +0.51 0.85 0.000000 0.000000 0.000000 +0.52 0.85 0.000000 0.000000 0.000000 +0.53 0.85 0.000000 0.000000 0.000000 +0.54 0.85 0.000000 0.000000 0.000000 +0.55 0.85 0.000000 0.000000 0.000000 +0.56 0.85 0.000000 0.000000 0.000000 +0.57 0.85 0.000000 0.000000 0.000000 +0.58 0.85 0.000000 0.000000 0.000000 +0.58 0.85 0.000000 0.000000 0.000000 +0.59 0.85 0.000000 0.000000 0.000000 +0.60 0.85 0.000000 0.000000 0.000000 +0.61 0.85 0.000000 0.000000 0.000000 +0.62 0.85 0.000000 0.000000 0.000000 +0.64 0.85 0.000000 0.000000 0.000000 +0.65 0.85 0.000000 0.000000 0.000000 +0.66 0.85 0.000000 0.000000 0.000000 +0.67 0.85 0.000000 0.000000 0.000000 +0.68 0.85 0.000000 0.000000 0.000000 +0.69 0.85 0.000000 0.000000 0.000000 +0.70 0.85 0.000000 0.000000 0.000000 +0.70 0.85 0.000000 0.000000 0.000000 +0.71 0.85 0.000000 0.000000 0.000000 +0.72 0.85 0.000000 0.000000 0.000000 +0.73 0.85 0.000000 0.000000 0.000000 +0.74 0.85 0.000000 0.000000 0.000000 +0.76 0.85 0.000000 0.000000 0.000000 +0.77 0.85 0.000000 0.000000 0.000000 +0.78 0.85 0.000000 0.000000 0.000000 +0.79 0.85 0.000000 0.000000 0.000000 +0.80 0.85 0.000000 0.000000 0.000000 +0.81 0.85 0.000000 0.000000 0.000000 +0.82 0.85 0.000000 0.000000 0.000000 +0.83 0.85 0.000000 0.000000 0.000000 +0.83 0.85 0.000000 0.000000 0.000000 +0.84 0.85 0.000000 0.000000 0.000000 +0.85 0.85 0.000000 0.000000 0.000000 +0.86 0.85 0.000000 0.000000 0.000000 +0.88 0.85 0.000000 0.000000 0.000000 +0.89 0.85 0.000000 0.000000 0.000000 +0.90 0.85 0.000000 0.000000 0.000000 +0.91 0.85 0.000000 0.000000 0.000000 +0.92 0.85 0.000000 0.000000 0.000000 +0.93 0.85 0.000000 0.000000 0.000000 +0.94 0.85 0.000000 0.000000 0.000000 +0.95 0.85 0.000000 0.000000 0.000000 +0.96 0.85 0.000000 0.000000 0.000000 +0.96 0.85 0.000000 0.000000 0.000000 +0.97 0.85 0.000000 0.000000 0.000000 +0.98 0.85 0.000000 0.000000 0.000000 +0.01 0.86 0.000000 0.000000 0.000000 +0.01 0.86 0.000000 0.000000 0.000000 +0.03 0.86 0.000000 0.000000 0.000000 +0.04 0.86 0.000000 0.000000 0.000000 +0.04 0.86 0.000000 0.000000 0.000000 +0.06 0.86 0.000000 0.000000 0.000000 +0.07 0.86 0.000000 0.000000 0.000000 +0.07 0.86 0.000000 0.000000 0.000000 +0.09 0.86 0.000000 0.000000 0.000000 +0.10 0.86 0.000000 0.000000 0.000000 +0.10 0.86 0.000000 0.000000 0.000000 +0.12 0.86 0.000000 0.000000 0.000000 +0.12 0.86 0.000000 0.000000 0.000000 +0.14 0.86 0.000000 0.000000 0.000000 +0.14 0.86 0.000000 0.000000 0.000000 +0.15 0.86 0.000000 0.000000 0.000000 +0.17 0.86 0.000000 0.000000 0.000000 +0.18 0.86 0.000000 0.000000 0.000000 +0.18 0.86 0.000000 0.000000 0.000000 +0.20 0.86 0.000000 0.000000 0.000000 +0.21 0.86 0.000000 0.000000 0.000000 +0.21 0.86 0.000000 0.000000 0.000000 +0.23 0.86 0.000000 0.000000 0.000000 +0.24 0.86 0.000000 0.000000 0.000000 +0.24 0.86 0.000000 0.000000 0.000000 +0.26 0.86 0.000000 0.000000 0.000000 +0.27 0.86 0.000000 0.000000 0.000000 +0.28 0.86 0.000000 0.000000 0.000000 +0.29 0.86 0.000000 0.000000 0.000000 +0.29 0.86 0.000000 0.000000 0.000000 +0.30 0.86 0.000000 0.000000 0.000000 +0.32 0.86 0.000000 0.000000 0.000000 +0.33 0.86 0.000000 0.000000 0.000000 +0.34 0.86 0.000000 0.000000 0.000000 +0.35 0.86 0.000000 0.000000 0.000000 +0.35 0.86 0.000000 0.000000 0.000000 +0.36 0.86 0.000000 0.000000 0.000000 +0.38 0.86 0.000000 0.000000 0.000000 +0.39 0.86 0.000000 0.000000 0.000000 +0.40 0.86 0.000000 0.000000 0.000000 +0.41 0.86 0.000000 0.000000 0.000000 +0.42 0.86 0.000000 0.000000 0.000000 +0.42 0.86 0.000000 0.000000 0.000000 +0.43 0.86 0.000000 0.000000 0.000000 +0.45 0.86 0.000000 0.000000 0.000000 +0.46 0.86 0.000000 0.000000 0.000000 +0.47 0.86 0.000000 0.000000 0.000000 +0.48 0.86 0.000000 0.000000 0.000000 +0.48 0.86 0.000000 0.000000 0.000000 +0.49 0.86 0.000000 0.000000 0.000000 +0.51 0.86 0.000000 0.000000 0.000000 +0.52 0.86 0.000000 0.000000 0.000000 +0.53 0.86 0.000000 0.000000 0.000000 +0.54 0.86 0.000000 0.000000 0.000000 +0.55 0.86 0.000000 0.000000 0.000000 +0.56 0.86 0.000000 0.000000 0.000000 +0.57 0.86 0.000000 0.000000 0.000000 +0.58 0.86 0.000000 0.000000 0.000000 +0.58 0.86 0.000000 0.000000 0.000000 +0.59 0.86 0.000000 0.000000 0.000000 +0.60 0.86 0.000000 0.000000 0.000000 +0.61 0.86 0.000000 0.000000 0.000000 +0.62 0.86 0.000000 0.000000 0.000000 +0.64 0.86 0.000000 0.000000 0.000000 +0.65 0.86 0.000000 0.000000 0.000000 +0.66 0.86 0.000000 0.000000 0.000000 +0.67 0.86 0.000000 0.000000 0.000000 +0.68 0.86 0.000000 0.000000 0.000000 +0.69 0.86 0.000000 0.000000 0.000000 +0.70 0.86 0.000000 0.000000 0.000000 +0.70 0.86 0.000000 0.000000 0.000000 +0.71 0.86 0.000000 0.000000 0.000000 +0.72 0.86 0.000000 0.000000 0.000000 +0.73 0.86 0.000000 0.000000 0.000000 +0.74 0.86 0.000000 0.000000 0.000000 +0.76 0.86 0.000000 0.000000 0.000000 +0.77 0.86 0.000000 0.000000 0.000000 +0.78 0.86 0.000000 0.000000 0.000000 +0.79 0.86 0.000000 0.000000 0.000000 +0.80 0.86 0.000000 0.000000 0.000000 +0.81 0.86 0.000000 0.000000 0.000000 +0.82 0.86 0.000000 0.000000 0.000000 +0.83 0.86 0.000000 0.000000 0.000000 +0.83 0.86 0.000000 0.000000 0.000000 +0.84 0.86 0.000000 0.000000 0.000000 +0.85 0.86 0.000000 0.000000 0.000000 +0.86 0.86 0.000000 0.000000 0.000000 +0.88 0.86 0.000000 0.000000 0.000000 +0.89 0.86 0.000000 0.000000 0.000000 +0.90 0.86 0.000000 0.000000 0.000000 +0.91 0.86 0.000000 0.000000 0.000000 +0.92 0.86 0.000000 0.000000 0.000000 +0.93 0.86 0.000000 0.000000 0.000000 +0.94 0.86 0.000000 0.000000 0.000000 +0.95 0.86 0.000000 0.000000 0.000000 +0.96 0.86 0.000000 0.000000 0.000000 +0.96 0.86 0.000000 0.000000 0.000000 +0.97 0.86 0.000000 0.000000 0.000000 +0.98 0.86 0.000000 0.000000 0.000000 +0.01 0.88 0.000000 0.000000 0.000000 +0.01 0.88 0.000000 0.000000 0.000000 +0.03 0.88 0.000000 0.000000 0.000000 +0.04 0.88 0.000000 0.000000 0.000000 +0.04 0.88 0.000000 0.000000 0.000000 +0.06 0.88 0.000000 0.000000 0.000000 +0.07 0.88 0.000000 0.000000 0.000000 +0.07 0.88 0.000000 0.000000 0.000000 +0.09 0.88 0.000000 0.000000 0.000000 +0.10 0.88 0.000000 0.000000 0.000000 +0.10 0.88 0.000000 0.000000 0.000000 +0.12 0.88 0.000000 0.000000 0.000000 +0.12 0.88 0.000000 0.000000 0.000000 +0.14 0.88 0.000000 0.000000 0.000000 +0.14 0.88 0.000000 0.000000 0.000000 +0.15 0.88 0.000000 0.000000 0.000000 +0.17 0.88 0.000000 0.000000 0.000000 +0.18 0.88 0.000000 0.000000 0.000000 +0.18 0.88 0.000000 0.000000 0.000000 +0.20 0.88 0.000000 0.000000 0.000000 +0.21 0.88 0.000000 0.000000 0.000000 +0.21 0.88 0.000000 0.000000 0.000000 +0.23 0.88 0.000000 0.000000 0.000000 +0.24 0.88 0.000000 0.000000 0.000000 +0.24 0.88 0.000000 0.000000 0.000000 +0.26 0.88 0.000000 0.000000 0.000000 +0.27 0.88 0.000000 0.000000 0.000000 +0.28 0.88 0.000000 0.000000 0.000000 +0.29 0.88 0.000000 0.000000 0.000000 +0.29 0.88 0.000000 0.000000 0.000000 +0.30 0.88 0.000000 0.000000 0.000000 +0.32 0.88 0.000000 0.000000 0.000000 +0.33 0.88 0.000000 0.000000 0.000000 +0.34 0.88 0.000000 0.000000 0.000000 +0.35 0.88 0.000000 0.000000 0.000000 +0.35 0.88 0.000000 0.000000 0.000000 +0.36 0.88 0.000000 0.000000 0.000000 +0.38 0.88 0.000000 0.000000 0.000000 +0.39 0.88 0.000000 0.000000 0.000000 +0.40 0.88 0.000000 0.000000 0.000000 +0.41 0.88 0.000000 0.000000 0.000000 +0.42 0.88 0.000000 0.000000 0.000000 +0.42 0.88 0.000000 0.000000 0.000000 +0.43 0.88 0.000000 0.000000 0.000000 +0.45 0.88 0.000000 0.000000 0.000000 +0.46 0.88 0.000000 0.000000 0.000000 +0.47 0.88 0.000000 0.000000 0.000000 +0.48 0.88 0.000000 0.000000 0.000000 +0.48 0.88 0.000000 0.000000 0.000000 +0.49 0.88 0.000000 0.000000 0.000000 +0.51 0.88 0.000000 0.000000 0.000000 +0.52 0.88 0.000000 0.000000 0.000000 +0.53 0.88 0.000000 0.000000 0.000000 +0.54 0.88 0.000000 0.000000 0.000000 +0.55 0.88 0.000000 0.000000 0.000000 +0.56 0.88 0.000000 0.000000 0.000000 +0.57 0.88 0.000000 0.000000 0.000000 +0.58 0.88 0.000000 0.000000 0.000000 +0.58 0.88 0.000000 0.000000 0.000000 +0.59 0.88 0.000000 0.000000 0.000000 +0.60 0.88 0.000000 0.000000 0.000000 +0.61 0.88 0.000000 0.000000 0.000000 +0.62 0.88 0.000000 0.000000 0.000000 +0.64 0.88 0.000000 0.000000 0.000000 +0.65 0.88 0.000000 0.000000 0.000000 +0.66 0.88 0.000000 0.000000 0.000000 +0.67 0.88 0.000000 0.000000 0.000000 +0.68 0.88 0.000000 0.000000 0.000000 +0.69 0.88 0.000000 0.000000 0.000000 +0.70 0.88 0.000000 0.000000 0.000000 +0.70 0.88 0.000000 0.000000 0.000000 +0.71 0.88 0.000000 0.000000 0.000000 +0.72 0.88 0.000000 0.000000 0.000000 +0.73 0.88 0.000000 0.000000 0.000000 +0.74 0.88 0.000000 0.000000 0.000000 +0.76 0.88 0.000000 0.000000 0.000000 +0.77 0.88 0.000000 0.000000 0.000000 +0.78 0.88 0.000000 0.000000 0.000000 +0.79 0.88 0.000000 0.000000 0.000000 +0.80 0.88 0.000000 0.000000 0.000000 +0.81 0.88 0.000000 0.000000 0.000000 +0.82 0.88 0.000000 0.000000 0.000000 +0.83 0.88 0.000000 0.000000 0.000000 +0.83 0.88 0.000000 0.000000 0.000000 +0.84 0.88 0.000000 0.000000 0.000000 +0.85 0.88 0.000000 0.000000 0.000000 +0.86 0.88 0.000000 0.000000 0.000000 +0.88 0.88 0.000000 0.000000 0.000000 +0.89 0.88 0.000000 0.000000 0.000000 +0.90 0.88 0.000000 0.000000 0.000000 +0.91 0.88 0.000000 0.000000 0.000000 +0.92 0.88 0.000000 0.000000 0.000000 +0.93 0.88 0.000000 0.000000 0.000000 +0.94 0.88 0.000000 0.000000 0.000000 +0.95 0.88 0.000000 0.000000 0.000000 +0.96 0.88 0.000000 0.000000 0.000000 +0.96 0.88 0.000000 0.000000 0.000000 +0.97 0.88 0.000000 0.000000 0.000000 +0.98 0.88 0.000000 0.000000 0.000000 +0.01 0.89 0.000000 0.000000 0.000000 +0.01 0.89 0.000000 0.000000 0.000000 +0.03 0.89 0.000000 0.000000 0.000000 +0.04 0.89 0.000000 0.000000 0.000000 +0.04 0.89 0.000000 0.000000 0.000000 +0.06 0.89 0.000000 0.000000 0.000000 +0.07 0.89 0.000000 0.000000 0.000000 +0.07 0.89 0.000000 0.000000 0.000000 +0.09 0.89 0.000000 0.000000 0.000000 +0.10 0.89 0.000000 0.000000 0.000000 +0.10 0.89 0.000000 0.000000 0.000000 +0.12 0.89 0.000000 0.000000 0.000000 +0.12 0.89 0.000000 0.000000 0.000000 +0.14 0.89 0.000000 0.000000 0.000000 +0.14 0.89 0.000000 0.000000 0.000000 +0.15 0.89 0.000000 0.000000 0.000000 +0.17 0.89 0.000000 0.000000 0.000000 +0.18 0.89 0.000000 0.000000 0.000000 +0.18 0.89 0.000000 0.000000 0.000000 +0.20 0.89 0.000000 0.000000 0.000000 +0.21 0.89 0.000000 0.000000 0.000000 +0.21 0.89 0.000000 0.000000 0.000000 +0.23 0.89 0.000000 0.000000 0.000000 +0.24 0.89 0.000000 0.000000 0.000000 +0.24 0.89 0.000000 0.000000 0.000000 +0.26 0.89 0.000000 0.000000 0.000000 +0.27 0.89 0.000000 0.000000 0.000000 +0.28 0.89 0.000000 0.000000 0.000000 +0.29 0.89 0.000000 0.000000 0.000000 +0.29 0.89 0.000000 0.000000 0.000000 +0.30 0.89 0.000000 0.000000 0.000000 +0.32 0.89 0.000000 0.000000 0.000000 +0.33 0.89 0.000000 0.000000 0.000000 +0.34 0.89 0.000000 0.000000 0.000000 +0.35 0.89 0.000000 0.000000 0.000000 +0.35 0.89 0.000000 0.000000 0.000000 +0.36 0.89 0.000000 0.000000 0.000000 +0.38 0.89 0.000000 0.000000 0.000000 +0.39 0.89 0.000000 0.000000 0.000000 +0.40 0.89 0.000000 0.000000 0.000000 +0.41 0.89 0.000000 0.000000 0.000000 +0.42 0.89 0.000000 0.000000 0.000000 +0.42 0.89 0.000000 0.000000 0.000000 +0.43 0.89 0.000000 0.000000 0.000000 +0.45 0.89 0.000000 0.000000 0.000000 +0.46 0.89 0.000000 0.000000 0.000000 +0.47 0.89 0.000000 0.000000 0.000000 +0.48 0.89 0.000000 0.000000 0.000000 +0.48 0.89 0.000000 0.000000 0.000000 +0.49 0.89 0.000000 0.000000 0.000000 +0.51 0.89 0.000000 0.000000 0.000000 +0.52 0.89 0.000000 0.000000 0.000000 +0.53 0.89 0.000000 0.000000 0.000000 +0.54 0.89 0.000000 0.000000 0.000000 +0.55 0.89 0.000000 0.000000 0.000000 +0.56 0.89 0.000000 0.000000 0.000000 +0.57 0.89 0.000000 0.000000 0.000000 +0.58 0.89 0.000000 0.000000 0.000000 +0.58 0.89 0.000000 0.000000 0.000000 +0.59 0.89 0.000000 0.000000 0.000000 +0.60 0.89 0.000000 0.000000 0.000000 +0.61 0.89 0.000000 0.000000 0.000000 +0.62 0.89 0.000000 0.000000 0.000000 +0.64 0.89 0.000000 0.000000 0.000000 +0.65 0.89 0.000000 0.000000 0.000000 +0.66 0.89 0.000000 0.000000 0.000000 +0.67 0.89 0.000000 0.000000 0.000000 +0.68 0.89 0.000000 0.000000 0.000000 +0.69 0.89 0.000000 0.000000 0.000000 +0.70 0.89 0.000000 0.000000 0.000000 +0.70 0.89 0.000000 0.000000 0.000000 +0.71 0.89 0.000000 0.000000 0.000000 +0.72 0.89 0.000000 0.000000 0.000000 +0.73 0.89 0.000000 0.000000 0.000000 +0.74 0.89 0.000000 0.000000 0.000000 +0.76 0.89 0.000000 0.000000 0.000000 +0.77 0.89 0.000000 0.000000 0.000000 +0.78 0.89 0.000000 0.000000 0.000000 +0.79 0.89 0.000000 0.000000 0.000000 +0.80 0.89 0.000000 0.000000 0.000000 +0.81 0.89 0.000000 0.000000 0.000000 +0.82 0.89 0.000000 0.000000 0.000000 +0.83 0.89 0.000000 0.000000 0.000000 +0.83 0.89 0.000000 0.000000 0.000000 +0.84 0.89 0.000000 0.000000 0.000000 +0.85 0.89 0.000000 0.000000 0.000000 +0.86 0.89 0.000000 0.000000 0.000000 +0.88 0.89 0.000000 0.000000 0.000000 +0.89 0.89 0.000000 0.000000 0.000000 +0.90 0.89 0.000000 0.000000 0.000000 +0.91 0.89 0.000000 0.000000 0.000000 +0.92 0.89 0.000000 0.000000 0.000000 +0.93 0.89 0.000000 0.000000 0.000000 +0.94 0.89 0.000000 0.000000 0.000000 +0.95 0.89 0.000000 0.000000 0.000000 +0.96 0.89 0.000000 0.000000 0.000000 +0.96 0.89 0.000000 0.000000 0.000000 +0.97 0.89 0.000000 0.000000 0.000000 +0.98 0.89 0.000000 0.000000 0.000000 +0.01 0.90 0.000000 0.000000 0.000000 +0.01 0.90 0.000000 0.000000 0.000000 +0.03 0.90 0.000000 0.000000 0.000000 +0.04 0.90 0.000000 0.000000 0.000000 +0.04 0.90 0.000000 0.000000 0.000000 +0.06 0.90 0.000000 0.000000 0.000000 +0.07 0.90 0.000000 0.000000 0.000000 +0.07 0.90 0.000000 0.000000 0.000000 +0.09 0.90 0.000000 0.000000 0.000000 +0.10 0.90 0.000000 0.000000 0.000000 +0.10 0.90 0.000000 0.000000 0.000000 +0.12 0.90 0.000000 0.000000 0.000000 +0.12 0.90 0.000000 0.000000 0.000000 +0.14 0.90 0.000000 0.000000 0.000000 +0.14 0.90 0.000000 0.000000 0.000000 +0.15 0.90 0.000000 0.000000 0.000000 +0.17 0.90 0.000000 0.000000 0.000000 +0.18 0.90 0.000000 0.000000 0.000000 +0.18 0.90 0.000000 0.000000 0.000000 +0.20 0.90 0.000000 0.000000 0.000000 +0.21 0.90 0.000000 0.000000 0.000000 +0.21 0.90 0.000000 0.000000 0.000000 +0.23 0.90 0.000000 0.000000 0.000000 +0.24 0.90 0.000000 0.000000 0.000000 +0.24 0.90 0.000000 0.000000 0.000000 +0.26 0.90 0.000000 0.000000 0.000000 +0.27 0.90 0.000000 0.000000 0.000000 +0.28 0.90 0.000000 0.000000 0.000000 +0.29 0.90 0.000000 0.000000 0.000000 +0.29 0.90 0.000000 0.000000 0.000000 +0.30 0.90 0.000000 0.000000 0.000000 +0.32 0.90 0.000000 0.000000 0.000000 +0.33 0.90 0.000000 0.000000 0.000000 +0.34 0.90 0.000000 0.000000 0.000000 +0.35 0.90 0.000000 0.000000 0.000000 +0.35 0.90 0.000000 0.000000 0.000000 +0.36 0.90 0.000000 0.000000 0.000000 +0.38 0.90 0.000000 0.000000 0.000000 +0.39 0.90 0.000000 0.000000 0.000000 +0.40 0.90 0.000000 0.000000 0.000000 +0.41 0.90 0.000000 0.000000 0.000000 +0.42 0.90 0.000000 0.000000 0.000000 +0.42 0.90 0.000000 0.000000 0.000000 +0.43 0.90 0.000000 0.000000 0.000000 +0.45 0.90 0.000000 0.000000 0.000000 +0.46 0.90 0.000000 0.000000 0.000000 +0.47 0.90 0.000000 0.000000 0.000000 +0.48 0.90 0.000000 0.000000 0.000000 +0.48 0.90 0.000000 0.000000 0.000000 +0.49 0.90 0.000000 0.000000 0.000000 +0.51 0.90 0.000000 0.000000 0.000000 +0.52 0.90 0.000000 0.000000 0.000000 +0.53 0.90 0.000000 0.000000 0.000000 +0.54 0.90 0.000000 0.000000 0.000000 +0.55 0.90 0.000000 0.000000 0.000000 +0.56 0.90 0.000000 0.000000 0.000000 +0.57 0.90 0.000000 0.000000 0.000000 +0.58 0.90 0.000000 0.000000 0.000000 +0.58 0.90 0.000000 0.000000 0.000000 +0.59 0.90 0.000000 0.000000 0.000000 +0.60 0.90 0.000000 0.000000 0.000000 +0.61 0.90 0.000000 0.000000 0.000000 +0.62 0.90 0.000000 0.000000 0.000000 +0.64 0.90 0.000000 0.000000 0.000000 +0.65 0.90 0.000000 0.000000 0.000000 +0.66 0.90 0.000000 0.000000 0.000000 +0.67 0.90 0.000000 0.000000 0.000000 +0.68 0.90 0.000000 0.000000 0.000000 +0.69 0.90 0.000000 0.000000 0.000000 +0.70 0.90 0.000000 0.000000 0.000000 +0.70 0.90 0.000000 0.000000 0.000000 +0.71 0.90 0.000000 0.000000 0.000000 +0.72 0.90 0.000000 0.000000 0.000000 +0.73 0.90 0.000000 0.000000 0.000000 +0.74 0.90 0.000000 0.000000 0.000000 +0.76 0.90 0.000000 0.000000 0.000000 +0.77 0.90 0.000000 0.000000 0.000000 +0.78 0.90 0.000000 0.000000 0.000000 +0.79 0.90 0.000000 0.000000 0.000000 +0.80 0.90 0.000000 0.000000 0.000000 +0.81 0.90 0.000000 0.000000 0.000000 +0.82 0.90 0.000000 0.000000 0.000000 +0.83 0.90 0.000000 0.000000 0.000000 +0.83 0.90 0.000000 0.000000 0.000000 +0.84 0.90 0.000000 0.000000 0.000000 +0.85 0.90 0.000000 0.000000 0.000000 +0.86 0.90 0.000000 0.000000 0.000000 +0.88 0.90 0.000000 0.000000 0.000000 +0.89 0.90 0.000000 0.000000 0.000000 +0.90 0.90 0.000000 0.000000 0.000000 +0.91 0.90 0.000000 0.000000 0.000000 +0.92 0.90 0.000000 0.000000 0.000000 +0.93 0.90 0.000000 0.000000 0.000000 +0.94 0.90 0.000000 0.000000 0.000000 +0.95 0.90 0.000000 0.000000 0.000000 +0.96 0.90 0.000000 0.000000 0.000000 +0.96 0.90 0.000000 0.000000 0.000000 +0.97 0.90 0.000000 0.000000 0.000000 +0.98 0.90 0.000000 0.000000 0.000000 +0.01 0.91 0.000000 0.000000 0.000000 +0.01 0.91 0.000000 0.000000 0.000000 +0.03 0.91 0.000000 0.000000 0.000000 +0.04 0.91 0.000000 0.000000 0.000000 +0.04 0.91 0.000000 0.000000 0.000000 +0.06 0.91 0.000000 0.000000 0.000000 +0.07 0.91 0.000000 0.000000 0.000000 +0.07 0.91 0.000000 0.000000 0.000000 +0.09 0.91 0.000000 0.000000 0.000000 +0.10 0.91 0.000000 0.000000 0.000000 +0.10 0.91 0.000000 0.000000 0.000000 +0.12 0.91 0.000000 0.000000 0.000000 +0.12 0.91 0.000000 0.000000 0.000000 +0.14 0.91 0.000000 0.000000 0.000000 +0.14 0.91 0.000000 0.000000 0.000000 +0.15 0.91 0.000000 0.000000 0.000000 +0.17 0.91 0.000000 0.000000 0.000000 +0.18 0.91 0.000000 0.000000 0.000000 +0.18 0.91 0.000000 0.000000 0.000000 +0.20 0.91 0.000000 0.000000 0.000000 +0.21 0.91 0.000000 0.000000 0.000000 +0.21 0.91 0.000000 0.000000 0.000000 +0.23 0.91 0.000000 0.000000 0.000000 +0.24 0.91 0.000000 0.000000 0.000000 +0.24 0.91 0.000000 0.000000 0.000000 +0.26 0.91 0.000000 0.000000 0.000000 +0.27 0.91 0.000000 0.000000 0.000000 +0.28 0.91 0.000000 0.000000 0.000000 +0.29 0.91 0.000000 0.000000 0.000000 +0.29 0.91 0.000000 0.000000 0.000000 +0.30 0.91 0.000000 0.000000 0.000000 +0.32 0.91 0.000000 0.000000 0.000000 +0.33 0.91 0.000000 0.000000 0.000000 +0.34 0.91 0.000000 0.000000 0.000000 +0.35 0.91 0.000000 0.000000 0.000000 +0.35 0.91 0.000000 0.000000 0.000000 +0.36 0.91 0.000000 0.000000 0.000000 +0.38 0.91 0.000000 0.000000 0.000000 +0.39 0.91 0.000000 0.000000 0.000000 +0.40 0.91 0.000000 0.000000 0.000000 +0.41 0.91 0.000000 0.000000 0.000000 +0.42 0.91 0.000000 0.000000 0.000000 +0.42 0.91 0.000000 0.000000 0.000000 +0.43 0.91 0.000000 0.000000 0.000000 +0.45 0.91 0.000000 0.000000 0.000000 +0.46 0.91 0.000000 0.000000 0.000000 +0.47 0.91 0.000000 0.000000 0.000000 +0.48 0.91 0.000000 0.000000 0.000000 +0.48 0.91 0.000000 0.000000 0.000000 +0.49 0.91 0.000000 0.000000 0.000000 +0.51 0.91 0.000000 0.000000 0.000000 +0.52 0.91 0.000000 0.000000 0.000000 +0.53 0.91 0.000000 0.000000 0.000000 +0.54 0.91 0.000000 0.000000 0.000000 +0.55 0.91 0.000000 0.000000 0.000000 +0.56 0.91 0.000000 0.000000 0.000000 +0.57 0.91 0.000000 0.000000 0.000000 +0.58 0.91 0.000000 0.000000 0.000000 +0.58 0.91 0.000000 0.000000 0.000000 +0.59 0.91 0.000000 0.000000 0.000000 +0.60 0.91 0.000000 0.000000 0.000000 +0.61 0.91 0.000000 0.000000 0.000000 +0.62 0.91 0.000000 0.000000 0.000000 +0.64 0.91 0.000000 0.000000 0.000000 +0.65 0.91 0.000000 0.000000 0.000000 +0.66 0.91 0.000000 0.000000 0.000000 +0.67 0.91 0.000000 0.000000 0.000000 +0.68 0.91 0.000000 0.000000 0.000000 +0.69 0.91 0.000000 0.000000 0.000000 +0.70 0.91 0.000000 0.000000 0.000000 +0.70 0.91 0.000000 0.000000 0.000000 +0.71 0.91 0.000000 0.000000 0.000000 +0.72 0.91 0.000000 0.000000 0.000000 +0.73 0.91 0.000000 0.000000 0.000000 +0.74 0.91 0.000000 0.000000 0.000000 +0.76 0.91 0.000000 0.000000 0.000000 +0.77 0.91 0.000000 0.000000 0.000000 +0.78 0.91 0.000000 0.000000 0.000000 +0.79 0.91 0.000000 0.000000 0.000000 +0.80 0.91 0.000000 0.000000 0.000000 +0.81 0.91 0.000000 0.000000 0.000000 +0.82 0.91 0.000000 0.000000 0.000000 +0.83 0.91 0.000000 0.000000 0.000000 +0.83 0.91 0.000000 0.000000 0.000000 +0.84 0.91 0.000000 0.000000 0.000000 +0.85 0.91 0.000000 0.000000 0.000000 +0.86 0.91 0.000000 0.000000 0.000000 +0.88 0.91 0.000000 0.000000 0.000000 +0.89 0.91 0.000000 0.000000 0.000000 +0.90 0.91 0.000000 0.000000 0.000000 +0.91 0.91 0.000000 0.000000 0.000000 +0.92 0.91 0.000000 0.000000 0.000000 +0.93 0.91 0.000000 0.000000 0.000000 +0.94 0.91 0.000000 0.000000 0.000000 +0.95 0.91 0.000000 0.000000 0.000000 +0.96 0.91 0.000000 0.000000 0.000000 +0.96 0.91 0.000000 0.000000 0.000000 +0.97 0.91 0.000000 0.000000 0.000000 +0.98 0.91 0.000000 0.000000 0.000000 +0.01 0.92 0.000000 0.000000 0.000000 +0.01 0.92 0.000000 0.000000 0.000000 +0.03 0.92 0.000000 0.000000 0.000000 +0.04 0.92 0.000000 0.000000 0.000000 +0.04 0.92 0.000000 0.000000 0.000000 +0.06 0.92 0.000000 0.000000 0.000000 +0.07 0.92 0.000000 0.000000 0.000000 +0.07 0.92 0.000000 0.000000 0.000000 +0.09 0.92 0.000000 0.000000 0.000000 +0.10 0.92 0.000000 0.000000 0.000000 +0.10 0.92 0.000000 0.000000 0.000000 +0.12 0.92 0.000000 0.000000 0.000000 +0.12 0.92 0.000000 0.000000 0.000000 +0.14 0.92 0.000000 0.000000 0.000000 +0.14 0.92 0.000000 0.000000 0.000000 +0.15 0.92 0.000000 0.000000 0.000000 +0.17 0.92 0.000000 0.000000 0.000000 +0.18 0.92 0.000000 0.000000 0.000000 +0.18 0.92 0.000000 0.000000 0.000000 +0.20 0.92 0.000000 0.000000 0.000000 +0.21 0.92 0.000000 0.000000 0.000000 +0.21 0.92 0.000000 0.000000 0.000000 +0.23 0.92 0.000000 0.000000 0.000000 +0.24 0.92 0.000000 0.000000 0.000000 +0.24 0.92 0.000000 0.000000 0.000000 +0.26 0.92 0.000000 0.000000 0.000000 +0.27 0.92 0.000000 0.000000 0.000000 +0.28 0.92 0.000000 0.000000 0.000000 +0.29 0.92 0.000000 0.000000 0.000000 +0.29 0.92 0.000000 0.000000 0.000000 +0.30 0.92 0.000000 0.000000 0.000000 +0.32 0.92 0.000000 0.000000 0.000000 +0.33 0.92 0.000000 0.000000 0.000000 +0.34 0.92 0.000000 0.000000 0.000000 +0.35 0.92 0.000000 0.000000 0.000000 +0.35 0.92 0.000000 0.000000 0.000000 +0.36 0.92 0.000000 0.000000 0.000000 +0.38 0.92 0.000000 0.000000 0.000000 +0.39 0.92 0.000000 0.000000 0.000000 +0.40 0.92 0.000000 0.000000 0.000000 +0.41 0.92 0.000000 0.000000 0.000000 +0.42 0.92 0.000000 0.000000 0.000000 +0.42 0.92 0.000000 0.000000 0.000000 +0.43 0.92 0.000000 0.000000 0.000000 +0.45 0.92 0.000000 0.000000 0.000000 +0.46 0.92 0.000000 0.000000 0.000000 +0.47 0.92 0.000000 0.000000 0.000000 +0.48 0.92 0.000000 0.000000 0.000000 +0.48 0.92 0.000000 0.000000 0.000000 +0.49 0.92 0.000000 0.000000 0.000000 +0.51 0.92 0.000000 0.000000 0.000000 +0.52 0.92 0.000000 0.000000 0.000000 +0.53 0.92 0.000000 0.000000 0.000000 +0.54 0.92 0.000000 0.000000 0.000000 +0.55 0.92 0.000000 0.000000 0.000000 +0.56 0.92 0.000000 0.000000 0.000000 +0.57 0.92 0.000000 0.000000 0.000000 +0.58 0.92 0.000000 0.000000 0.000000 +0.58 0.92 0.000000 0.000000 0.000000 +0.59 0.92 0.000000 0.000000 0.000000 +0.60 0.92 0.000000 0.000000 0.000000 +0.61 0.92 0.000000 0.000000 0.000000 +0.62 0.92 0.000000 0.000000 0.000000 +0.64 0.92 0.000000 0.000000 0.000000 +0.65 0.92 0.000000 0.000000 0.000000 +0.66 0.92 0.000000 0.000000 0.000000 +0.67 0.92 0.000000 0.000000 0.000000 +0.68 0.92 0.000000 0.000000 0.000000 +0.69 0.92 0.000000 0.000000 0.000000 +0.70 0.92 0.000000 0.000000 0.000000 +0.70 0.92 0.000000 0.000000 0.000000 +0.71 0.92 0.000000 0.000000 0.000000 +0.72 0.92 0.000000 0.000000 0.000000 +0.73 0.92 0.000000 0.000000 0.000000 +0.74 0.92 0.000000 0.000000 0.000000 +0.76 0.92 0.000000 0.000000 0.000000 +0.77 0.92 0.000000 0.000000 0.000000 +0.78 0.92 0.000000 0.000000 0.000000 +0.79 0.92 0.000000 0.000000 0.000000 +0.80 0.92 0.000000 0.000000 0.000000 +0.81 0.92 0.000000 0.000000 0.000000 +0.82 0.92 0.000000 0.000000 0.000000 +0.83 0.92 0.000000 0.000000 0.000000 +0.83 0.92 0.000000 0.000000 0.000000 +0.84 0.92 0.000000 0.000000 0.000000 +0.85 0.92 0.000000 0.000000 0.000000 +0.86 0.92 0.000000 0.000000 0.000000 +0.88 0.92 0.000000 0.000000 0.000000 +0.89 0.92 0.000000 0.000000 0.000000 +0.90 0.92 0.000000 0.000000 0.000000 +0.91 0.92 0.000000 0.000000 0.000000 +0.92 0.92 0.000000 0.000000 0.000000 +0.93 0.92 0.000000 0.000000 0.000000 +0.94 0.92 0.000000 0.000000 0.000000 +0.95 0.92 0.000000 0.000000 0.000000 +0.96 0.92 0.000000 0.000000 0.000000 +0.96 0.92 0.000000 0.000000 0.000000 +0.97 0.92 0.000000 0.000000 0.000000 +0.98 0.92 0.000000 0.000000 0.000000 +0.01 0.93 0.000000 0.000000 0.000000 +0.01 0.93 0.000000 0.000000 0.000000 +0.03 0.93 0.000000 0.000000 0.000000 +0.04 0.93 0.000000 0.000000 0.000000 +0.04 0.93 0.000000 0.000000 0.000000 +0.06 0.93 0.000000 0.000000 0.000000 +0.07 0.93 0.000000 0.000000 0.000000 +0.07 0.93 0.000000 0.000000 0.000000 +0.09 0.93 0.000000 0.000000 0.000000 +0.10 0.93 0.000000 0.000000 0.000000 +0.10 0.93 0.000000 0.000000 0.000000 +0.12 0.93 0.000000 0.000000 0.000000 +0.12 0.93 0.000000 0.000000 0.000000 +0.14 0.93 0.000000 0.000000 0.000000 +0.14 0.93 0.000000 0.000000 0.000000 +0.15 0.93 0.000000 0.000000 0.000000 +0.17 0.93 0.000000 0.000000 0.000000 +0.18 0.93 0.000000 0.000000 0.000000 +0.18 0.93 0.000000 0.000000 0.000000 +0.20 0.93 0.000000 0.000000 0.000000 +0.21 0.93 0.000000 0.000000 0.000000 +0.21 0.93 0.000000 0.000000 0.000000 +0.23 0.93 0.000000 0.000000 0.000000 +0.24 0.93 0.000000 0.000000 0.000000 +0.24 0.93 0.000000 0.000000 0.000000 +0.26 0.93 0.000000 0.000000 0.000000 +0.27 0.93 0.000000 0.000000 0.000000 +0.28 0.93 0.000000 0.000000 0.000000 +0.29 0.93 0.000000 0.000000 0.000000 +0.29 0.93 0.000000 0.000000 0.000000 +0.30 0.93 0.000000 0.000000 0.000000 +0.32 0.93 0.000000 0.000000 0.000000 +0.33 0.93 0.000000 0.000000 0.000000 +0.34 0.93 0.000000 0.000000 0.000000 +0.35 0.93 0.000000 0.000000 0.000000 +0.35 0.93 0.000000 0.000000 0.000000 +0.36 0.93 0.000000 0.000000 0.000000 +0.38 0.93 0.000000 0.000000 0.000000 +0.39 0.93 0.000000 0.000000 0.000000 +0.40 0.93 0.000000 0.000000 0.000000 +0.41 0.93 0.000000 0.000000 0.000000 +0.42 0.93 0.000000 0.000000 0.000000 +0.42 0.93 0.000000 0.000000 0.000000 +0.43 0.93 0.000000 0.000000 0.000000 +0.45 0.93 0.000000 0.000000 0.000000 +0.46 0.93 0.000000 0.000000 0.000000 +0.47 0.93 0.000000 0.000000 0.000000 +0.48 0.93 0.000000 0.000000 0.000000 +0.48 0.93 0.000000 0.000000 0.000000 +0.49 0.93 0.000000 0.000000 0.000000 +0.51 0.93 0.000000 0.000000 0.000000 +0.52 0.93 0.000000 0.000000 0.000000 +0.53 0.93 0.000000 0.000000 0.000000 +0.54 0.93 0.000000 0.000000 0.000000 +0.55 0.93 0.000000 0.000000 0.000000 +0.56 0.93 0.000000 0.000000 0.000000 +0.57 0.93 0.000000 0.000000 0.000000 +0.58 0.93 0.000000 0.000000 0.000000 +0.58 0.93 0.000000 0.000000 0.000000 +0.59 0.93 0.000000 0.000000 0.000000 +0.60 0.93 0.000000 0.000000 0.000000 +0.61 0.93 0.000000 0.000000 0.000000 +0.62 0.93 0.000000 0.000000 0.000000 +0.64 0.93 0.000000 0.000000 0.000000 +0.65 0.93 0.000000 0.000000 0.000000 +0.66 0.93 0.000000 0.000000 0.000000 +0.67 0.93 0.000000 0.000000 0.000000 +0.68 0.93 0.000000 0.000000 0.000000 +0.69 0.93 0.000000 0.000000 0.000000 +0.70 0.93 0.000000 0.000000 0.000000 +0.70 0.93 0.000000 0.000000 0.000000 +0.71 0.93 0.000000 0.000000 0.000000 +0.72 0.93 0.000000 0.000000 0.000000 +0.73 0.93 0.000000 0.000000 0.000000 +0.74 0.93 0.000000 0.000000 0.000000 +0.76 0.93 0.000000 0.000000 0.000000 +0.77 0.93 0.000000 0.000000 0.000000 +0.78 0.93 0.000000 0.000000 0.000000 +0.79 0.93 0.000000 0.000000 0.000000 +0.80 0.93 0.000000 0.000000 0.000000 +0.81 0.93 0.000000 0.000000 0.000000 +0.82 0.93 0.000000 0.000000 0.000000 +0.83 0.93 0.000000 0.000000 0.000000 +0.83 0.93 0.000000 0.000000 0.000000 +0.84 0.93 0.000000 0.000000 0.000000 +0.85 0.93 0.000000 0.000000 0.000000 +0.86 0.93 0.000000 0.000000 0.000000 +0.88 0.93 0.000000 0.000000 0.000000 +0.89 0.93 0.000000 0.000000 0.000000 +0.90 0.93 0.000000 0.000000 0.000000 +0.91 0.93 0.000000 0.000000 0.000000 +0.92 0.93 0.000000 0.000000 0.000000 +0.93 0.93 0.000000 0.000000 0.000000 +0.94 0.93 0.000000 0.000000 0.000000 +0.95 0.93 0.000000 0.000000 0.000000 +0.96 0.93 0.000000 0.000000 0.000000 +0.96 0.93 0.000000 0.000000 0.000000 +0.97 0.93 0.000000 0.000000 0.000000 +0.98 0.93 0.000000 0.000000 0.000000 +0.01 0.94 0.000000 0.000000 0.000000 +0.01 0.94 0.000000 0.000000 0.000000 +0.03 0.94 0.000000 0.000000 0.000000 +0.04 0.94 0.000000 0.000000 0.000000 +0.04 0.94 0.000000 0.000000 0.000000 +0.06 0.94 0.000000 0.000000 0.000000 +0.07 0.94 0.000000 0.000000 0.000000 +0.07 0.94 0.000000 0.000000 0.000000 +0.09 0.94 0.000000 0.000000 0.000000 +0.10 0.94 0.000000 0.000000 0.000000 +0.10 0.94 0.000000 0.000000 0.000000 +0.12 0.94 0.000000 0.000000 0.000000 +0.12 0.94 0.000000 0.000000 0.000000 +0.14 0.94 0.000000 0.000000 0.000000 +0.14 0.94 0.000000 0.000000 0.000000 +0.15 0.94 0.000000 0.000000 0.000000 +0.17 0.94 0.000000 0.000000 0.000000 +0.18 0.94 0.000000 0.000000 0.000000 +0.18 0.94 0.000000 0.000000 0.000000 +0.20 0.94 0.000000 0.000000 0.000000 +0.21 0.94 0.000000 0.000000 0.000000 +0.21 0.94 0.000000 0.000000 0.000000 +0.23 0.94 0.000000 0.000000 0.000000 +0.24 0.94 0.000000 0.000000 0.000000 +0.24 0.94 0.000000 0.000000 0.000000 +0.26 0.94 0.000000 0.000000 0.000000 +0.27 0.94 0.000000 0.000000 0.000000 +0.28 0.94 0.000000 0.000000 0.000000 +0.29 0.94 0.000000 0.000000 0.000000 +0.29 0.94 0.000000 0.000000 0.000000 +0.30 0.94 0.000000 0.000000 0.000000 +0.32 0.94 0.000000 0.000000 0.000000 +0.33 0.94 0.000000 0.000000 0.000000 +0.34 0.94 0.000000 0.000000 0.000000 +0.35 0.94 0.000000 0.000000 0.000000 +0.35 0.94 0.000000 0.000000 0.000000 +0.36 0.94 0.000000 0.000000 0.000000 +0.38 0.94 0.000000 0.000000 0.000000 +0.39 0.94 0.000000 0.000000 0.000000 +0.40 0.94 0.000000 0.000000 0.000000 +0.41 0.94 0.000000 0.000000 0.000000 +0.42 0.94 0.000000 0.000000 0.000000 +0.42 0.94 0.000000 0.000000 0.000000 +0.43 0.94 0.000000 0.000000 0.000000 +0.45 0.94 0.000000 0.000000 0.000000 +0.46 0.94 0.000000 0.000000 0.000000 +0.47 0.94 0.000000 0.000000 0.000000 +0.48 0.94 0.000000 0.000000 0.000000 +0.48 0.94 0.000000 0.000000 0.000000 +0.49 0.94 0.000000 0.000000 0.000000 +0.51 0.94 0.000000 0.000000 0.000000 +0.52 0.94 0.000000 0.000000 0.000000 +0.53 0.94 0.000000 0.000000 0.000000 +0.54 0.94 0.000000 0.000000 0.000000 +0.55 0.94 0.000000 0.000000 0.000000 +0.56 0.94 0.000000 0.000000 0.000000 +0.57 0.94 0.000000 0.000000 0.000000 +0.58 0.94 0.000000 0.000000 0.000000 +0.58 0.94 0.000000 0.000000 0.000000 +0.59 0.94 0.000000 0.000000 0.000000 +0.60 0.94 0.000000 0.000000 0.000000 +0.61 0.94 0.000000 0.000000 0.000000 +0.62 0.94 0.000000 0.000000 0.000000 +0.64 0.94 0.000000 0.000000 0.000000 +0.65 0.94 0.000000 0.000000 0.000000 +0.66 0.94 0.000000 0.000000 0.000000 +0.67 0.94 0.000000 0.000000 0.000000 +0.68 0.94 0.000000 0.000000 0.000000 +0.69 0.94 0.000000 0.000000 0.000000 +0.70 0.94 0.000000 0.000000 0.000000 +0.70 0.94 0.000000 0.000000 0.000000 +0.71 0.94 0.000000 0.000000 0.000000 +0.72 0.94 0.000000 0.000000 0.000000 +0.73 0.94 0.000000 0.000000 0.000000 +0.74 0.94 0.000000 0.000000 0.000000 +0.76 0.94 0.000000 0.000000 0.000000 +0.77 0.94 0.000000 0.000000 0.000000 +0.78 0.94 0.000000 0.000000 0.000000 +0.79 0.94 0.000000 0.000000 0.000000 +0.80 0.94 0.000000 0.000000 0.000000 +0.81 0.94 0.000000 0.000000 0.000000 +0.82 0.94 0.000000 0.000000 0.000000 +0.83 0.94 0.000000 0.000000 0.000000 +0.83 0.94 0.000000 0.000000 0.000000 +0.84 0.94 0.000000 0.000000 0.000000 +0.85 0.94 0.000000 0.000000 0.000000 +0.86 0.94 0.000000 0.000000 0.000000 +0.88 0.94 0.000000 0.000000 0.000000 +0.89 0.94 0.000000 0.000000 0.000000 +0.90 0.94 0.000000 0.000000 0.000000 +0.91 0.94 0.000000 0.000000 0.000000 +0.92 0.94 0.000000 0.000000 0.000000 +0.93 0.94 0.000000 0.000000 0.000000 +0.94 0.94 0.000000 0.000000 0.000000 +0.95 0.94 0.000000 0.000000 0.000000 +0.96 0.94 0.000000 0.000000 0.000000 +0.96 0.94 0.000000 0.000000 0.000000 +0.97 0.94 0.000000 0.000000 0.000000 +0.98 0.94 0.000000 0.000000 0.000000 +0.01 0.95 0.000000 0.000000 0.000000 +0.01 0.95 0.000000 0.000000 0.000000 +0.03 0.95 0.000000 0.000000 0.000000 +0.04 0.95 0.000000 0.000000 0.000000 +0.04 0.95 0.000000 0.000000 0.000000 +0.06 0.95 0.000000 0.000000 0.000000 +0.07 0.95 0.000000 0.000000 0.000000 +0.07 0.95 0.000000 0.000000 0.000000 +0.09 0.95 0.000000 0.000000 0.000000 +0.10 0.95 0.000000 0.000000 0.000000 +0.10 0.95 0.000000 0.000000 0.000000 +0.12 0.95 0.000000 0.000000 0.000000 +0.12 0.95 0.000000 0.000000 0.000000 +0.14 0.95 0.000000 0.000000 0.000000 +0.14 0.95 0.000000 0.000000 0.000000 +0.15 0.95 0.000000 0.000000 0.000000 +0.17 0.95 0.000000 0.000000 0.000000 +0.18 0.95 0.000000 0.000000 0.000000 +0.18 0.95 0.000000 0.000000 0.000000 +0.20 0.95 0.000000 0.000000 0.000000 +0.21 0.95 0.000000 0.000000 0.000000 +0.21 0.95 0.000000 0.000000 0.000000 +0.23 0.95 0.000000 0.000000 0.000000 +0.24 0.95 0.000000 0.000000 0.000000 +0.24 0.95 0.000000 0.000000 0.000000 +0.26 0.95 0.000000 0.000000 0.000000 +0.27 0.95 0.000000 0.000000 0.000000 +0.28 0.95 0.000000 0.000000 0.000000 +0.29 0.95 0.000000 0.000000 0.000000 +0.29 0.95 0.000000 0.000000 0.000000 +0.30 0.95 0.000000 0.000000 0.000000 +0.32 0.95 0.000000 0.000000 0.000000 +0.33 0.95 0.000000 0.000000 0.000000 +0.34 0.95 0.000000 0.000000 0.000000 +0.35 0.95 0.000000 0.000000 0.000000 +0.35 0.95 0.000000 0.000000 0.000000 +0.36 0.95 0.000000 0.000000 0.000000 +0.38 0.95 0.000000 0.000000 0.000000 +0.39 0.95 0.000000 0.000000 0.000000 +0.40 0.95 0.000000 0.000000 0.000000 +0.41 0.95 0.000000 0.000000 0.000000 +0.42 0.95 0.000000 0.000000 0.000000 +0.42 0.95 0.000000 0.000000 0.000000 +0.43 0.95 0.000000 0.000000 0.000000 +0.45 0.95 0.000000 0.000000 0.000000 +0.46 0.95 0.000000 0.000000 0.000000 +0.47 0.95 0.000000 0.000000 0.000000 +0.48 0.95 0.000000 0.000000 0.000000 +0.48 0.95 0.000000 0.000000 0.000000 +0.49 0.95 0.000000 0.000000 0.000000 +0.51 0.95 0.000000 0.000000 0.000000 +0.52 0.95 0.000000 0.000000 0.000000 +0.53 0.95 0.000000 0.000000 0.000000 +0.54 0.95 0.000000 0.000000 0.000000 +0.55 0.95 0.000000 0.000000 0.000000 +0.56 0.95 0.000000 0.000000 0.000000 +0.57 0.95 0.000000 0.000000 0.000000 +0.58 0.95 0.000000 0.000000 0.000000 +0.58 0.95 0.000000 0.000000 0.000000 +0.59 0.95 0.000000 0.000000 0.000000 +0.60 0.95 0.000000 0.000000 0.000000 +0.61 0.95 0.000000 0.000000 0.000000 +0.62 0.95 0.000000 0.000000 0.000000 +0.64 0.95 0.000000 0.000000 0.000000 +0.65 0.95 0.000000 0.000000 0.000000 +0.66 0.95 0.000000 0.000000 0.000000 +0.67 0.95 0.000000 0.000000 0.000000 +0.68 0.95 0.000000 0.000000 0.000000 +0.69 0.95 0.000000 0.000000 0.000000 +0.70 0.95 0.000000 0.000000 0.000000 +0.70 0.95 0.000000 0.000000 0.000000 +0.71 0.95 0.000000 0.000000 0.000000 +0.72 0.95 0.000000 0.000000 0.000000 +0.73 0.95 0.000000 0.000000 0.000000 +0.74 0.95 0.000000 0.000000 0.000000 +0.76 0.95 0.000000 0.000000 0.000000 +0.77 0.95 0.000000 0.000000 0.000000 +0.78 0.95 0.000000 0.000000 0.000000 +0.79 0.95 0.000000 0.000000 0.000000 +0.80 0.95 0.000000 0.000000 0.000000 +0.81 0.95 0.000000 0.000000 0.000000 +0.82 0.95 0.000000 0.000000 0.000000 +0.83 0.95 0.000000 0.000000 0.000000 +0.83 0.95 0.000000 0.000000 0.000000 +0.84 0.95 0.000000 0.000000 0.000000 +0.85 0.95 0.000000 0.000000 0.000000 +0.86 0.95 0.000000 0.000000 0.000000 +0.88 0.95 0.000000 0.000000 0.000000 +0.89 0.95 0.000000 0.000000 0.000000 +0.90 0.95 0.000000 0.000000 0.000000 +0.91 0.95 0.000000 0.000000 0.000000 +0.92 0.95 0.000000 0.000000 0.000000 +0.93 0.95 0.000000 0.000000 0.000000 +0.94 0.95 0.000000 0.000000 0.000000 +0.95 0.95 0.000000 0.000000 0.000000 +0.96 0.95 0.000000 0.000000 0.000000 +0.96 0.95 0.000000 0.000000 0.000000 +0.97 0.95 0.000000 0.000000 0.000000 +0.98 0.95 0.000000 0.000000 0.000000 +0.01 0.96 0.000000 0.000000 0.000000 +0.01 0.96 0.000000 0.000000 0.000000 +0.03 0.96 0.000000 0.000000 0.000000 +0.04 0.96 0.000000 0.000000 0.000000 +0.04 0.96 0.000000 0.000000 0.000000 +0.06 0.96 0.000000 0.000000 0.000000 +0.07 0.96 0.000000 0.000000 0.000000 +0.07 0.96 0.000000 0.000000 0.000000 +0.09 0.96 0.000000 0.000000 0.000000 +0.10 0.96 0.000000 0.000000 0.000000 +0.10 0.96 0.000000 0.000000 0.000000 +0.12 0.96 0.000000 0.000000 0.000000 +0.12 0.96 0.000000 0.000000 0.000000 +0.14 0.96 0.000000 0.000000 0.000000 +0.14 0.96 0.000000 0.000000 0.000000 +0.15 0.96 0.000000 0.000000 0.000000 +0.17 0.96 0.000000 0.000000 0.000000 +0.18 0.96 0.000000 0.000000 0.000000 +0.18 0.96 0.000000 0.000000 0.000000 +0.20 0.96 0.000000 0.000000 0.000000 +0.21 0.96 0.000000 0.000000 0.000000 +0.21 0.96 0.000000 0.000000 0.000000 +0.23 0.96 0.000000 0.000000 0.000000 +0.24 0.96 0.000000 0.000000 0.000000 +0.24 0.96 0.000000 0.000000 0.000000 +0.26 0.96 0.000000 0.000000 0.000000 +0.27 0.96 0.000000 0.000000 0.000000 +0.28 0.96 0.000000 0.000000 0.000000 +0.29 0.96 0.000000 0.000000 0.000000 +0.29 0.96 0.000000 0.000000 0.000000 +0.30 0.96 0.000000 0.000000 0.000000 +0.32 0.96 0.000000 0.000000 0.000000 +0.33 0.96 0.000000 0.000000 0.000000 +0.34 0.96 0.000000 0.000000 0.000000 +0.35 0.96 0.000000 0.000000 0.000000 +0.35 0.96 0.000000 0.000000 0.000000 +0.36 0.96 0.000000 0.000000 0.000000 +0.38 0.96 0.000000 0.000000 0.000000 +0.39 0.96 0.000000 0.000000 0.000000 +0.40 0.96 0.000000 0.000000 0.000000 +0.41 0.96 0.000000 0.000000 0.000000 +0.42 0.96 0.000000 0.000000 0.000000 +0.42 0.96 0.000000 0.000000 0.000000 +0.43 0.96 0.000000 0.000000 0.000000 +0.45 0.96 0.000000 0.000000 0.000000 +0.46 0.96 0.000000 0.000000 0.000000 +0.47 0.96 0.000000 0.000000 0.000000 +0.48 0.96 0.000000 0.000000 0.000000 +0.48 0.96 0.000000 0.000000 0.000000 +0.49 0.96 0.000000 0.000000 0.000000 +0.51 0.96 0.000000 0.000000 0.000000 +0.52 0.96 0.000000 0.000000 0.000000 +0.53 0.96 0.000000 0.000000 0.000000 +0.54 0.96 0.000000 0.000000 0.000000 +0.55 0.96 0.000000 0.000000 0.000000 +0.56 0.96 0.000000 0.000000 0.000000 +0.57 0.96 0.000000 0.000000 0.000000 +0.58 0.96 0.000000 0.000000 0.000000 +0.58 0.96 0.000000 0.000000 0.000000 +0.59 0.96 0.000000 0.000000 0.000000 +0.60 0.96 0.000000 0.000000 0.000000 +0.61 0.96 0.000000 0.000000 0.000000 +0.62 0.96 0.000000 0.000000 0.000000 +0.64 0.96 0.000000 0.000000 0.000000 +0.65 0.96 0.000000 0.000000 0.000000 +0.66 0.96 0.000000 0.000000 0.000000 +0.67 0.96 0.000000 0.000000 0.000000 +0.68 0.96 0.000000 0.000000 0.000000 +0.69 0.96 0.000000 0.000000 0.000000 +0.70 0.96 0.000000 0.000000 0.000000 +0.70 0.96 0.000000 0.000000 0.000000 +0.71 0.96 0.000000 0.000000 0.000000 +0.72 0.96 0.000000 0.000000 0.000000 +0.73 0.96 0.000000 0.000000 0.000000 +0.74 0.96 0.000000 0.000000 0.000000 +0.76 0.96 0.000000 0.000000 0.000000 +0.77 0.96 0.000000 0.000000 0.000000 +0.78 0.96 0.000000 0.000000 0.000000 +0.79 0.96 0.000000 0.000000 0.000000 +0.80 0.96 0.000000 0.000000 0.000000 +0.81 0.96 0.000000 0.000000 0.000000 +0.82 0.96 0.000000 0.000000 0.000000 +0.83 0.96 0.000000 0.000000 0.000000 +0.83 0.96 0.000000 0.000000 0.000000 +0.84 0.96 0.000000 0.000000 0.000000 +0.85 0.96 0.000000 0.000000 0.000000 +0.86 0.96 0.000000 0.000000 0.000000 +0.88 0.96 0.000000 0.000000 0.000000 +0.89 0.96 0.000000 0.000000 0.000000 +0.90 0.96 0.000000 0.000000 0.000000 +0.91 0.96 0.000000 0.000000 0.000000 +0.92 0.96 0.000000 0.000000 0.000000 +0.93 0.96 0.000000 0.000000 0.000000 +0.94 0.96 0.000000 0.000000 0.000000 +0.95 0.96 0.000000 0.000000 0.000000 +0.96 0.96 0.000000 0.000000 0.000000 +0.96 0.96 0.000000 0.000000 0.000000 +0.97 0.96 0.000000 0.000000 0.000000 +0.98 0.96 0.000000 0.000000 0.000000 +0.01 0.96 0.000000 0.000000 0.000000 +0.01 0.96 0.000000 0.000000 0.000000 +0.03 0.96 0.000000 0.000000 0.000000 +0.04 0.96 0.000000 0.000000 0.000000 +0.04 0.96 0.000000 0.000000 0.000000 +0.06 0.96 0.000000 0.000000 0.000000 +0.07 0.96 0.000000 0.000000 0.000000 +0.07 0.96 0.000000 0.000000 0.000000 +0.09 0.96 0.000000 0.000000 0.000000 +0.10 0.96 0.000000 0.000000 0.000000 +0.10 0.96 0.000000 0.000000 0.000000 +0.12 0.96 0.000000 0.000000 0.000000 +0.12 0.96 0.000000 0.000000 0.000000 +0.14 0.96 0.000000 0.000000 0.000000 +0.14 0.96 0.000000 0.000000 0.000000 +0.15 0.96 0.000000 0.000000 0.000000 +0.17 0.96 0.000000 0.000000 0.000000 +0.18 0.96 0.000000 0.000000 0.000000 +0.18 0.96 0.000000 0.000000 0.000000 +0.20 0.96 0.000000 0.000000 0.000000 +0.21 0.96 0.000000 0.000000 0.000000 +0.21 0.96 0.000000 0.000000 0.000000 +0.23 0.96 0.000000 0.000000 0.000000 +0.24 0.96 0.000000 0.000000 0.000000 +0.24 0.96 0.000000 0.000000 0.000000 +0.26 0.96 0.000000 0.000000 0.000000 +0.27 0.96 0.000000 0.000000 0.000000 +0.28 0.96 0.000000 0.000000 0.000000 +0.29 0.96 0.000000 0.000000 0.000000 +0.29 0.96 0.000000 0.000000 0.000000 +0.30 0.96 0.000000 0.000000 0.000000 +0.32 0.96 0.000000 0.000000 0.000000 +0.33 0.96 0.000000 0.000000 0.000000 +0.34 0.96 0.000000 0.000000 0.000000 +0.35 0.96 0.000000 0.000000 0.000000 +0.35 0.96 0.000000 0.000000 0.000000 +0.36 0.96 0.000000 0.000000 0.000000 +0.38 0.96 0.000000 0.000000 0.000000 +0.39 0.96 0.000000 0.000000 0.000000 +0.40 0.96 0.000000 0.000000 0.000000 +0.41 0.96 0.000000 0.000000 0.000000 +0.42 0.96 0.000000 0.000000 0.000000 +0.42 0.96 0.000000 0.000000 0.000000 +0.43 0.96 0.000000 0.000000 0.000000 +0.45 0.96 0.000000 0.000000 0.000000 +0.46 0.96 0.000000 0.000000 0.000000 +0.47 0.96 0.000000 0.000000 0.000000 +0.48 0.96 0.000000 0.000000 0.000000 +0.48 0.96 0.000000 0.000000 0.000000 +0.49 0.96 0.000000 0.000000 0.000000 +0.51 0.96 0.000000 0.000000 0.000000 +0.52 0.96 0.000000 0.000000 0.000000 +0.53 0.96 0.000000 0.000000 0.000000 +0.54 0.96 0.000000 0.000000 0.000000 +0.55 0.96 0.000000 0.000000 0.000000 +0.56 0.96 0.000000 0.000000 0.000000 +0.57 0.96 0.000000 0.000000 0.000000 +0.58 0.96 0.000000 0.000000 0.000000 +0.58 0.96 0.000000 0.000000 0.000000 +0.59 0.96 0.000000 0.000000 0.000000 +0.60 0.96 0.000000 0.000000 0.000000 +0.61 0.96 0.000000 0.000000 0.000000 +0.62 0.96 0.000000 0.000000 0.000000 +0.64 0.96 0.000000 0.000000 0.000000 +0.65 0.96 0.000000 0.000000 0.000000 +0.66 0.96 0.000000 0.000000 0.000000 +0.67 0.96 0.000000 0.000000 0.000000 +0.68 0.96 0.000000 0.000000 0.000000 +0.69 0.96 0.000000 0.000000 0.000000 +0.70 0.96 0.000000 0.000000 0.000000 +0.70 0.96 0.000000 0.000000 0.000000 +0.71 0.96 0.000000 0.000000 0.000000 +0.72 0.96 0.000000 0.000000 0.000000 +0.73 0.96 0.000000 0.000000 0.000000 +0.74 0.96 0.000000 0.000000 0.000000 +0.76 0.96 0.000000 0.000000 0.000000 +0.77 0.96 0.000000 0.000000 0.000000 +0.78 0.96 0.000000 0.000000 0.000000 +0.79 0.96 0.000000 0.000000 0.000000 +0.80 0.96 0.000000 0.000000 0.000000 +0.81 0.96 0.000000 0.000000 0.000000 +0.82 0.96 0.000000 0.000000 0.000000 +0.83 0.96 0.000000 0.000000 0.000000 +0.83 0.96 0.000000 0.000000 0.000000 +0.84 0.96 0.000000 0.000000 0.000000 +0.85 0.96 0.000000 0.000000 0.000000 +0.86 0.96 0.000000 0.000000 0.000000 +0.88 0.96 0.000000 0.000000 0.000000 +0.89 0.96 0.000000 0.000000 0.000000 +0.90 0.96 0.000000 0.000000 0.000000 +0.91 0.96 0.000000 0.000000 0.000000 +0.92 0.96 0.000000 0.000000 0.000000 +0.93 0.96 0.000000 0.000000 0.000000 +0.94 0.96 0.000000 0.000000 0.000000 +0.95 0.96 0.000000 0.000000 0.000000 +0.96 0.96 0.000000 0.000000 0.000000 +0.96 0.96 0.000000 0.000000 0.000000 +0.97 0.96 0.000000 0.000000 0.000000 +0.98 0.96 0.000000 0.000000 0.000000 +0.01 0.97 0.000000 0.000000 0.000000 +0.01 0.97 0.000000 0.000000 0.000000 +0.03 0.97 0.000000 0.000000 0.000000 +0.04 0.97 0.000000 0.000000 0.000000 +0.04 0.97 0.000000 0.000000 0.000000 +0.06 0.97 0.000000 0.000000 0.000000 +0.07 0.97 0.000000 0.000000 0.000000 +0.07 0.97 0.000000 0.000000 0.000000 +0.09 0.97 0.000000 0.000000 0.000000 +0.10 0.97 0.000000 0.000000 0.000000 +0.10 0.97 0.000000 0.000000 0.000000 +0.12 0.97 0.000000 0.000000 0.000000 +0.12 0.97 0.000000 0.000000 0.000000 +0.14 0.97 0.000000 0.000000 0.000000 +0.14 0.97 0.000000 0.000000 0.000000 +0.15 0.97 0.000000 0.000000 0.000000 +0.17 0.97 0.000000 0.000000 0.000000 +0.18 0.97 0.000000 0.000000 0.000000 +0.18 0.97 0.000000 0.000000 0.000000 +0.20 0.97 0.000000 0.000000 0.000000 +0.21 0.97 0.000000 0.000000 0.000000 +0.21 0.97 0.000000 0.000000 0.000000 +0.23 0.97 0.000000 0.000000 0.000000 +0.24 0.97 0.000000 0.000000 0.000000 +0.24 0.97 0.000000 0.000000 0.000000 +0.26 0.97 0.000000 0.000000 0.000000 +0.27 0.97 0.000000 0.000000 0.000000 +0.28 0.97 0.000000 0.000000 0.000000 +0.29 0.97 0.000000 0.000000 0.000000 +0.29 0.97 0.000000 0.000000 0.000000 +0.30 0.97 0.000000 0.000000 0.000000 +0.32 0.97 0.000000 0.000000 0.000000 +0.33 0.97 0.000000 0.000000 0.000000 +0.34 0.97 0.000000 0.000000 0.000000 +0.35 0.97 0.000000 0.000000 0.000000 +0.35 0.97 0.000000 0.000000 0.000000 +0.36 0.97 0.000000 0.000000 0.000000 +0.38 0.97 0.000000 0.000000 0.000000 +0.39 0.97 0.000000 0.000000 0.000000 +0.40 0.97 0.000000 0.000000 0.000000 +0.41 0.97 0.000000 0.000000 0.000000 +0.42 0.97 0.000000 0.000000 0.000000 +0.42 0.97 0.000000 0.000000 0.000000 +0.43 0.97 0.000000 0.000000 0.000000 +0.45 0.97 0.000000 0.000000 0.000000 +0.46 0.97 0.000000 0.000000 0.000000 +0.47 0.97 0.000000 0.000000 0.000000 +0.48 0.97 0.000000 0.000000 0.000000 +0.48 0.97 0.000000 0.000000 0.000000 +0.49 0.97 0.000000 0.000000 0.000000 +0.51 0.97 0.000000 0.000000 0.000000 +0.52 0.97 0.000000 0.000000 0.000000 +0.53 0.97 0.000000 0.000000 0.000000 +0.54 0.97 0.000000 0.000000 0.000000 +0.55 0.97 0.000000 0.000000 0.000000 +0.56 0.97 0.000000 0.000000 0.000000 +0.57 0.97 0.000000 0.000000 0.000000 +0.58 0.97 0.000000 0.000000 0.000000 +0.58 0.97 0.000000 0.000000 0.000000 +0.59 0.97 0.000000 0.000000 0.000000 +0.60 0.97 0.000000 0.000000 0.000000 +0.61 0.97 0.000000 0.000000 0.000000 +0.62 0.97 0.000000 0.000000 0.000000 +0.64 0.97 0.000000 0.000000 0.000000 +0.65 0.97 0.000000 0.000000 0.000000 +0.66 0.97 0.000000 0.000000 0.000000 +0.67 0.97 0.000000 0.000000 0.000000 +0.68 0.97 0.000000 0.000000 0.000000 +0.69 0.97 0.000000 0.000000 0.000000 +0.70 0.97 0.000000 0.000000 0.000000 +0.70 0.97 0.000000 0.000000 0.000000 +0.71 0.97 0.000000 0.000000 0.000000 +0.72 0.97 0.000000 0.000000 0.000000 +0.73 0.97 0.000000 0.000000 0.000000 +0.74 0.97 0.000000 0.000000 0.000000 +0.76 0.97 0.000000 0.000000 0.000000 +0.77 0.97 0.000000 0.000000 0.000000 +0.78 0.97 0.000000 0.000000 0.000000 +0.79 0.97 0.000000 0.000000 0.000000 +0.80 0.97 0.000000 0.000000 0.000000 +0.81 0.97 0.000000 0.000000 0.000000 +0.82 0.97 0.000000 0.000000 0.000000 +0.83 0.97 0.000000 0.000000 0.000000 +0.83 0.97 0.000000 0.000000 0.000000 +0.84 0.97 0.000000 0.000000 0.000000 +0.85 0.97 0.000000 0.000000 0.000000 +0.86 0.97 0.000000 0.000000 0.000000 +0.88 0.97 0.000000 0.000000 0.000000 +0.89 0.97 0.000000 0.000000 0.000000 +0.90 0.97 0.000000 0.000000 0.000000 +0.91 0.97 0.000000 0.000000 0.000000 +0.92 0.97 0.000000 0.000000 0.000000 +0.93 0.97 0.000000 0.000000 0.000000 +0.94 0.97 0.000000 0.000000 0.000000 +0.95 0.97 0.000000 0.000000 0.000000 +0.96 0.97 0.000000 0.000000 0.000000 +0.96 0.97 0.000000 0.000000 0.000000 +0.97 0.97 0.000000 0.000000 0.000000 +0.98 0.97 0.000000 0.000000 0.000000 +0.01 0.98 0.000000 0.000000 0.000000 +0.01 0.98 0.000000 0.000000 0.000000 +0.03 0.98 0.000000 0.000000 0.000000 +0.04 0.98 0.000000 0.000000 0.000000 +0.04 0.98 0.000000 0.000000 0.000000 +0.06 0.98 0.000000 0.000000 0.000000 +0.07 0.98 0.000000 0.000000 0.000000 +0.07 0.98 0.000000 0.000000 0.000000 +0.09 0.98 0.000000 0.000000 0.000000 +0.10 0.98 0.000000 0.000000 0.000000 +0.10 0.98 0.000000 0.000000 0.000000 +0.12 0.98 0.000000 0.000000 0.000000 +0.12 0.98 0.000000 0.000000 0.000000 +0.14 0.98 0.000000 0.000000 0.000000 +0.14 0.98 0.000000 0.000000 0.000000 +0.15 0.98 0.000000 0.000000 0.000000 +0.17 0.98 0.000000 0.000000 0.000000 +0.18 0.98 0.000000 0.000000 0.000000 +0.18 0.98 0.000000 0.000000 0.000000 +0.20 0.98 0.000000 0.000000 0.000000 +0.21 0.98 0.000000 0.000000 0.000000 +0.21 0.98 0.000000 0.000000 0.000000 +0.23 0.98 0.000000 0.000000 0.000000 +0.24 0.98 0.000000 0.000000 0.000000 +0.24 0.98 0.000000 0.000000 0.000000 +0.26 0.98 0.000000 0.000000 0.000000 +0.27 0.98 0.000000 0.000000 0.000000 +0.28 0.98 0.000000 0.000000 0.000000 +0.29 0.98 0.000000 0.000000 0.000000 +0.29 0.98 0.000000 0.000000 0.000000 +0.30 0.98 0.000000 0.000000 0.000000 +0.32 0.98 0.000000 0.000000 0.000000 +0.33 0.98 0.000000 0.000000 0.000000 +0.34 0.98 0.000000 0.000000 0.000000 +0.35 0.98 0.000000 0.000000 0.000000 +0.35 0.98 0.000000 0.000000 0.000000 +0.36 0.98 0.000000 0.000000 0.000000 +0.38 0.98 0.000000 0.000000 0.000000 +0.39 0.98 0.000000 0.000000 0.000000 +0.40 0.98 0.000000 0.000000 0.000000 +0.41 0.98 0.000000 0.000000 0.000000 +0.42 0.98 0.000000 0.000000 0.000000 +0.42 0.98 0.000000 0.000000 0.000000 +0.43 0.98 0.000000 0.000000 0.000000 +0.45 0.98 0.000000 0.000000 0.000000 +0.46 0.98 0.000000 0.000000 0.000000 +0.47 0.98 0.000000 0.000000 0.000000 +0.48 0.98 0.000000 0.000000 0.000000 +0.48 0.98 0.000000 0.000000 0.000000 +0.49 0.98 0.000000 0.000000 0.000000 +0.51 0.98 0.000000 0.000000 0.000000 +0.52 0.98 0.000000 0.000000 0.000000 +0.53 0.98 0.000000 0.000000 0.000000 +0.54 0.98 0.000000 0.000000 0.000000 +0.55 0.98 0.000000 0.000000 0.000000 +0.56 0.98 0.000000 0.000000 0.000000 +0.57 0.98 0.000000 0.000000 0.000000 +0.58 0.98 0.000000 0.000000 0.000000 +0.58 0.98 0.000000 0.000000 0.000000 +0.59 0.98 0.000000 0.000000 0.000000 +0.60 0.98 0.000000 0.000000 0.000000 +0.61 0.98 0.000000 0.000000 0.000000 +0.62 0.98 0.000000 0.000000 0.000000 +0.64 0.98 0.000000 0.000000 0.000000 +0.65 0.98 0.000000 0.000000 0.000000 +0.66 0.98 0.000000 0.000000 0.000000 +0.67 0.98 0.000000 0.000000 0.000000 +0.68 0.98 0.000000 0.000000 0.000000 +0.69 0.98 0.000000 0.000000 0.000000 +0.70 0.98 0.000000 0.000000 0.000000 +0.70 0.98 0.000000 0.000000 0.000000 +0.71 0.98 0.000000 0.000000 0.000000 +0.72 0.98 0.000000 0.000000 0.000000 +0.73 0.98 0.000000 0.000000 0.000000 +0.74 0.98 0.000000 0.000000 0.000000 +0.76 0.98 0.000000 0.000000 0.000000 +0.77 0.98 0.000000 0.000000 0.000000 +0.78 0.98 0.000000 0.000000 0.000000 +0.79 0.98 0.000000 0.000000 0.000000 +0.80 0.98 0.000000 0.000000 0.000000 +0.81 0.98 0.000000 0.000000 0.000000 +0.82 0.98 0.000000 0.000000 0.000000 +0.83 0.98 0.000000 0.000000 0.000000 +0.83 0.98 0.000000 0.000000 0.000000 +0.84 0.98 0.000000 0.000000 0.000000 +0.85 0.98 0.000000 0.000000 0.000000 +0.86 0.98 0.000000 0.000000 0.000000 +0.88 0.98 0.000000 0.000000 0.000000 +0.89 0.98 0.000000 0.000000 0.000000 +0.90 0.98 0.000000 0.000000 0.000000 +0.91 0.98 0.000000 0.000000 0.000000 +0.92 0.98 0.000000 0.000000 0.000000 +0.93 0.98 0.000000 0.000000 0.000000 +0.94 0.98 0.000000 0.000000 0.000000 +0.95 0.98 0.000000 0.000000 0.000000 +0.96 0.98 0.000000 0.000000 0.000000 +0.96 0.98 0.000000 0.000000 0.000000 +0.97 0.98 0.000000 0.000000 0.000000 +0.98 0.98 0.000000 0.000000 0.000000 diff --git a/BasicSolver/2D-mpi/velocity.png b/BasicSolver/2D-mpi/velocity.png new file mode 100644 index 0000000000000000000000000000000000000000..a8c9f05c0921deb610bbfa718d22e88f394d794f GIT binary patch literal 9306 zcma)C3tW@}8dKbzk>&-_P^<X%002P@ozB0kOEo z4Ky}_&qD=JjWwXS)leYYgOn5i+$aE|KxF92wbiTPs&3C7!u8C}&F$>$JUu;+963TJ zle4n2Zr!?7Q&ZF2+}zjK2X`UexG`OvUi7C>>MeZ+L1KO)RCc#SEQWXh?hPD}5dtlbAd_;%cXK%^QF5pxEk=15P@k)59NDrSx%Ztetr?j&KZDq*ggG!LT8gZ@1a`d(k1q_0ZW zpZH<^_>c4DDF$O_3`SB7#A%3^=?LM^hR@F#^3N~$^McXi45Q|(h4mK~)?_cL$Tj}s zlJRvq^2%jY_OFZ2U0M9o)g_7fOYsFJk-seq`Tc`{>!y2eEdN)L+2_UP4katr-LzQw zAGFymOXO`U{ZeaC_90-b1mz!rici3uRe<>^s9FuG*8tW!P_rJ~wFT@ApwgW{@~9* z@bn<)_!e{?0bNG{{}|{F2i=jN=M;Dr0|a>RoCtc!;IBl`mjr}A08t7MrGXbegZ}g2 zWfmC70fTv9=n4=Q0P%GoDF)I%z;GEDxdUWXVDv5+tpj5Zz^fJ@e+(4uK-mG5Jz%^S zOuPV-gFrP5RAWFr4l@fZmDzT|Ha3j>>cA-gjPGgw^v|(2nJbZXh7>8gct&Eu(^Dd`YGN=!%%o>+ zy(Q-^g*$!wV=+h@db}N2^GKN)u51zK3TlotSQrXl)QUX0pHcxaJ!*L}z8^GgeWS0i92w8d>vPACMxJe8H0OAf4|#6 zDvzc~`<`XXGd3r*SKHIV`gU{n;a6*pMIG)6C>~#qy2S5V=`UPP7ws;xxc+pt7yQWL z`gv3tiVc@5eSj;H?KN(DaM{-|*a&mu%Wj84*I~6CB6wTlLhJe%CSh2G3a5FgeYTroh}9jdx}na>i5pEl zWsIKGTE{BZVORf+J56nPq&sTwn?b`J@{*6K%QOvEB8>&prE?15(oe<6eggSX|IhqB z05Qh=G!UkL4R+6}kecn)0WT)4} z5OKT!XZwn4yK|yIbOzny1jQCK#TIzP78sbzYlnSlLKjwc?Xvoz$arM0fG!jj=5|B3 zWZz$Hfa)eEQKJ6u2kU#GS0;tZzFcN*cxJ0a%}-#;u-*P6@plJG5)-PgnzJPdk`$lC z?4Q!j+u{2jJhLE211 zSgN7tiNWA0-LWtB$Vq-BF12pTTBgmw1M z=Y*b#Z;U$K736rs&%S~YfAHDY4ZQEBz)j)%#g?rt_{N7a zG&7VC_C$uRfqJdWat|Hf)6H;=4TFB1vW} zA-(juyNix*X<3yZXLOin?q!P7vRl<$m>UdrZ2ig>xK!Y);iK4ENLn`Bv+ifK5-vZnCNw5=C=EZrub zZ)Ml=8##Uvfs*FjlJO*zWl0n>BsL*U+Jx!q8hAGGfDd`!TxWBDS-X{uT)c5^3=CkCYw)rNRp+*y92VFNa$Q4DwW|i!dxI2>ity| z8?9P$wv#}0@ClVBEpRPOmXhU5zDbA8o;jq{Cp3hdb>&p@`5|Y;bh`Z=ay6sWY?tAR zme$)(Ev%$2_NPc47E3Ew)q7%?2fK}F&Y2CSM~?^kDnDRzlndyj_7@Yzmu#L|wnQuF zgY@-5Uc>4GS1M)R!V>7}Xngx`sK5nTdQ|-HjY?YPkfb^_va8A^__Rx%q>m8aEBlVy zpR}leH^*{~{A8&^c5ifQ9YI-f)Cc2Ibxv+WkDE9_5Xn4Sk%X3#jN|zt8Oe#-V7l4Z z6q`}>HG{W!f#6!Y8KA2^`!!m26B;?+?v7aFegup{kG`AWrY%vWGfAi^VnoJoOYmv% z@jlbpEg4%mWeeboQ?RC~2&gwdIXoY@?_aucF2LM?y^w-xr#6E;dn$A*BRvs5bweD3 z8@8s}W9$HmeG%>z5$xHx@O8~1%%oKn)#Ri1HS*7zM;8gml8>)izqgU8?kVjkH6ze{ z!IHN}^3O)5yT6?*Wxbj*<69&7#~j>_As`PMjQ-4Y{=1RsIlw^B2~$4#zkryo`OusV zJ@NSJbL~K+=?cV)H$beA#aJYV%3|(D#zb8kYXy1#hYEapT<#?ck6RHu)*}#fg+cOeYNZg&l&PxWAw3Sqb zabIVUzikAqX^HE-g}@gfK+D3**9y3s0-4I?cHpQrH8vng5=s~w`H@iaJ9u`P$T^{u z2O&z>u}j$zaVEg^7gYC{xNoVvoAiqrAb#zKfC6j=Y#L)6tmqb}#w`W5u3Lv%B3ey- ze4&NF+)!|yv&p#kG^x0G19hTbv63fjT2n-x56)6=e)v9aXb4uOonWwWBDAw~#yDjD ztj6)t(KxKZH^zCF>hBP2n|6Z1ETvD2&HrNQ-=213Rv^4|WGVTvb|7XvtyjCJ!S0HE zsUr}e?h&W4V$i|t&;0On^@u;GHS;7BHtoZ87qD}8`6?rd@}b(#t9|JU?prs#bITuS(+(J*po{zeUKo!`lfh0S>QL!BhPJDZs&j zWQHwdhIY10D>|&p&e|CdYZ_}7x0zu}(X4Fg(ba9f)XtU}UFKV z-KI}F5HkVFX6!A-zZ+yV0qT&BK$yPKKxsY6*|>KTE5J4-u=xU8WJPH48zp2m>aiZ54&JjlnkEDJT}K0ZY0p6Z_SRpu zlN1he4ceLhHxTW!0&zu0Agube0x@j|4M`)llN1heNbOAj8;F~;0%7om>C^6_*fufb zc!o>F3wk-=*-}^snds!uo&+C<>>+hrz4G~#M#mx2ksC6$+23+Ed<&)|p{k2m9_pg5KocY{k4Y>sZr88j9CtNfW7pU;ZE7}+D48q= zxBXnG1q~KHYEL%54GnkDh@l3Rtte(CZE^AY%MqL&wrBL&@Ko#dC-2b+6xx>#4(QWq zDC_Nh2r}6QsW_aK-LQ=ghnvR77=KmMDseR2`$>2}xVb!4;&{Ju17E31f}Qi*VN|hd z`wjlb?@@SGiFT-F6uwHkxnYIx;cDm1jKYhx^B#5{iH`XIIYc{WW)$A}ZF9e83Bsz< z5r`AB1JR))5ErxqF{AJf+LJrjdB}To1Y*S-0gzr&*|G@794pJ~aqt=}3Duoh5KNK_ z;g!IHmGF%pQGqLvv5+t}!U*HG3BxQkgHH`HZnxp&BRSCWS8E^4l)4A@Vvc}^?udwh zWGTFI&G?x3gBiGuV*5E@i18QeQyi=g%?InK()L_bcuAmZl*E1+aQUUAVx1{_07bfT z4oC%44^z2y*~krZDIc7`OpaB#eDQf8&hd!L=$* zx}2>#U4HPQO^FP^!D*#t6XE?U#^_ni6*P`}C+UwqU4 zC`&|LvNTD6JlR}X$iBU>xFgCAA_@efn|>i+i1?b*XfZYNMC)<=XN3vyYPczy<)MdF z3c@YTw>~ZYbfb9joO#EL1g(;5efKx6f;Wa8{zF7Wz}|DRnqF=UOZsso%F>Swc#JPetghw(ZB!JU&E zV%3{WQc#38F_5z}FS8rU9`9_6I2AUMC!aKD22dJm~2$LbCD*L#$!&A_H5>@{+C#R#Rp z@b%791^(#h;dc`KJ&1@)Yfqg-vm;_GfNLfyumZ2VcV9_*K2-_c79taEkh{~PCdOgj zN3ve?-lq6<2;g%=)?FQeKx)qvH9*w74Fu__CbcKy zpRNvZ+`EPrR`y`e7R|BrCzP00(R;O}ergY>?z^;!Ym@C3*V|SjOIuzSNk+ABZRK50 z+NL*zQjDzr`(2qn#)1}{S(*AW)I5zG-tfLM*Vj?8c^MN{Vq(d$-1>fnNIdfFzkBO)Se%4rM`Wa+h4y0{9vJA6O(u7y2%CwG9*gMh zz8q7_W*`~l3}KWK6-Wo?SukyLmui@O1z>3_B<+;5-hhb*A~& zdbZtKhVBc#@sh^rvN*Dc)jcc@IU1;dew4G0i|8L$W;)X@=GQ@C{G71<6_2iTvid#9 z7caRgGr|JM+*YjhN#C*-9_`{TsKKw6Cq23d9%6_$mNeQDd;wCQ}>P7 z;)v3wpWphBB=F-BXcrm1%*NorE{ia`!3>D^SN0yXIsxxLp1l(uLp|SdEwN`n-7{JW z&+$xltyL3ARct&qsdyMxU3)vxk3%~p_c2-Dry1MN-KvG_RMwNZICOZoDXE1!c6k>A z*+i4J@FKd`CwRn1{}$C{!IG5@ie7eygbn*8XXA#D4=>OgaHwdsqL;JwM*G=_m} zIO|FZ`o)707_SE0m7tTN-IaCp1$d~bY_w>i@ABq??O_>60pF{Q-P2m(`b0UT`hy{f zh4NcdEc!T{?;9H!qNMCo6P}c);VxWIdP+3w0AEpRi zso&n@nmUG{Ul%bPnv$h0@N~#$D-r@9^nYR0v$hLzE}dL8P^TG62;YXjzcyo3cz;!> z5GM4f=6?Zpr(oBQk2=Cn?gkan6nk?e_;voG&ekU*v4{xcv6CUk1OM98Dpu$Es`FpA z{ggUlyKk&Az6X885gKS;P<_>h=8H3kFfQeD8bT;*)yK02YB$H~T=$=8J+r=6bo z-Mm31kc>ZX)vIuM!koleIlvU@)gN;pb47Kmc*%`t`(%O}W7}`!Mp(Y){T(Zp=sWn# zRIL0&zfJF0hU6$r`P`R^m3J;cQTuuH>%Ct4{l@Ll4m967mUWO-*a_^Rq0_$wFK2i_ pEJ@qyaQhj5edD)NN6Hb%s&{|M_jPS|=4)>3*tYA7V)uiI{|_I`UIG9B literal 0 HcmV?d00001 diff --git a/BasicSolver/2D-seq-pt/Makefile b/BasicSolver/2D-seq-pt/Makefile new file mode 100644 index 0000000..57f99f4 --- /dev/null +++ b/BasicSolver/2D-seq-pt/Makefile @@ -0,0 +1,71 @@ +#======================================================================================= +# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(wildcard $(SRC_DIR)/*.c) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm format + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/BasicSolver/2D-seq-pt/README.md b/BasicSolver/2D-seq-pt/README.md new file mode 100644 index 0000000..d980b54 --- /dev/null +++ b/BasicSolver/2D-seq-pt/README.md @@ -0,0 +1,78 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about solver, affinity settings, allocation sizes and timer resolution. +For debugging you may want to set the VERBOSE option: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +` + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. + +## Usage + +You have to provide a parameter file describing the problem you want to solve: +``` +./exe-CLANG dcavity.par +``` + +Examples are given in in dcavity (a lid driven cavity test case) and canal (simulating a empty canal). + +You can plot the resulting velocity and pressure fields using gnuplot: +``` +gnuplot vector.plot +``` +and for the pressure: +``` +gnuplot surface.plot +``` diff --git a/BasicSolver/2D-seq-pt/animate.plot b/BasicSolver/2D-seq-pt/animate.plot new file mode 100644 index 0000000..6205e77 --- /dev/null +++ b/BasicSolver/2D-seq-pt/animate.plot @@ -0,0 +1,7 @@ +unset border; unset tics; unset key; +set term gif animate delay 50 +set output "trace.gif" +do for [ts=0:23] { + plot "particles_".ts.".dat" with points pointtype 7 +} +unset output diff --git a/BasicSolver/2D-seq-pt/canal.par b/BasicSolver/2D-seq-pt/canal.par new file mode 100644 index 0000000..2eb04b6 --- /dev/null +++ b/BasicSolver/2D-seq-pt/canal.par @@ -0,0 +1,59 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 100.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 50 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 60.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.00001 # stopping tolerance for pressure iteration +omg 1.8 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Visualization Data: +# ------------------ + +traceStart 10.0 # time for starting visualization +traceWrite 2.0 # time stepsize for saving particle data +traceInject 2.0 # time stepsize for particle injection +lineX1 2.0 # Coordinates of line segment for particle injection +lineY1 0.5 +lineX2 2.0 +lineY2 3.5 +nparticles 30 # number of particles to inject + +#=============================================================================== diff --git a/BasicSolver/2D-seq-pt/config.mk b/BasicSolver/2D-seq-pt/config.mk new file mode 100644 index 0000000..af5f1a0 --- /dev/null +++ b/BasicSolver/2D-seq-pt/config.mk @@ -0,0 +1,12 @@ +# Supported: GCC, CLANG, ICC +TAG ?= CLANG +ENABLE_OPENMP ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DDEBUG +#OPTIONS += -DBOUNDCHECK +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/2D-seq-pt/dcavity.par b/BasicSolver/2D-seq-pt/dcavity.par new file mode 100644 index 0000000..4241393 --- /dev/null +++ b/BasicSolver/2D-seq-pt/dcavity.par @@ -0,0 +1,46 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 1 # 2 = free-slip 4 = periodic +bcRight 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 10.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +imax 40 # number of interior cells in x-direction +jmax 40 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 10.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-seq-pt/include_CLANG.mk b/BasicSolver/2D-seq-pt/include_CLANG.mk new file mode 100644 index 0000000..0caff0c --- /dev/null +++ b/BasicSolver/2D-seq-pt/include_CLANG.mk @@ -0,0 +1,17 @@ +CC = clang +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +# CFLAGS = -O3 -std=c17 $(OPENMP) +CFLAGS = -O3 std=c17 +#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG +LFLAGS = $(OPENMP) -lm +DEFINES = -D_GNU_SOURCE# -DDEBUG +INCLUDES = diff --git a/BasicSolver/2D-seq-pt/include_GCC.mk b/BasicSolver/2D-seq-pt/include_GCC.mk new file mode 100644 index 0000000..427e798 --- /dev/null +++ b/BasicSolver/2D-seq-pt/include_GCC.mk @@ -0,0 +1,14 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/2D-seq-pt/include_ICC.mk b/BasicSolver/2D-seq-pt/include_ICC.mk new file mode 100644 index 0000000..94b8e20 --- /dev/null +++ b/BasicSolver/2D-seq-pt/include_ICC.mk @@ -0,0 +1,14 @@ +CC = icc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/2D-seq-pt/src/affinity.c b/BasicSolver/2D-seq-pt/src/affinity.c new file mode 100644 index 0000000..b501665 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/affinity.c @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifdef __linux__ +#ifdef _OPENMP +#include +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_THREADS 128 +#define gettid() syscall(SYS_gettid) + +static int getProcessorID(cpu_set_t* cpu_set) +{ + int processorId; + + for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { + if (CPU_ISSET(processorId, cpu_set)) { + break; + } + } + return processorId; +} + +int affinity_getProcessorId() +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); + + return getProcessorID(&cpu_set); +} + +void affinity_pinThread(int processorId) +{ + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); +} + +void affinity_pinProcess(int processorId) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} +#endif /*_OPENMP*/ +#endif /*__linux__*/ diff --git a/BasicSolver/2D-seq-pt/src/affinity.h b/BasicSolver/2D-seq-pt/src/affinity.h new file mode 100644 index 0000000..d844fe5 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/affinity.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef AFFINITY_H +#define AFFINITY_H + +extern int affinity_getProcessorId(); +extern void affinity_pinProcess(int); +extern void affinity_pinThread(int); + +#endif /*AFFINITY_H*/ diff --git a/BasicSolver/2D-seq-pt/src/allocate.c b/BasicSolver/2D-seq-pt/src/allocate.c new file mode 100644 index 0000000..81e1e9d --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/allocate.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +void* allocate(int alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/BasicSolver/2D-seq-pt/src/allocate.h b/BasicSolver/2D-seq-pt/src/allocate.h new file mode 100644 index 0000000..54cfe06 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(int alignment, size_t bytesize); + +#endif diff --git a/BasicSolver/2D-seq-pt/src/grid.h b/BasicSolver/2D-seq-pt/src/grid.h new file mode 100644 index 0000000..c963429 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/grid.h @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __GRID_H_ +#define __GRID_H_ + +typedef struct { + double dx, dy, dz; + int imax, jmax, kmax; + double xlength, ylength, zlength; +} Grid; + +#endif // __GRID_H_ diff --git a/BasicSolver/2D-seq-pt/src/likwid-marker.h b/BasicSolver/2D-seq-pt/src/likwid-marker.h new file mode 100644 index 0000000..eb7cc78 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/BasicSolver/2D-seq-pt/src/main.c b/BasicSolver/2D-seq-pt/src/main.c new file mode 100644 index 0000000..a67fe84 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/main.c @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "parameter.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" +#include "trace.h" + +int main(int argc, char** argv) +{ + double timeStart, timeEnd; + Parameter p; + Solver s; + Tracing t; + initParameter(&p); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(&p, argv[1]); + printParameter(&p); + initSolver(&s, &p); + initTrace(&t, &p); +#ifndef VERBOSE + initProgress(s.te); +#endif + + double tau = s.tau; + double te = s.te; + double time = 0.0; + int nt = 0; + + timeStart = getTimeStamp(); + while (time <= te) { + if (tau > 0.0) computeTimestep(&s); + setBoundaryConditions(&s); + setSpecialBoundaryCondition(&s); + computeFG(&s); + computeRHS(&s); + if (nt % 100 == 0) normalizePressure(&s); + solve(&s); + adaptUV(&s); + time += s.dt; + nt++; + + trace(&t, s.u, s.v, time); + +#ifdef VERBOSE + printf("TIME %f , TIMESTEP %f\n", time, s.dt); +#else + printProgress(time); +#endif + } + timeEnd = getTimeStamp(); + stopProgress(); + printf("Solution took %.2fs\n", timeEnd - timeStart); + writeResult(&s); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/2D-seq-pt/src/parameter.c b/BasicSolver/2D-seq-pt/src/parameter.c new file mode 100644 index 0000000..f76e032 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/parameter.c @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_STRING(name); + PARSE_INT(bcLeft); + PARSE_INT(bcRight); + PARSE_INT(bcBottom); + PARSE_INT(bcTop); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(p_init); + PARSE_REAL(traceStart); + PARSE_REAL(traceWrite); + PARSE_REAL(traceInject); + PARSE_REAL(lineX1); + PARSE_REAL(lineX2); + PARSE_REAL(lineY1); + PARSE_REAL(lineY2); + PARSE_INT(nparticles); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + param->bcLeft, + param->bcRight, + param->bcBottom, + param->bcTop); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength); + printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); + printf("Particle tracing:\n"); + printf("\tTrace start: %f\n", param->traceStart); + printf("\tTrace write: %f\n", param->traceWrite); + printf("\tTrace inject: %f\n", param->traceInject); + printf("\tNumber of particles: %d\n", param->nparticles); + printf("\tLine (x1, y1, x2, y2): %f,%f / %f,%f\n", + param->lineX1, + param->lineY1, + param->lineX2, + param->lineY2); +} diff --git a/BasicSolver/2D-seq-pt/src/parameter.h b/BasicSolver/2D-seq-pt/src/parameter.h new file mode 100644 index 0000000..d3ee2e9 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/parameter.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + double xlength, ylength; + int imax, jmax; + int itermax; + double eps, omg; + double re, tau, gamma; + double te, dt; + double gx, gy; + char* name; + int bcLeft, bcRight, bcBottom, bcTop; + double u_init, v_init, p_init; + double traceStart, traceWrite, traceInject; + double lineX1, lineX2, lineY1, lineY2; + int nparticles; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/BasicSolver/2D-seq-pt/src/progress.c b/BasicSolver/2D-seq-pt/src/progress.c new file mode 100644 index 0000000..a9b82bd --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/progress.c @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; + +void initProgress(double end) +{ + _end = end; + _current = 0; + + printf("[ ]"); + fflush(stdout); +} + +void printProgress(double current) +{ + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); +} + +void stopProgress() +{ + printf("\n"); + fflush(stdout); +} diff --git a/BasicSolver/2D-seq-pt/src/progress.h b/BasicSolver/2D-seq-pt/src/progress.h new file mode 100644 index 0000000..9ef2d96 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/progress.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ + +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(); + +#endif diff --git a/BasicSolver/2D-seq-pt/src/solver.c b/BasicSolver/2D-seq-pt/src/solver.c new file mode 100644 index 0000000..73333bc --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/solver.c @@ -0,0 +1,500 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +#define P(i, j) p[(j) * (imax + 2) + (i)] +#define F(i, j) f[(j) * (imax + 2) + (i)] +#define G(i, j) g[(j) * (imax + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imax + 2) + (i)] + +static void print(Solver* solver, double* grid) +{ + int imax = solver->imax; + + for (int j = 0; j < solver->jmax + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < solver->imax + 2; i++) { + printf("%12.8f ", grid[j * (imax + 2) + i]); + } + printf("\n"); + } + fflush(stdout); +} + +static void printConfig(Solver* solver) +{ + printf("Parameters for #%s#\n", solver->problem); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + solver->bcLeft, + solver->bcRight, + solver->bcBottom, + solver->bcTop); + printf("\tReynolds number: %.2f\n", solver->re); + printf("\tGx Gy: %.2f %.2f\n", solver->gx, solver->gy); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", solver->xlength, solver->ylength); + printf("\tCells (x, y): %d, %d\n", solver->imax, solver->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", solver->dt, solver->te); + printf("\tdt bound: %.6f\n", solver->dtBound); + printf("\tTau factor: %.2f\n", solver->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", solver->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", solver->eps); + printf("\tgamma factor: %f\n", solver->gamma); + printf("\tomega (SOR relaxation): %f\n", solver->omega); +} + +void initSolver(Solver* solver, Parameter* params) +{ + solver->problem = params->name; + solver->bcLeft = params->bcLeft; + solver->bcRight = params->bcRight; + solver->bcBottom = params->bcBottom; + solver->bcTop = params->bcTop; + solver->imax = params->imax; + solver->jmax = params->jmax; + solver->xlength = params->xlength; + solver->ylength = params->ylength; + solver->dx = params->xlength / params->imax; + solver->dy = params->ylength / params->jmax; + solver->eps = params->eps; + solver->omega = params->omg; + solver->itermax = params->itermax; + solver->re = params->re; + solver->gx = params->gx; + solver->gy = params->gy; + solver->dt = params->dt; + solver->te = params->te; + solver->tau = params->tau; + solver->gamma = params->gamma; + + int imax = solver->imax; + int jmax = solver->jmax; + size_t size = (imax + 2) * (jmax + 2) * sizeof(double); + solver->u = allocate(64, size); + solver->v = allocate(64, size); + solver->p = allocate(64, size); + solver->rhs = allocate(64, size); + solver->f = allocate(64, size); + solver->g = allocate(64, size); + + for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { + solver->u[i] = params->u_init; + solver->v[i] = params->v_init; + solver->p[i] = params->p_init; + solver->rhs[i] = 0.0; + solver->f[i] = 0.0; + solver->g[i] = 0.0; + } + + double dx = solver->dx; + double dy = solver->dy; + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy); + solver->dtBound = 0.5 * solver->re * 1.0 / invSqrSum; +#ifdef VERBOSE + printConfig(solver); +#endif +} + +void computeRHS(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double idx = 1.0 / solver->dx; + double idy = 1.0 / solver->dy; + double idt = 1.0 / solver->dt; + double* rhs = solver->rhs; + double* f = solver->f; + double* g = solver->g; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + RHS(i, j) = idt * + ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy); + } + } +} + +void solve(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double eps = solver->eps; + int itermax = solver->itermax; + double dx2 = solver->dx * solver->dx; + double dy2 = solver->dy * solver->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* p = solver->p; + double* rhs = solver->rhs; + double epssq = eps * eps; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + } + + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmax + 1) = P(i, jmax); + } + + for (int j = 1; j < jmax + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } + + res = res / (double)(imax * jmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif +} + +static double maxElement(Solver* solver, double* m) +{ + int size = (solver->imax + 2) * (solver->jmax + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + return maxval; +} + +void normalizePressure(Solver* solver) +{ + int size = (solver->imax + 2) * (solver->jmax + 2); + double* p = solver->p; + double avgP = 0.0; + + for (int i = 0; i < size; i++) { + avgP += p[i]; + } + avgP /= size; + + for (int i = 0; i < size; i++) { + p[i] = p[i] - avgP; + } +} + +void computeTimestep(Solver* solver) +{ + double dt = solver->dtBound; + double dx = solver->dx; + double dy = solver->dy; + double umax = maxElement(solver, solver->u); + double vmax = maxElement(solver, solver->v); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + + solver->dt = dt * solver->tau; +} + +void setBoundaryConditions(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double* u = solver->u; + double* v = solver->v; + + // Left boundary + switch (solver->bcLeft) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + U(0, j) = 0.0; + V(0, j) = -V(1, j); + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + U(0, j) = 0.0; + V(0, j) = V(1, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + U(0, j) = U(1, j); + V(0, j) = V(1, j); + } + break; + case PERIODIC: + break; + } + + // Right boundary + switch (solver->bcRight) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + U(imax, j) = 0.0; + V(imax + 1, j) = -V(imax, j); + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + U(imax, j) = 0.0; + V(imax + 1, j) = V(imax, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + U(imax, j) = U(imax - 1, j); + V(imax + 1, j) = V(imax, j); + } + break; + case PERIODIC: + break; + } + + // Bottom boundary + switch (solver->bcBottom) { + case NOSLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = -U(i, 1); + } + break; + case SLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = U(i, 1); + } + break; + case OUTFLOW: + for (int i = 1; i < imax + 1; i++) { + U(i, 0) = U(i, 1); + V(i, 0) = V(i, 1); + } + break; + case PERIODIC: + break; + } + + // Top boundary + switch (solver->bcTop) { + case NOSLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, jmax) = 0.0; + U(i, jmax + 1) = -U(i, jmax); + } + break; + case SLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, jmax) = 0.0; + U(i, jmax + 1) = U(i, jmax); + } + break; + case OUTFLOW: + for (int i = 1; i < imax + 1; i++) { + U(i, jmax + 1) = U(i, jmax); + V(i, jmax) = V(i, jmax - 1); + } + break; + case PERIODIC: + break; + } +} + +void setSpecialBoundaryCondition(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double mDy = solver->dy; + double* u = solver->u; + + if (strcmp(solver->problem, "dcavity") == 0) { + for (int i = 1; i < imax; i++) { + U(i, jmax + 1) = 2.0 - U(i, jmax); + } + } else if (strcmp(solver->problem, "canal") == 0) { + double ylength = solver->ylength; + double y; + + for (int j = 1; j < jmax + 1; j++) { + y = mDy * (j - 0.5); + U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } +} + +void computeFG(Solver* solver) +{ + double* u = solver->u; + double* v = solver->v; + double* f = solver->f; + double* g = solver->g; + int imax = solver->imax; + int jmax = solver->jmax; + double gx = solver->gx; + double gy = solver->gy; + double gamma = solver->gamma; + double dt = solver->dt; + double inverseRe = 1.0 / solver->re; + double inverseDx = 1.0 / solver->dx; + double inverseDy = 1.0 / solver->dy; + double du2dx, dv2dy, duvdx, duvdy; + double du2dx2, du2dy2, dv2dx2, dv2dy2; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + du2dx = inverseDx * 0.25 * + ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - + (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + + fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); + + duvdy = inverseDy * 0.25 * + ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - + (V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + + fabs(V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) - U(i, j - 1))); + + du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); + du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); + F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - + (U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + + fabs(U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) - V(i - 1, j))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - + (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + + fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); + + dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); + dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); + G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); + } + } + + /* ---------------------- boundary of F --------------------------- */ + for (int j = 1; j < jmax + 1; j++) { + F(0, j) = U(0, j); + F(imax, j) = U(imax, j); + } + + /* ---------------------- boundary of G --------------------------- */ + for (int i = 1; i < imax + 1; i++) { + G(i, 0) = V(i, 0); + G(i, jmax) = V(i, jmax); + } +} + +void adaptUV(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double* p = solver->p; + double* u = solver->u; + double* v = solver->v; + double* f = solver->f; + double* g = solver->g; + double factorX = solver->dt / solver->dx; + double factorY = solver->dt / solver->dy; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; + V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; + } + } +} + +void writeResult(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double dx = solver->dx; + double dy = solver->dy; + double* p = solver->p; + double* u = solver->u; + double* v = solver->v; + double x = 0.0, y = 0.0; + + FILE* fp; + fp = fopen("pressure.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax + 1; j++) { + y = (double)(j - 0.5) * dy; + for (int i = 1; i < imax + 1; i++) { + x = (double)(i - 0.5) * dx; + fprintf(fp, "%.2f %.2f %f\n", x, y, P(i, j)); + } + fprintf(fp, "\n"); + } + + fclose(fp); + + fp = fopen("velocity.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax + 1; j++) { + y = dy * (j - 0.5); + for (int i = 1; i < imax + 1; i++) { + x = dx * (i - 0.5); + double vel_u = (U(i, j) + U(i - 1, j)) / 2.0; + double vel_v = (V(i, j) + V(i, j - 1)) / 2.0; + double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); + } + } + + fclose(fp); +} diff --git a/BasicSolver/2D-seq-pt/src/solver.h b/BasicSolver/2D-seq-pt/src/solver.h new file mode 100644 index 0000000..42d5fee --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/solver.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "parameter.h" + +#define U(i, j) u[(j) * (imax + 2) + (i)] +#define V(i, j) v[(j) * (imax + 2) + (i)] + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + double dx, dy; + int imax, jmax; + double xlength, ylength; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop; +} Solver; + +void initSolver(Solver*, Parameter*); +void computeRHS(Solver*); +void solve(Solver*); +void normalizePressure(Solver*); +void computeTimestep(Solver*); +void setBoundaryConditions(Solver*); +void setSpecialBoundaryCondition(Solver*); +void computeFG(Solver*); +void adaptUV(Solver*); +void writeResult(Solver*); +#endif diff --git a/BasicSolver/2D-seq-pt/src/timing.c b/BasicSolver/2D-seq-pt/src/timing.c new file mode 100644 index 0000000..c4025a4 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/timing.c @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution() +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-seq-pt/src/timing.h b/BasicSolver/2D-seq-pt/src/timing.h new file mode 100644 index 0000000..db95329 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/timing.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(); +extern double getTimeResolution(); +extern double getTimeStamp_(); + +#endif // __TIMING_H_ diff --git a/BasicSolver/2D-seq-pt/src/trace.c b/BasicSolver/2D-seq-pt/src/trace.c new file mode 100644 index 0000000..6221eb9 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/trace.c @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "trace.h" +#define U(i, j) u[(j) * (imax + 2) + (i)] +#define V(i, j) v[(j) * (imax + 2) + (i)] + +static int ts = 0; + +static void printState(Tracing* t) +{ + printf("Cursor: %d Total particles: %d\n", t->cursor, t->totalParticles); +} + +static void advanceParticles( + Tracing* t, double delt, double* restrict u, double* restrict v) +{ + double delx = t->grid.dx; + double dely = t->grid.dy; + + double* m = t->memorypool; + int* p = t->particles; + int imax = t->grid.imax; + int jmax = t->grid.jmax; + + for (int i = 0; i < t->totalParticles; i++) { + int particleId = p[i]; + + double x = m[particleId * NCOORD + X]; + double y = m[particleId * NCOORD + Y]; + // printf("P%d - X %f Y %f\n", i, x, y); + + // Interpolate U + int iCoord = (int)(x / delx) + 1; + int jCoord = (int)((y + 0.5 * dely) / dely) + 1; + + double x1 = (double)(iCoord - 1) * delx; + double y1 = ((double)(jCoord - 1) - 0.5) * dely; + double x2 = (double)iCoord * delx; + double y2 = ((double)jCoord - 0.5) * dely; + + // printf("U - iCoord %d jCoord %d\n", iCoord, jCoord); + + double un = (1.0 / (delx * dely)) * + ((x2 - x) * (y2 - y) * U(iCoord - 1, jCoord - 1) + + (x - x1) * (y2 - y) * U(iCoord, jCoord - 1) + + (x2 - x) * (y - y1) * U(iCoord - 1, jCoord) + + (x - x1) * (y - y1) * U(iCoord, jCoord)); + + double xn = x + delt * un; + m[particleId * NCOORD + X] = xn; + + // Interpolate V + iCoord = (int)((x + 0.5 * delx) / delx) + 1; + jCoord = (int)(y / dely) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * delx; + y1 = (double)(jCoord - 1) * dely; + x2 = ((double)iCoord - 0.5) * delx; + y2 = (double)jCoord * dely; + + // printf("V - iCoord %d jCoord %d\n", iCoord, jCoord); + + double vn = (1.0 / (delx * dely)) * + ((x2 - x) * (y2 - y) * V(iCoord - 1, jCoord - 1) + + (x - x1) * (y2 - y) * V(iCoord, jCoord - 1) + + (x2 - x) * (y - y1) * V(iCoord - 1, jCoord) + + (x - x1) * (y - y1) * V(iCoord, jCoord)); + + double yn = y + delt * vn; + m[particleId * NCOORD + Y] = yn; + printf("P%i VEL %f %f dt %f OP %f %f NP %f %f\n", i, un, vn, delt, x, y, xn, yn); + } + + double xlength = t->grid.xlength; + double ylength = t->grid.ylength; + int cntNew = 0; + int tmp[t->totalParticles]; + + // Check for particles to remove + for (int i = 0; i < t->totalParticles; i++) { + int particleId = p[i]; + + double x = m[particleId * NCOORD + X]; + double y = m[particleId * NCOORD + Y]; + + if (!((x < 0.0) || (x > xlength) || (y < 0.0) || (y > ylength))) { + tmp[cntNew++] = i; + } + } + + t->totalParticles = cntNew; + memcpy(t->particles, tmp, cntNew * sizeof(int)); +} + +static void injectParticles(Tracing* t) +{ + double* line = t->line; + double* m = t->memorypool; + + for (int i = 0; i < t->numParticles; i++) { + printf("Inject %d as %d mem %d\n", i, t->totalParticles, t->cursor); + t->particles[t->totalParticles] = t->cursor; + m[(t->cursor) * NCOORD + X] = line[i * NCOORD + X]; + m[(t->cursor) * NCOORD + Y] = line[i * NCOORD + Y]; + t->cursor++; + t->totalParticles++; + } +} + +static void writeParticles(Tracing* t) +{ + FILE* fp; + double* m = t->memorypool; + int* p = t->particles; + + char filename[50]; + snprintf(filename, 50, "particles_%d.dat", ts++); + fp = fopen(filename, "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int i = 0; i < t->totalParticles; i++) { + int particleId = p[i]; + + double x = m[particleId * NCOORD + X]; + double y = m[particleId * NCOORD + Y]; + fprintf(fp, "%f %f\n", x, y); + } + fclose(fp); +} + +void trace(Tracing* t, double* restrict u, double* restrict v, double time) +{ + if (time >= t->traceStart) { + if ((time - t->lastUpdate[INJECT]) > t->traceInject) { + printf("Inject at %f\n", time); + printState(t); + injectParticles(t); + t->lastUpdate[INJECT] = time; + } + + if ((time - t->lastUpdate[WRITE]) > t->traceWrite) { + printf("Write at %f\n", time); + writeParticles(t); + t->lastUpdate[WRITE] = time; + } + + advanceParticles(t, time - t->lastUpdate[ADVANCE], u, v); + t->lastUpdate[ADVANCE] = time; + } +} + +void initTrace(Tracing* t, Parameter* p) +{ + size_t numParticles = p->nparticles; + size_t totalParticles = (size_t)(p->te - p->traceStart) / (size_t)p->traceInject; + totalParticles += 2; + totalParticles *= numParticles; + + double x1 = p->lineX1; + double y1 = p->lineY1; + double x2 = p->lineX2; + double y2 = p->lineY2; + + for (int i = 0; i < NUMTIMERS; i++) { + t->lastUpdate[i] = p->traceStart; + } + t->grid.imax = p->imax; + t->grid.jmax = p->jmax; + t->grid.xlength = p->xlength; + t->grid.ylength = p->ylength; + t->grid.dx = p->xlength / p->imax; + t->grid.dy = p->ylength / p->jmax; + t->numParticles = numParticles; + t->totalParticles = 0; + t->cursor = 0; + t->traceStart = p->traceStart; + t->traceWrite = p->traceWrite; + t->traceInject = p->traceInject; + t->particles = (int*)malloc(totalParticles * sizeof(int)); + t->memorypool = (double*)malloc(totalParticles * NCOORD * sizeof(double)); + t->line = (double*)malloc(numParticles * NCOORD * sizeof(double)); + double* line = t->line; + + for (int i = 0; i < numParticles; i++) { + double spacing = (double)i / (double)(numParticles - 1); + double x = spacing * x1 + (1.0 - spacing) * x2; + double y = spacing * y1 + (1.0 - spacing) * y2; + + printf("S: %f x: %f y: %f\n", spacing, x, y); + line[i * NCOORD + X] = x; + line[i * NCOORD + Y] = y; + } +} + +void freeTrace(Tracing* t) { free(t->line); } diff --git a/BasicSolver/2D-seq-pt/src/trace.h b/BasicSolver/2D-seq-pt/src/trace.h new file mode 100644 index 0000000..4ab56a3 --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/trace.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __TRACE_H_ +#define __TRACE_H_ +#include "grid.h" +#include "parameter.h" + +typedef enum COORD { X = 0, Y, NCOORD } COORD; +typedef enum { ADVANCE = 0, INJECT, WRITE, NUMTIMERS } TIMER; + +typedef struct Tracing { + double traceStart; + double traceWrite; + double traceInject; + double dt; + double lastUpdate[NUMTIMERS]; + double* memorypool; + double* line; + int cursor; + int* particles; + int numParticles; + int totalParticles; + Grid grid; +} Tracing; + +extern void initTrace(Tracing* t, Parameter* p); +extern void trace(Tracing* t, double* u, double* v, double time); +#endif diff --git a/BasicSolver/2D-seq-pt/src/util.h b/BasicSolver/2D-seq-pt/src/util.h new file mode 100644 index 0000000..61a1dff --- /dev/null +++ b/BasicSolver/2D-seq-pt/src/util.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "------------------------------------------------------------------------" \ + "----\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#endif // __UTIL_H_ diff --git a/BasicSolver/2D-seq-pt/surface.plot b/BasicSolver/2D-seq-pt/surface.plot new file mode 100644 index 0000000..4f7ccd9 --- /dev/null +++ b/BasicSolver/2D-seq-pt/surface.plot @@ -0,0 +1,7 @@ +set terminal png size 1024,768 enhanced font ,12 +set output 'p.png' +set datafile separator whitespace + +set grid +set hidden3d +splot 'pressure.dat' using 1:2:3 with lines diff --git a/BasicSolver/2D-seq-pt/vector.plot b/BasicSolver/2D-seq-pt/vector.plot new file mode 100644 index 0000000..0934ab2 --- /dev/null +++ b/BasicSolver/2D-seq-pt/vector.plot @@ -0,0 +1,5 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'velocity.png' +set datafile separator whitespace + +plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/BasicSolver/2D-seq/Makefile b/BasicSolver/2D-seq/Makefile new file mode 100644 index 0000000..57f99f4 --- /dev/null +++ b/BasicSolver/2D-seq/Makefile @@ -0,0 +1,71 @@ +#======================================================================================= +# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(wildcard $(SRC_DIR)/*.c) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm format + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/BasicSolver/2D-seq/README.md b/BasicSolver/2D-seq/README.md new file mode 100644 index 0000000..d980b54 --- /dev/null +++ b/BasicSolver/2D-seq/README.md @@ -0,0 +1,78 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about solver, affinity settings, allocation sizes and timer resolution. +For debugging you may want to set the VERBOSE option: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +` + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. + +## Usage + +You have to provide a parameter file describing the problem you want to solve: +``` +./exe-CLANG dcavity.par +``` + +Examples are given in in dcavity (a lid driven cavity test case) and canal (simulating a empty canal). + +You can plot the resulting velocity and pressure fields using gnuplot: +``` +gnuplot vector.plot +``` +and for the pressure: +``` +gnuplot surface.plot +``` diff --git a/BasicSolver/2D-seq/canal.par b/BasicSolver/2D-seq/canal.par new file mode 100644 index 0000000..8b9fd9c --- /dev/null +++ b/BasicSolver/2D-seq/canal.par @@ -0,0 +1,46 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 100.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 50 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 100.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.00001 # stopping tolerance for pressure iteration +omg 1.8 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-seq/config.mk b/BasicSolver/2D-seq/config.mk new file mode 100644 index 0000000..af5f1a0 --- /dev/null +++ b/BasicSolver/2D-seq/config.mk @@ -0,0 +1,12 @@ +# Supported: GCC, CLANG, ICC +TAG ?= CLANG +ENABLE_OPENMP ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DDEBUG +#OPTIONS += -DBOUNDCHECK +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/2D-seq/dcavity.par b/BasicSolver/2D-seq/dcavity.par new file mode 100644 index 0000000..4241393 --- /dev/null +++ b/BasicSolver/2D-seq/dcavity.par @@ -0,0 +1,46 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 1 # 2 = free-slip 4 = periodic +bcRight 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 10.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +imax 40 # number of interior cells in x-direction +jmax 40 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 10.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/2D-seq/include_CLANG.mk b/BasicSolver/2D-seq/include_CLANG.mk new file mode 100644 index 0000000..466f441 --- /dev/null +++ b/BasicSolver/2D-seq/include_CLANG.mk @@ -0,0 +1,17 @@ +CC = clang +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +# CFLAGS = -O3 -std=c17 $(OPENMP) +CFLAGS = -Ofast -std=c17 +#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG +LFLAGS = $(OPENMP) -lm +DEFINES = -D_GNU_SOURCE# -DDEBUG +INCLUDES = diff --git a/BasicSolver/2D-seq/include_GCC.mk b/BasicSolver/2D-seq/include_GCC.mk new file mode 100644 index 0000000..427e798 --- /dev/null +++ b/BasicSolver/2D-seq/include_GCC.mk @@ -0,0 +1,14 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/2D-seq/include_ICC.mk b/BasicSolver/2D-seq/include_ICC.mk new file mode 100644 index 0000000..94b8e20 --- /dev/null +++ b/BasicSolver/2D-seq/include_ICC.mk @@ -0,0 +1,14 @@ +CC = icc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/2D-seq/src/affinity.c b/BasicSolver/2D-seq/src/affinity.c new file mode 100644 index 0000000..b501665 --- /dev/null +++ b/BasicSolver/2D-seq/src/affinity.c @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifdef __linux__ +#ifdef _OPENMP +#include +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_THREADS 128 +#define gettid() syscall(SYS_gettid) + +static int getProcessorID(cpu_set_t* cpu_set) +{ + int processorId; + + for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { + if (CPU_ISSET(processorId, cpu_set)) { + break; + } + } + return processorId; +} + +int affinity_getProcessorId() +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); + + return getProcessorID(&cpu_set); +} + +void affinity_pinThread(int processorId) +{ + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); +} + +void affinity_pinProcess(int processorId) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} +#endif /*_OPENMP*/ +#endif /*__linux__*/ diff --git a/BasicSolver/2D-seq/src/affinity.h b/BasicSolver/2D-seq/src/affinity.h new file mode 100644 index 0000000..d844fe5 --- /dev/null +++ b/BasicSolver/2D-seq/src/affinity.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef AFFINITY_H +#define AFFINITY_H + +extern int affinity_getProcessorId(); +extern void affinity_pinProcess(int); +extern void affinity_pinThread(int); + +#endif /*AFFINITY_H*/ diff --git a/BasicSolver/2D-seq/src/allocate.c b/BasicSolver/2D-seq/src/allocate.c new file mode 100644 index 0000000..81e1e9d --- /dev/null +++ b/BasicSolver/2D-seq/src/allocate.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +void* allocate(int alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/BasicSolver/2D-seq/src/allocate.h b/BasicSolver/2D-seq/src/allocate.h new file mode 100644 index 0000000..54cfe06 --- /dev/null +++ b/BasicSolver/2D-seq/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(int alignment, size_t bytesize); + +#endif diff --git a/BasicSolver/2D-seq/src/likwid-marker.h b/BasicSolver/2D-seq/src/likwid-marker.h new file mode 100644 index 0000000..eb7cc78 --- /dev/null +++ b/BasicSolver/2D-seq/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/BasicSolver/2D-seq/src/main.c b/BasicSolver/2D-seq/src/main.c new file mode 100644 index 0000000..0a28cde --- /dev/null +++ b/BasicSolver/2D-seq/src/main.c @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "parameter.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" + +int main(int argc, char** argv) +{ + double S, E; + Parameter params; + Solver solver; + initParameter(¶ms); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(¶ms, argv[1]); + printParameter(¶ms); + initSolver(&solver, ¶ms); +#ifndef VERBOSE + initProgress(solver.te); +#endif + + double tau = solver.tau; + double te = solver.te; + double t = 0.0; + int nt = 0; + + S = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) computeTimestep(&solver); + setBoundaryConditions(&solver); + setSpecialBoundaryCondition(&solver); + computeFG(&solver); + computeRHS(&solver); + if (nt % 100 == 0) normalizePressure(&solver); + solve(&solver); + adaptUV(&solver); + t += solver.dt; + nt++; + +#ifdef VERBOSE + printf("TIME %f , TIMESTEP %f\n", t, solver.dt); +#else + printProgress(t); +#endif + } + E = getTimeStamp(); + stopProgress(); + printf("Solution took %.2fs\n", E - S); + writeResult(&solver); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/2D-seq/src/parameter.c b/BasicSolver/2D-seq/src/parameter.c new file mode 100644 index 0000000..d691627 --- /dev/null +++ b/BasicSolver/2D-seq/src/parameter.c @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_STRING(name); + PARSE_INT(bcLeft); + PARSE_INT(bcRight); + PARSE_INT(bcBottom); + PARSE_INT(bcTop); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(p_init); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + param->bcLeft, + param->bcRight, + param->bcBottom, + param->bcTop); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength); + printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); +} diff --git a/BasicSolver/2D-seq/src/parameter.h b/BasicSolver/2D-seq/src/parameter.h new file mode 100644 index 0000000..f4c331a --- /dev/null +++ b/BasicSolver/2D-seq/src/parameter.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + double xlength, ylength; + int imax, jmax; + int itermax; + double eps, omg; + double re, tau, gamma; + double te, dt; + double gx, gy; + char* name; + int bcLeft, bcRight, bcBottom, bcTop; + double u_init, v_init, p_init; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/BasicSolver/2D-seq/src/progress.c b/BasicSolver/2D-seq/src/progress.c new file mode 100644 index 0000000..a9b82bd --- /dev/null +++ b/BasicSolver/2D-seq/src/progress.c @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; + +void initProgress(double end) +{ + _end = end; + _current = 0; + + printf("[ ]"); + fflush(stdout); +} + +void printProgress(double current) +{ + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); +} + +void stopProgress() +{ + printf("\n"); + fflush(stdout); +} diff --git a/BasicSolver/2D-seq/src/progress.h b/BasicSolver/2D-seq/src/progress.h new file mode 100644 index 0000000..9ef2d96 --- /dev/null +++ b/BasicSolver/2D-seq/src/progress.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ + +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(); + +#endif diff --git a/BasicSolver/2D-seq/src/solver.c b/BasicSolver/2D-seq/src/solver.c new file mode 100644 index 0000000..6102821 --- /dev/null +++ b/BasicSolver/2D-seq/src/solver.c @@ -0,0 +1,502 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +#define P(i, j) p[(j) * (imax + 2) + (i)] +#define F(i, j) f[(j) * (imax + 2) + (i)] +#define G(i, j) g[(j) * (imax + 2) + (i)] +#define U(i, j) u[(j) * (imax + 2) + (i)] +#define V(i, j) v[(j) * (imax + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imax + 2) + (i)] + +static void print(Solver* solver, double* grid) +{ + int imax = solver->imax; + + for (int j = 0; j < solver->jmax + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < solver->imax + 2; i++) { + printf("%12.8f ", grid[j * (imax + 2) + i]); + } + printf("\n"); + } + fflush(stdout); +} + +static void printConfig(Solver* solver) +{ + printf("Parameters for #%s#\n", solver->problem); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + solver->bcLeft, + solver->bcRight, + solver->bcBottom, + solver->bcTop); + printf("\tReynolds number: %.2f\n", solver->re); + printf("\tGx Gy: %.2f %.2f\n", solver->gx, solver->gy); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", solver->xlength, solver->ylength); + printf("\tCells (x, y): %d, %d\n", solver->imax, solver->jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", solver->dt, solver->te); + printf("\tdt bound: %.6f\n", solver->dtBound); + printf("\tTau factor: %.2f\n", solver->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", solver->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", solver->eps); + printf("\tgamma factor: %f\n", solver->gamma); + printf("\tomega (SOR relaxation): %f\n", solver->omega); +} + +void initSolver(Solver* solver, Parameter* params) +{ + solver->problem = params->name; + solver->bcLeft = params->bcLeft; + solver->bcRight = params->bcRight; + solver->bcBottom = params->bcBottom; + solver->bcTop = params->bcTop; + solver->imax = params->imax; + solver->jmax = params->jmax; + solver->xlength = params->xlength; + solver->ylength = params->ylength; + solver->dx = params->xlength / params->imax; + solver->dy = params->ylength / params->jmax; + solver->eps = params->eps; + solver->omega = params->omg; + solver->itermax = params->itermax; + solver->re = params->re; + solver->gx = params->gx; + solver->gy = params->gy; + solver->dt = params->dt; + solver->te = params->te; + solver->tau = params->tau; + solver->gamma = params->gamma; + + int imax = solver->imax; + int jmax = solver->jmax; + size_t size = (imax + 2) * (jmax + 2) * sizeof(double); + solver->u = allocate(64, size); + solver->v = allocate(64, size); + solver->p = allocate(64, size); + solver->rhs = allocate(64, size); + solver->f = allocate(64, size); + solver->g = allocate(64, size); + + for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { + solver->u[i] = params->u_init; + solver->v[i] = params->v_init; + solver->p[i] = params->p_init; + solver->rhs[i] = 0.0; + solver->f[i] = 0.0; + solver->g[i] = 0.0; + } + + double dx = solver->dx; + double dy = solver->dy; + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy); + solver->dtBound = 0.5 * solver->re * 1.0 / invSqrSum; +#ifdef VERBOSE + printConfig(solver); +#endif +} + +void computeRHS(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double idx = 1.0 / solver->dx; + double idy = 1.0 / solver->dy; + double idt = 1.0 / solver->dt; + double* rhs = solver->rhs; + double* f = solver->f; + double* g = solver->g; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + RHS(i, j) = idt * + ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy); + } + } +} + +void solve(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double eps = solver->eps; + int itermax = solver->itermax; + double dx2 = solver->dx * solver->dx; + double dy2 = solver->dy * solver->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* p = solver->p; + double* rhs = solver->rhs; + double epssq = eps * eps; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + } + + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmax + 1) = P(i, jmax); + } + + for (int j = 1; j < jmax + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } + + res = res / (double)(imax * jmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif +} + +static double maxElement(Solver* solver, double* m) +{ + int size = (solver->imax + 2) * (solver->jmax + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + return maxval; +} + +void normalizePressure(Solver* solver) +{ + int size = (solver->imax + 2) * (solver->jmax + 2); + double* p = solver->p; + double avgP = 0.0; + + for (int i = 0; i < size; i++) { + avgP += p[i]; + } + avgP /= size; + + for (int i = 0; i < size; i++) { + p[i] = p[i] - avgP; + } +} + +void computeTimestep(Solver* solver) +{ + double dt = solver->dtBound; + double dx = solver->dx; + double dy = solver->dy; + double umax = maxElement(solver, solver->u); + double vmax = maxElement(solver, solver->v); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + + solver->dt = dt * solver->tau; +} + +void setBoundaryConditions(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double* u = solver->u; + double* v = solver->v; + + // Left boundary + switch (solver->bcLeft) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + U(0, j) = 0.0; + V(0, j) = -V(1, j); + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + U(0, j) = 0.0; + V(0, j) = V(1, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + U(0, j) = U(1, j); + V(0, j) = V(1, j); + } + break; + case PERIODIC: + break; + } + + // Right boundary + switch (solver->bcRight) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + U(imax, j) = 0.0; + V(imax + 1, j) = -V(imax, j); + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + U(imax, j) = 0.0; + V(imax + 1, j) = V(imax, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + U(imax, j) = U(imax - 1, j); + V(imax + 1, j) = V(imax, j); + } + break; + case PERIODIC: + break; + } + + // Bottom boundary + switch (solver->bcBottom) { + case NOSLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = -U(i, 1); + } + break; + case SLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = U(i, 1); + } + break; + case OUTFLOW: + for (int i = 1; i < imax + 1; i++) { + U(i, 0) = U(i, 1); + V(i, 0) = V(i, 1); + } + break; + case PERIODIC: + break; + } + + // Top boundary + switch (solver->bcTop) { + case NOSLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, jmax) = 0.0; + U(i, jmax + 1) = -U(i, jmax); + } + break; + case SLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, jmax) = 0.0; + U(i, jmax + 1) = U(i, jmax); + } + break; + case OUTFLOW: + for (int i = 1; i < imax + 1; i++) { + U(i, jmax + 1) = U(i, jmax); + V(i, jmax) = V(i, jmax - 1); + } + break; + case PERIODIC: + break; + } +} + +void setSpecialBoundaryCondition(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double mDy = solver->dy; + double* u = solver->u; + + if (strcmp(solver->problem, "dcavity") == 0) { + for (int i = 1; i < imax; i++) { + U(i, jmax + 1) = 2.0 - U(i, jmax); + } + } else if (strcmp(solver->problem, "canal") == 0) { + double ylength = solver->ylength; + double y; + + for (int j = 1; j < jmax + 1; j++) { + y = mDy * (j - 0.5); + U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } +} + +void computeFG(Solver* solver) +{ + double* u = solver->u; + double* v = solver->v; + double* f = solver->f; + double* g = solver->g; + int imax = solver->imax; + int jmax = solver->jmax; + double gx = solver->gx; + double gy = solver->gy; + double gamma = solver->gamma; + double dt = solver->dt; + double inverseRe = 1.0 / solver->re; + double inverseDx = 1.0 / solver->dx; + double inverseDy = 1.0 / solver->dy; + double du2dx, dv2dy, duvdx, duvdy; + double du2dx2, du2dy2, dv2dx2, dv2dy2; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + du2dx = inverseDx * 0.25 * + ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - + (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + + fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); + + duvdy = inverseDy * 0.25 * + ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - + (V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + + fabs(V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) - U(i, j - 1))); + + du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); + du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); + F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - + (U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + + fabs(U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) - V(i - 1, j))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - + (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + + fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); + + dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); + dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); + G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); + } + } + + /* ---------------------- boundary of F --------------------------- */ + for (int j = 1; j < jmax + 1; j++) { + F(0, j) = U(0, j); + F(imax, j) = U(imax, j); + } + + /* ---------------------- boundary of G --------------------------- */ + for (int i = 1; i < imax + 1; i++) { + G(i, 0) = V(i, 0); + G(i, jmax) = V(i, jmax); + } +} + +void adaptUV(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double* p = solver->p; + double* u = solver->u; + double* v = solver->v; + double* f = solver->f; + double* g = solver->g; + double factorX = solver->dt / solver->dx; + double factorY = solver->dt / solver->dy; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; + V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; + } + } +} + +void writeResult(Solver* solver) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double dx = solver->dx; + double dy = solver->dy; + double* p = solver->p; + double* u = solver->u; + double* v = solver->v; + double x = 0.0, y = 0.0; + + FILE* fp; + fp = fopen("pressure.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax + 1; j++) { + y = (double)(j - 0.5) * dy; + for (int i = 1; i < imax + 1; i++) { + x = (double)(i - 0.5) * dx; + fprintf(fp, "%.2f %.2f %f\n", x, y, P(i, j)); + } + fprintf(fp, "\n"); + } + + fclose(fp); + + fp = fopen("velocity.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax + 1; j++) { + y = dy * (j - 0.5); + for (int i = 1; i < imax + 1; i++) { + x = dx * (i - 0.5); + double vel_u = (U(i, j) + U(i - 1, j)) / 2.0; + double vel_v = (V(i, j) + V(i, j - 1)) / 2.0; + double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); + } + } + + fclose(fp); +} diff --git a/BasicSolver/2D-seq/src/solver.h b/BasicSolver/2D-seq/src/solver.h new file mode 100644 index 0000000..1869d6d --- /dev/null +++ b/BasicSolver/2D-seq/src/solver.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "parameter.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + double dx, dy; + int imax, jmax; + double xlength, ylength; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop; +} Solver; + +void initSolver(Solver*, Parameter*); +void computeRHS(Solver*); +void solve(Solver*); +void normalizePressure(Solver*); +void computeTimestep(Solver*); +void setBoundaryConditions(Solver*); +void setSpecialBoundaryCondition(Solver*); +void computeFG(Solver*); +void adaptUV(Solver*); +void writeResult(Solver*); +#endif diff --git a/BasicSolver/2D-seq/src/timing.c b/BasicSolver/2D-seq/src/timing.c new file mode 100644 index 0000000..c4025a4 --- /dev/null +++ b/BasicSolver/2D-seq/src/timing.c @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution() +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-seq/src/timing.h b/BasicSolver/2D-seq/src/timing.h new file mode 100644 index 0000000..db95329 --- /dev/null +++ b/BasicSolver/2D-seq/src/timing.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(); +extern double getTimeResolution(); +extern double getTimeStamp_(); + +#endif // __TIMING_H_ diff --git a/BasicSolver/2D-seq/src/util.h b/BasicSolver/2D-seq/src/util.h new file mode 100644 index 0000000..61a1dff --- /dev/null +++ b/BasicSolver/2D-seq/src/util.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "------------------------------------------------------------------------" \ + "----\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#endif // __UTIL_H_ diff --git a/BasicSolver/2D-seq/surface.plot b/BasicSolver/2D-seq/surface.plot new file mode 100644 index 0000000..4f7ccd9 --- /dev/null +++ b/BasicSolver/2D-seq/surface.plot @@ -0,0 +1,7 @@ +set terminal png size 1024,768 enhanced font ,12 +set output 'p.png' +set datafile separator whitespace + +set grid +set hidden3d +splot 'pressure.dat' using 1:2:3 with lines diff --git a/BasicSolver/2D-seq/vector.plot b/BasicSolver/2D-seq/vector.plot new file mode 100644 index 0000000..0934ab2 --- /dev/null +++ b/BasicSolver/2D-seq/vector.plot @@ -0,0 +1,5 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'velocity.png' +set datafile separator whitespace + +plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/affinity.c.B7C05C30E5FF4519.idx b/BasicSolver/3D-mpi/.cache/clangd/index/affinity.c.B7C05C30E5FF4519.idx new file mode 100644 index 0000000000000000000000000000000000000000..1f2e039d67427d24a624f93dfa516d1f99fe51da GIT binary patch literal 368 zcmWIYbaTsKWMFVk@vO*AElFfyU|vw+LWZ|{P zmawxpNolGC^vs);8d>mmlXu#JiS_Sq9ec6yqsjd}Gtca?IcE8|Fk;%n+lptlW$yF- z^f#^c_0j1|YP41zxvY5hwR7l|)F-=h83M(gy`0IR`q))HBk9iMWvv%1?^_%ToH#pB z>t5C!2VX8uyO!1i7MeU4b~~_sTf#2EV#&RDg8M~nrp%VNx92C76gx)TWi>xIt8mt> z-k7__CrX-`#5=BfN}6l6oZxNVkh^T{E$eiSRX=7XsHX((WxFDGQ(k_p)xnOv|E+sw z`G0;AP#yp3Gvl6uzt74!6n6PV+v+x^7zjHpetXo^y8Wwl`I^gawoHG1{}b;loVc&J zGB*hrFbqYhX~hsaClN#!7bO>S18GK~?ahb7?zG+8#=wxAo02006yp;R;9y~5W@BaN M;S%KL9cFX_>|{W|#A{XGx<A$I%h;|!!s zco>8j7{s{PxcI;XPzC~uQqzjVfMLsU+gCMeruceB9x*XVF_2C$kgYve@MPB6EkI!z zei>nyu!-#vt{at6B0ym-IW85LB`}(i3&oP^#CxghS+{#I^N5KEipatY;SrM%ltALi z@ybcUe7Ibg@e83MF`Uw~f@X7KISr`PvAX59obiAO{LCJZ!*M*@i_$05f9lY8>| zm%GW=sx?3@!aTylFkv1saUO9bo-BtfKTPgJZJ@W3=$5lU%Y`I`;9>DURU_JH$2NA5 zNg~2hFsA}d5)nt@$qLAd!Q_fka}q)9;-chYeGmg4f((peVsZuh!}fiZWfm3@We^3) kLVyU2!N8E5o00>JHwHcd0S*=>W;RxK9xg#{PF_ZS0OOJGQUCw| literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/allocate.h.E842D58079DB450B.idx b/BasicSolver/3D-mpi/.cache/clangd/index/allocate.h.E842D58079DB450B.idx new file mode 100644 index 0000000000000000000000000000000000000000..296f7db46b4d9e62879a0975538099ef0d80d57b GIT binary patch literal 390 zcmWIYbaSg?WMFVk@vO*AElFfyU|czQzx6Jn0mv5A2wew1EOOj`*ycOTmT^lB! zj-C8;^8K#d^R4?Im!>VN-_ZKk{S41aH}}@w>}QKB#hlgt&qF8+_vgf zRphjVM^u(P=x4vf_2kKi;@8jm1NY3Gt?!~7x{9H=GB-&P=#vn;efDt%QYJhM%s?Ip zuyOHni8639F*2~TFtRZ+6s4vWy8^{cY>#l=sEiU}}gwc$g5RG6n7#JRE z3MI1!-xFr!Vq@oE7ln#408PtD1aXRsl8eQGG!xsd>KQL~9xKXXWMyE5D`NlvD^P|h literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/main.c.DC3EC91E37026717.idx b/BasicSolver/3D-mpi/.cache/clangd/index/main.c.DC3EC91E37026717.idx new file mode 100644 index 0000000000000000000000000000000000000000..a3b3c6919b22f4b80e7902c378a04e370d076673 GIT binary patch literal 1246 zcmWIYbaT7L!oc91;#rZKT9OE4G6({3aY<2TG9v@SDxg5coUOsW`G*X6&dPh)Zt3-% zY+>NQv9^IFSzcY_Qh?2+C7Uj1{rSy$L5yOpv{re^aov14}d%aJ8_sa|m zNq)8SbHDlO?Bhx6zx-3d?exQxjdT3PH^-fE2=Gg=!Dg-Qgwx+qMElH?_W9g>yp5inm~=|OJAJX{C!&Dz4{~xZQHr~Rp;a$?d&^q zB_K$yvRf#1MU#5Zt~1YVj~x1`|6T2f;04tvhodaoCGJNbC0zMt7~5` z*&1Hw;I=$>s$>V>0)I2{wjRHDvCTyiiT(!q+gR*HvR{Y(2xseNW&1bv(Vx8;SE99o zS~=ThO`87X^AB$6?)3cE&)z6XJ^!@qZt0DELUQf*XKU0NmB697TW6O4W9AW;=aA=v>E#i3;BkOEhezC#!xJuN&5$AQ z(Fpsz!zZ={+Fi}P>ltfsRnv)3fUvW`#aWjY^ z0wTbGS)6;p&4k3r3Cp-d#e{@KBqXJzWf&x2(hLkT2qrwl7#PG5qS6SaFoG$BV2UD` fQV3>pZb}YF5ubnn2MZH38!I~xmmoJMFC#wyK{mV% literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/parameter.c.C63CC045C4D27867.idx b/BasicSolver/3D-mpi/.cache/clangd/index/parameter.c.C63CC045C4D27867.idx new file mode 100644 index 0000000000000000000000000000000000000000..c64929204ffc7dd99a7221a8d8822b0405927b23 GIT binary patch literal 3254 zcmd6peN zi=EVCDFtB~5uG6Oh*$w9$vAcjPcNsmd{5N0;~@>o%3hx3dA8@@I{nek;c!1Yzu))w zdw$RNdG2(of`So15XxFryUpSz-oq`)ZvgfC)p|#%(SC4(oReo%o z{N-fi>4hyN50^YNaijL+;^1rIQNP#Q8YaFv?w6l=PIX z9rb3i-~Yv$)>>8nZ*MP8y4F$p$DHja=XP!~NN;SGZrV8R zr+G!S+Sy@SZHD`h-g9o^dUyOKK!^!~t>#q=xo}>%kXYpBLqyX)oKGgPW^wUT^b8=P zU_cNrD3qglbf`Dr(e88hz1fTn5Q|}em?sY97#;%yNZ&gOgu_zjyq`}Hyg1*ZlltHs zAHYTjM~hG*pQyd}s{GU;Kfs1jVZok)telp!C(6K66?F(2ftrdHN z%!y_x3gQHX`34M15v5>CsiIU5MY^tur*{2A0-3UM4B+NkQR*U$U2PHkZUlq zl2(iw$wrKt$R>-x++G6`>)tmigX+7O*`5w<73bQ#Z0j$sEJ(G?l{)*3(PI2 z%QRtKY>^9uQ)5dP9)^2tWQ>^a>e>2ew8!hdNbL1hpO8b}g-{`wpR<{4+|N`)C`x(M zbk_0~Z>$LJe|%tk1Uh4A8{yG&-{k1)4P`g$nccTp*hLv#hIL_P%`r%FLm#*Au<~*m zl&m7F_(*DNGBba9YZnKb3lauJdJlzID8>#&wWvA@DZM&RZwu4>_8OmL|E;(F`yCv- zH*cmhsOMF(X7%yd;m#O1G7Y2gTrx*@o|T<=y2^e13Mc_)*DNLrv#T}VT8swxQBOzvQorAg zaLOeSC71zg5^6jz&`*rzBa!=_OM<%LR5^rz z(O7_#zLf=}Yp)ONg^Q|WDzQb?mDd?j(CqiW7N#$(dlQz63y2Fw<>vVBX7yJ_r^8$w zt;2H7Dzg^l&S{_6X>XHgVNfYuih0vanLXb8A>u%lG^g}u*tn1qVvXm}Ie6pC8#9pZ z&C#6_)8L9Rm_k!Dwt)&-fwj_dcS{;dS!m!7CwFR7VJim4fCbf4^`2J#v@ENV9zNvn zUaKIEMz@c{(qwW!I~`_cq5HveT@gg2o9}!$^gs~d-bc^xGs1@rS~{nB@YjcS@tE)k eE|M_YYMb0wT&O?<0`BtXFQNoNVZnhRJmFtYEMo}( literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/parameter.h.4BBCC8C05260A054.idx b/BasicSolver/3D-mpi/.cache/clangd/index/parameter.h.4BBCC8C05260A054.idx new file mode 100644 index 0000000000000000000000000000000000000000..36e55aee91af6147f67e451f7787a73201e4801b GIT binary patch literal 2124 zcmYk64NR1E9LImW#qm6M|9kFv&btE+_2494g89NMf($Gr$RVJ0BhI4Yc&T0*m8g4Jw@FDqo`gnSJ$rgNaUS?l-zb3clW?~Tvro);eOLr(hQ z#n0*ho>`M#Sz72R@CN`BewTaPYwzJ}qYpr3Q{j%j%#*W~P4~0i!?i6ra^|Tt*7U?a z=}htK9Xl>fq}_{jwpUh^^tYrwcj3)jjky;$eD&l|uWNg8UG)6Uq4g6*rMvqdq{iG< zA3t#Nx0e$B8JypDboV^ZVBZB}Te>~)$Mw-yI*rYL2K65~F|T^h)<1^Bin`8q$5*PNSIJyVNoST)~b*t4tPfKg|rY{zygq{vF{Ai3iS>C7mmd zduMOT*?mj%AIr>dx^in^$+RdVX{zZA=}Bu__}))(mhc>b-Ap?fmXf&F{kdAdMT9%R+zM8>I)Xd0KF{Be3hrS=J zjl7|Uu*s`|VhgZo5aj0rPJ@r)lLW;xmeg7L^kAVkSOP5Se{aak9m}|O;xJy{h!-%( z1}26su;TpjkiMjvBMhQY%VDxp4C$4JzP^4pF|mh1IBHGSOkzkET<#b-oU^Njfr;O} znFWg>y|%S}Zs_dS3=ABojk3`ohV;6!l8-A2zw2aRMy+VdG%=)0{G%$}r>Y(>(4$r( z)F?5eJ*S+%JG)kNGf?@_vSJWJ`i+mwJ)IHbFEg;C)*`fMF;n`Nm?qzhfm8;MqL$5a zs2I}KwL@hsPgzU#FoPbeHNc9G)!%pWxxcmaZjp7@!w?2Czd1#}!_AK#({a{@uQfZJt<`oUwsHAU}Sdnka^j?>V^YjI({CgMo%xF)Ai8qFo?`s3j(;Vn|%!1sD!CeekT|xVQLdG z50fx8iI@jTm|8^4LnKTM@^@o^gsDBmJiICFZ4NOHjxe=`zr1M7%c(GaNz9AhO=bB@ zlSO&CUa@EvJ3U9eV%4m6I!F#~ z#oip>%&IHE{Z{FhD`g`=n@xL}cAF2NA|bj<%J2r#Nz**`MbyJ#PLo!y@RC;`$VW7t+xQiZgAd z1=*guW1yv8yUt!OUHrDd<;PDHEEx|@eYfI^NI{CYV_W*0*~cujd=JQpwtlHf*}GG* zsAuy()yGpqCf|%b^h&V9~c#QRg_h_bmns7p3eFEV`dASo^rg=GO)pv zJtks*D6jP9eXm$%u5vcm*L0?Wch%hkm#i+k{dsEhGI7$n^jlvx$L)-oosqiLD{Oa4 zPhRvb%hWrs)>$@x{b_z~|GM|}|HBjZu3dLuaZ2D7@kd>!{4Y$n`hjhdw9WM5%G{)d zz;I{R{5Gv^&)+B>26+Ys8A(=2Az3aSE=dMXW<~}@aYk82n@fRLZ@y4$=3$TnDiB~5 z5R~QQ;gmoqus8L|K6bLEf`>r?s6dZZPf(Vdha0GXi3wst_ErYZ^Lz*Vcp1ci3V8W= zg}?;Raxl2<;}0a^g^Cd>ku zG*B4iIhe4!&ZkxB#~E0F7Km|+!G(Ebw7IpBc!u1Da2_lmn7L2_fJa7%MTi4tJJ1`- z(&{i5c5JQ|Nz(bY9ApcN7?L-%S+u!eig;uUSq$OcIME}uQ)tqhJ|-R+6#*4-n0t9- z)CAPvyo+9MUiZ5Q9Rev*R?$S5q^7I}=fPrzl?x?Ccw~fGgn1C|mF1HaLl_{>AuoX7 zDT^q}z<9uz%t-_}r?@D&*d4@>1`%MuD04%C<+OCiCmU7?NhxVr1__u91A{DrDTQE4 fBACg!DLFzwuAqQ0H#;jECkGcluLvKH0E-X+CJ|LK literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/progress.h.00B7B5207E4C4A3D.idx b/BasicSolver/3D-mpi/.cache/clangd/index/progress.h.00B7B5207E4C4A3D.idx new file mode 100644 index 0000000000000000000000000000000000000000..8718df84ee4dbb30967967a80e119ffbe338f398 GIT binary patch literal 388 zcmWIYbaSg=WMFVk@vO*AElFfyU|&REpLtK2r&ii+hEwxQV^)_v7Be5) ziuqg#$aU=He_9*6TH<)m7n?-;3%Rc%#gxAv=G{?XThEyl^Y4?^FK1!D4~%=A48AZF zSLP-q1MOzl{5Gv^&)+B>26iA11X#IvxFi`knHd>4Sr}OvZ7u~~z4=11nTLT5BG16e z$-^muP+)KBlYQ)DO$84F2TTDs4>wQ&6BEP)hN9H8;uxSD%q~Vwh#gSnFdKlvV3VQ3 zFmr*zVDq5DuT}R6AA57tgOQ7sor7HzDhjkCClSOhE=n%u2GUHd=SpP0zjLyPW?%pS DSW{?O literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/solver.c.C754E0A082732D4F.idx b/BasicSolver/3D-mpi/.cache/clangd/index/solver.c.C754E0A082732D4F.idx new file mode 100644 index 0000000000000000000000000000000000000000..d4bfa763bc36a39bd7af35cffe51bec5eb03ab39 GIT binary patch literal 7350 zcmZ8m30M?I8tv-tip*Hz3?dA3FiZoY2*Wi9C}Ic*f)^l)ibhfKAb6VyF;P$s&qN7{ z7pNdnBzQqICSg4gk(iY=8aHaxs37Wk0UF~$bZf#^S7rPA9Orfab^OPB^-q%!6BV^j z&v6mcVlrl@E=mzOj#J}*3l}YzbsKQpWXN$DMSBy6&ei&=B zJMC7EU3sO4|1<7F#>3AaUhB2)&z3(kr>AGU-+n1QbxcyTcK@>C z+Q;qVnwr~Y-;WC3FsgiD+I{EH_a3|`dZ*y+^Y@4KDxUkKu{F*2yGNhydidmz_{?9o zPxz#IY2=d=GyDCkzB(o-HmT1C`Ncm?UQ_eg`n4Nty9|-0ytz?)DE`3u;P%6d3hYkp zTNg00v1xIN@5M37bxXbVC3}KTPsn)hfU12&Wy8*&Ybz7~yiju~Jbmeq-^%W+f4<{N zRVD~-&h7CCEJ@sO7=}E!66zS$?ee*w$3OJD|8ZM}cXRNoEg!z$Wl5<0=&?seS8jDI z%JCn2dV}M&Wp(8l-`?4!xckku;qH^~8O)a+1fM)S`q0JYaiuoZ$?Nwj%g(OxKes!4 zU9auc+7Tfh#_}xp_tH$So8q@VKHt)EB-uZ`%-r*oIsg6d!lu^;+Z>J_>C|QEgQ8E) zrup3Y>4Ua!>k^XHugBkn-@87|8@wZ;Y0b+p0=0*)a9;yi5OxTfgSj z(OpHuxTzD(mN#WvpKp48UTr_WFy)VjG;nm~fw#tTfeH#Cc$oM*uzo4z9k?*MXXh_4~7a z&ok}65a)nR${;L*$TbiKB5Lx1chk767gYs_>xE5j0oz+Z^S}SdUOtST6!l4sG8%DO zZ1Nhw*TCM@5Bk~DSw8&yV8^!}mF0-*i%o6{@TTD68UO=as7c;e(HBpJ4+=q?J2nZ2 z;oV)mp*Iei$aekR@>p)W8F8NY@kSnQOqoFfT#3Rm(O^!i$460m%V33NMa0^nWFAT?7f$n_W(NWz8@knZfibiNF z50-90#Vv}J+_Zl8!hb!C4Tl+rxyo)x34RHI1S!7{c#+~#o{fzH3;}L(a;1@IWDjDb zs#{|B)=~Sj0+K>QLkCf#QD{_mVuYk^K)($bNdfd6la!V@(Hskq6lIL^AZDZlf%32V zBS9jChi`XQ9eJSLclB&e+1z(UB+WoGnAPU1=F1yI(n_FR2@GU%B<%sZJ;0glMZf4? z%Isef*W64NUkqHxiZ~}Ax)}_Wc`vns!L1;MhIcEk*~;r_)NTt-w*`aD za4BDO%olZ}(vVaR4du{8({U1NPC`9Nhh<=$@>1q_dmN=H;uIy>bl@ib)#XN;D17@G z=yDC%k^uBq)fOMxy!f>vl3b9B9Swk_k{l>7{Z|bVt)n=5C{dL8~ zyx8ecf2kO6$2ZhBjLtk3jb&zDE$UW_1~OON?OeCOf@e?gpn-h{`qDuY(L_c#){45d zq6-<6kQ9>WI45R`)o~xQg${~F(afM7y!{T|iIjU{*wI1n`f4pUNSf*})tNds&0!jY zQXNtmG~Ho3gJw9)V9-p5nGBlcFpEKH4rwwX{KJaSd6biY5n|y7od-C1kUZ{bY}4@r zRy7`F93>0DgpJ7>Oe7=u9_w`y0HL#5v{K z;d=)XHf7ikaZ$~ z!Ff+`yeBx5K}dQmI6oHL37+-kYJZ*Y+-}(0WMi^?aHh>kFIb#f^9{Z|&NxoKoi6sy zSkRF90^g1{M$568dv5%qh={sF)_O+f9&{&z(O_mLS%Q6*pd)cx=+aGSo%>NLp6e_W z&q5ob%ZnP^ZnzztZ~{L_GA7AulVd?t{$CS_q#WKZhu6!^pp?to<+3=Bx67lL*x|D6 z>=;Seez?KD_8ZsVC*g(XndZq=L_&q_JKIOu{(@%_ANxx<&eRCw2)P)M6mN`|(UO$w zsr%tKL$J|#*zG*TeR=1Fq!!q%h2o_n+h;g#>d@lV2O9^=gn3BcoTK=B4t zk}x6Z85EyUoc(rUed%oXlh!1RG>@i9h(R$-u^$)gj|+0K=j^@vRT-~vQX(mEU?9`@ z#i2Ncl8a4?nf5XVxa0td9^*TJeg|#EkhBBz*#Y{}Qy`nyX4CK^X}##ao~kjD4nWTX z(2J-ElIozQ4r+;?dGAK%Hy(K3TA{ar{cWHl0VK@Cd9n8FKf}(=OX9UF=idR-Iv7h4gY&YJRhv=as9l%<% z+$CH-gg)g`wjS0;pTp@H;5~rR{LyGM!;$Skza6-e*lp+GhBr~lkjX2#t6Bne+>QV>5;s+8cb}homNAM?Wyr}rWp)Z zg*%WmBWamxnFH0h&7kXMpe90zaJS)f3m7Sp9|6Zlz(9%o8uWY(bfiELegUN0$eU;s zZwjWHf(K%l!bFo}AWfD#zKXPPqQ&>$%fxa6^XF9rHimhYvYPbtgy6})x@%l87&y@2I+ z74)oP`CSh+^_1Ti!}>a081Qj}Et2$(dL2!Lskh0CB1=CDb7sY&SVnOtIZu*3Mp8D= zX0wXB73jDAU2zS~z(f<(3UsZ&NE5Z0*KB4Lw@GknVih-6bj)QHcNaA5ViorU)SRG- zOFY2shxLshDdx*z*$X<{5ADJE8e<$I^Z(b}+rJ!bf>oG`=b_Dch)8~|ZJaqtD5G z>_X?8PU6C2<#(amN4r???uHnL$O`m0F0XLT;=XbCcBC;WQW`MuY*vu#iWE zye-v_b)s$^{rzigj|~T)k?O-iXgJ8?Drl&p_-SSTmT?=7eulH=>+8$BA4x(<%qCWe zx>aQWZ(5!HKG zTp_9}SX?Elt5{qus;gOCC#vgMd`eWGV)0o~eU`$tlHwO}{H@;`)*ZoJN*YRIs&bKP5mS|GfOZYr zrK}J&E7;S0KQ!#8Rrty$cg-f>Vf!&3u7m#9K^Q%vTA{HOx)R>uza*rmSUI;6I)`97%nVu77xJHg1hHxW@2Z#_5k5ySWL1jf$6b)(wR5nJVPw|Sn z-o|*Kg!u4YH7Y8KSrv%JW3!rP)Ws_v8(m&h)F8y{ugdw&@W=NtpKn!l_jFHxDknQL zbEQN``nc@!$%SPFod_ZDk(8E|Ox&d)B+QeL^37l8d^hE0jCOWqRXQECDzj*C#flkE zew-^=N8N~zToE$j*6DMr{911h2rs!)b<`B}=we+`-uQlXCuX(fK3&#p&4uPajyG05 zOP}%j`OfPxjfRSk8vb%}THwJ=o6p8iF55q_b)zO~{oPacm`(R5o*WsOlpFZ!^J{P9 z#QdWdUT%2c7u@y+JZ_b_EOSwPM(S2ha$*Kuu{c5Ww&D~qGN%6Yjp3m;3w@w}X zd8p|^!e#sNwZj%hw%OB{uC**%KF@S}U-PT>mS0*bx&}7wxZb_x+lu~S)v@tIOMj(x zUFzqS4IA!QxF@Tke?Q0bUa93JDXFnrN_#x|@!GM%^Gh|h&i6a5NuFN$BID=gb=hBj z`SNdK>**B`s<>VFJZ)$(6%p7`gbitP7{y85HJo<57c|`BkEf!5` z#nw^z3y#g1l3-5r>3y*0fI`R1Xeq_N;y_U+lcba;Y+in+yy4J*a|0`@*Hk~bvaRyh zxVuviwi|ya{oCX*gO!AMn71wH$gIy4NM}N1wV>|8gLuJRB}5@Byo~S9xYQ^VQ<5SO zS+>b`9t7sHKkpOqZ3RI!^C%e%wccMJ!UJpV*;%^IrwkV<35MENB?K`|<#irl> z{GLdB;oYBfrkh?6Z&~$eR&NfJy|Up)jY74i|=Rjd*t}1h5=darw-=9ufMAS z&LD34s6*TJ{e3Ah!~9TnsKf*3$2|=)E$+X4zDVAO6Wpi}HY&X3Xd&9$eS-0$CwQN1 zTK0yLBv?Ub(E0Gd6_Ts|>OWJlv4E1^PzNi6MIKnE=O-m>UNgu*35D9`YqRmdI%8LD zkt6Cs1|_LbTT~V`53C2Bs2%;b$mawl-Jo{p9ezBp9umDcH?eqr4@&eV<^!vg>-(Xw+;+9(gK$7W30by|2f4W2l5_e0f@JluDG z{QoK*KbNlFqNEqh5AqI@c;Nh^H=e(A@h>i3-s~GRsVzL*-{9X# z!KNdASyw2Dg9G?0{WU!B0IO8ls9WEtsm^Aq#x8~(X8fy_1AZvEWF*`nn+0&)B&0R zi3ip@o`+4S89c5pB?(aLs7}WN>s>2n`lPNou#b`qs6&(?A`h(p(WYw3%xo{GBnWCr zBS}24KGaT{U*}CcP6@nvWsAn51T%#ln zYFRBCcwk*yJ^x>Q#?s4_SfLJAhEpC`*L5Ei5!|_P2_+h+Z4O&353J8l@9J1>%GFXb z0BW_@$IYL2Xys^*mg4N`&d(J!dyj5)HLZsS|l% zeQV6!#q|}>qeRjdehwQvgbf}RIZlYPxPNc=R?fV$epk~aO8x{Z=nZ-+4_x7?vgFeN zMc)n;$v{}4UJ&X9t(+tzY27P4^Y|u7+_kNgl5AMPtTjtKaD^Avhv!4`0*gdqdZ%C_ zCklx=_x#qG)d%7-q*O}$VSa!rK+OZ^x9;ijaQLcIYbbF*ZB<#-Jg{yX;&Ja~)ACwM zBB3@JOhz79|NbnZxa|0Ednk#A+U^It0Tx)3lEU$&nebrXz5;B>HWkh@CAgaa%(fDU zaTfuYZ6Ofj?g22{Iv~bfgTm|GZo$cMuK<|u5S$qI2e1g+9N;3jFJKrp1&DD!0GMqC z@G@{00GMq7a5UN<4#}PG6na*#fwWvWx41K;6RS zaLGD{qhbMbu{tqol^qhCiff}z0W+5(MvcNET%S&k@`TaMoOtoeqRg!im(8^*L^;wo zvMWwTlfr1GQCu4h3Yh5=F`5%F)20hMjR~0P5;2+*Fw-PrG$dfAN5p7Gz)XvX(TISV z4iTdX0W%FEMgsz7`a_K71I)CC7>x&*=?*cP4lvUkVl*6JrZ>cBHo#14h|y?(na=Pw zpvM3+iQ#DUm+1Y@TTTx5N*igydnL+I+9(sgs6;th6)oY*LX>07G5+|Y66IK9tQ8+V zQI505*<6~D<8ARWK7pc~ph+;|TMjnD>TY`c4T^H2KGD~u202MeGNWqHr*Iuirg$Cb zQNT>1h|!;HG{_S%dJ`~{rVBfL37E+eFJYz9ry(tMab~5rtWv66Sd_#3!@A-iQI?G|?7iRp z!KZLJ%Kgc1u)AKIqHMR@Bi&uNCfxOiigLJLxa+NsdWG|0ZpA66Rlr=QPF_-2T)_S< cEg4(NUu&T_fq2MIuYUSy)%ORNC_>5q0f)@+>i_@% literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/timing.c.3A026594A4A42B70.idx b/BasicSolver/3D-mpi/.cache/clangd/index/timing.c.3A026594A4A42B70.idx new file mode 100644 index 0000000000000000000000000000000000000000..efe904e19ddb45c6661cc4b82a7769db9de5e9e9 GIT binary patch literal 810 zcmWIYbaPW;W?*nm@vO*AElFfyU|>_4$nx9?X`SU-d4t`q$Gbh39@)=$`&kJ2h={ zrrrD}-47q=&00VC)6NxYfoqrjED9ApT_m!6W<%h#XO|}{t(VX@o5b`uFw<&6xgK9% zVgHk=^%Jh&;!pH^!D+XIHF3@)whV3Crn|4jqCC$zsbBRo^H|Wc#kV%HCvLu~%J)ZZ zJQB+T_(MM(P`I3}|CjOD4)@1aUkr+JwR7iYc5^@HaWt({oZ%fBlHc@7cK4$dh5U2# zAMMUvUiqT=vsKZ<=gKaRpR4z)I$T)0tv7(D>eGg6-_D%z3ym$4y`9nQ&#+&`KhL#j ziG6DJYM*%JBVtLWNx$dMoG83r=>Ii`=0w5bQ$PML;e*C0w@CpRiE>$_Si**@-T=4rN!CBMZg41`j5?r z$J^z<1o1G40i`+FIfcOlOq!u6HLX|_7)vl)7`aeP7r1}4>E6%O~8p*@LBM4Kp{aZl& zmbRDsKzhY^WnjWQA|hfUGBCZjk9W;RxK9xg#{PF_ZS0NmFP>Hq)$ literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/timing.h.65881B97289F66D9.idx b/BasicSolver/3D-mpi/.cache/clangd/index/timing.h.65881B97289F66D9.idx new file mode 100644 index 0000000000000000000000000000000000000000..01b77f945df6a0ec0fb2703bfcb179758e6bb792 GIT binary patch literal 376 zcmWIYbaN|UWMFVk@vO*AElFfyU|pldE=*x9^cIt9sb-| zT@w8J-0n0)>4Y@7nD05c@tn|)lcqnHO?u=XS9QOB<0bK@yxOeAmAOgDK>Ju^B9C8E zy~oGHzzXDn01Fo%mplU}BO?Pd6C(>_)#p5`J$6x{JPhm*c?K44K5l7*fzSz)+N$RvZH|8EO|JC&Uh@a+nQ3VX(9U7SqD@NwgxI*l$w(WVip%A7jpw?CYG*6N~&`i)lV@n002QQVi5oU literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/util.h.47A029E807954814.idx b/BasicSolver/3D-mpi/.cache/clangd/index/util.h.47A029E807954814.idx new file mode 100644 index 0000000000000000000000000000000000000000..87b5030cc779c7d64c4049bcc0c97a6602e0c3ed GIT binary patch literal 264 zcmWIYbaP{1WMFVk@vO*AElFfyU|X-%lYmAs6s4vW7X#@SMZ4%r8(Xv)xft0w*g-ae!Rjx`|I=o7wgZKEIC!{W!pDw9 z7gmegb^?X@Ir!ni7bQNhec73~7%0ra!NCht{qp(Jw@!akDuBX5971r_K)Z7iL28PN Vl8Zs2OpLGg*S?;%dGBoo1_1eKN%sH% literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/vtkWriter.c.D6920E8109B5D2E8.idx b/BasicSolver/3D-mpi/.cache/clangd/index/vtkWriter.c.D6920E8109B5D2E8.idx new file mode 100644 index 0000000000000000000000000000000000000000..2b7b855117b6c7c35ef8757ca30ebe2dd9abfd9e GIT binary patch literal 2210 zcmcJPe@s(X6vuDj75d}8zLvI@{=TKOTA)0TpW4ZfR$&Dq#2K8!A9I2~!70Uo;E!eC zSmF@Ebejd{jHno7BbhSeP>DLlskqHd26M(mbQ%|&8|nnby<%?PbN{bNZ~Hmt+vef+@}lcZYbyIE?|rptW?hu}e4>5d^Jj%)0XD_3ye2;G8}CBq50XlNb z+CTQs;Qf~FX6nrLq#?!NjohnSTRX0LH!F+pd|OcYjj8xTLEN^}M zy)7oFcuO&g96PRPnKiFxqpdb<9460OeeTz|*=^PmacOebhQ7MyNLTsQRrPm=+Rqh4 zH;z;d-7;(+ZJ4|Cx2`>F7EQ^|-?u(uWLECt7{STi1CV#G^!cTOjSIp-5`aAJhjj@ z?ts7y0CMx)q>1E{BoGn=2-EXSJmoX`ep69i1_a?u0L7=ENyHaHAmsC21Q_ntmzme^ zDk6cF2_QpA)x^R~!o(+h)2u$1XorLZI-grj8YUL(PI$7j-dyozECd$DZRVRLCdh{p zAPfq4aeUPg-b3}++#4j&`1XrQ>BNHf@A)h*^6u#ZJqgr|89^-XDNwF;!ut#M}mEREk!SB2$6zN2ys$~^s)LxmQWOm zWy!9yo3N+^qAj!)6GSPx6cbj2KxxJ_#34E@hNX04I>(nzr*nKxqm$!v(oVk*_d1BV zKGSGi6+}ux$-REEeg=DC6pyB{6crk!XD>V>bcP;ZD(=nYqT{_EjxrrmDwLsk3l3&l z;hP?ubGr0Y%aL8>A?7%>k85faQ6q=ssWB@NVYNDH+b(m@@E^ia%%$@JGg2$rsiMTf-W30lbtHbCY^9!%Ej~snJubVMb zBqhh%U0m1p zXgH)Lv>ehAIu7XxJ%ILe Z!r~Cdqm-&Bu~0-piCh*MB2@?`{R_d_vw{Es literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/.cache/clangd/index/vtkWriter.h.27987C0C4DAEF54F.idx b/BasicSolver/3D-mpi/.cache/clangd/index/vtkWriter.h.27987C0C4DAEF54F.idx new file mode 100644 index 0000000000000000000000000000000000000000..9d7d11f314414e7ddb7c0aa69e1e2d17671c2b86 GIT binary patch literal 540 zcmWIYbaNA7VqkDi@vO*AElFfyU|oGHtjztHW8y34JKRlPE zak1IV{KA`+M-Q3$Ngh;rkj?JMkZp0}U1#do``gOm~adTxHRL$S{Pta%uCfIJZ3;$`LKW#D9CVc_OqYJeLL1206Lfs2cki-&=el@X%A^yk!m<%;!&JPdp=1w5=gd<>jyOb`Xjn-x~f z`qkmg!@v(yz|G1HRKU&*QBahcR@?=Yg1Lo}6XFJ_YM2Xv!eE=B!Z3S*!eINL!Z6!_ z!eCpV!rStf*f~zzc!H6Oi=BgA3@XZS+Gh2a9c=-77q Y18F8MRgTsAZq7gR895m^VVW5j0EZZs(f|Me literal 0 HcmV?d00001 diff --git a/BasicSolver/3D-mpi/Makefile b/BasicSolver/3D-mpi/Makefile new file mode 100644 index 0000000..57f99f4 --- /dev/null +++ b/BasicSolver/3D-mpi/Makefile @@ -0,0 +1,71 @@ +#======================================================================================= +# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(wildcard $(SRC_DIR)/*.c) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm format + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/BasicSolver/3D-mpi/README.md b/BasicSolver/3D-mpi/README.md new file mode 100644 index 0000000..d980b54 --- /dev/null +++ b/BasicSolver/3D-mpi/README.md @@ -0,0 +1,78 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about solver, affinity settings, allocation sizes and timer resolution. +For debugging you may want to set the VERBOSE option: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +` + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. + +## Usage + +You have to provide a parameter file describing the problem you want to solve: +``` +./exe-CLANG dcavity.par +``` + +Examples are given in in dcavity (a lid driven cavity test case) and canal (simulating a empty canal). + +You can plot the resulting velocity and pressure fields using gnuplot: +``` +gnuplot vector.plot +``` +and for the pressure: +``` +gnuplot surface.plot +``` diff --git a/BasicSolver/3D-mpi/canal.par b/BasicSolver/3D-mpi/canal.par new file mode 100644 index 0000000..3ff4a5f --- /dev/null +++ b/BasicSolver/3D-mpi/canal.par @@ -0,0 +1,52 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 100.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +zlength 4.0 # domain size in z-direction +imax 200 # number of interior cells in x-direction +jmax 50 # number of interior cells in y-direction +kmax 50 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 100.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +omg 1.3 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/3D-mpi/config.mk b/BasicSolver/3D-mpi/config.mk new file mode 100644 index 0000000..88fec3f --- /dev/null +++ b/BasicSolver/3D-mpi/config.mk @@ -0,0 +1,12 @@ +# Supported: GCC, CLANG, ICC +TAG ?= ICC +ENABLE_OPENMP ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DDEBUG +#OPTIONS += -DBOUNDCHECK +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/3D-mpi/dcavity.par b/BasicSolver/3D-mpi/dcavity.par new file mode 100644 index 0000000..c5cb201 --- /dev/null +++ b/BasicSolver/3D-mpi/dcavity.par @@ -0,0 +1,52 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcLeft 1 # flags for boundary conditions +bcRight 1 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 1000.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +zlength 1.0 # domain size in z-direction +imax 128 # number of interior cells in x-direction +jmax 128 # number of interior cells in y-direction +kmax 128 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 10.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +omg 1.8 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/3D-mpi/include_CLANG.mk b/BasicSolver/3D-mpi/include_CLANG.mk new file mode 100644 index 0000000..889fa93 --- /dev/null +++ b/BasicSolver/3D-mpi/include_CLANG.mk @@ -0,0 +1,17 @@ +CC = mpicc +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +# CFLAGS = -O3 -std=c17 $(OPENMP) +CFLAGS = -Ofast -std=c17 +#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG +LFLAGS = $(OPENMP) -lm +DEFINES = -D_GNU_SOURCE# -DDEBUG +INCLUDES = diff --git a/BasicSolver/3D-mpi/include_GCC.mk b/BasicSolver/3D-mpi/include_GCC.mk new file mode 100644 index 0000000..427e798 --- /dev/null +++ b/BasicSolver/3D-mpi/include_GCC.mk @@ -0,0 +1,14 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/3D-mpi/include_ICC.mk b/BasicSolver/3D-mpi/include_ICC.mk new file mode 100644 index 0000000..94b8e20 --- /dev/null +++ b/BasicSolver/3D-mpi/include_ICC.mk @@ -0,0 +1,14 @@ +CC = icc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/3D-mpi/src/allocate.c b/BasicSolver/3D-mpi/src/allocate.c new file mode 100644 index 0000000..81e1e9d --- /dev/null +++ b/BasicSolver/3D-mpi/src/allocate.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +void* allocate(int alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/BasicSolver/3D-mpi/src/allocate.h b/BasicSolver/3D-mpi/src/allocate.h new file mode 100644 index 0000000..54cfe06 --- /dev/null +++ b/BasicSolver/3D-mpi/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(int alignment, size_t bytesize); + +#endif diff --git a/BasicSolver/3D-mpi/src/comm.c b/BasicSolver/3D-mpi/src/comm.c new file mode 100644 index 0000000..568b550 --- /dev/null +++ b/BasicSolver/3D-mpi/src/comm.c @@ -0,0 +1,503 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" + +// subroutines local to this module +static int sizeOfRank(int rank, int size, int N) +{ + return N / size + ((N % size > rank) ? 1 : 0); +} + +static void setupCommunication(Comm* c, Direction direction, int layer) +{ + int imaxLocal = c->imaxLocal; + int jmaxLocal = c->jmaxLocal; + int kmaxLocal = c->kmaxLocal; + + size_t dblsize = sizeof(double); + int sizes[NDIMS]; + int subSizes[NDIMS]; + int starts[NDIMS]; + int offset = 0; + + sizes[IDIM] = imaxLocal + 2; + sizes[JDIM] = jmaxLocal + 2; + sizes[KDIM] = kmaxLocal + 2; + + if (layer == HALO) { + offset = 1; + } + + switch (direction) { + case LEFT: + subSizes[IDIM] = 1; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = 1 - offset; + starts[JDIM] = 1; + starts[KDIM] = 1; + break; + case RIGHT: + subSizes[IDIM] = 1; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = imaxLocal + offset; + starts[JDIM] = 1; + starts[KDIM] = 1; + break; + case BOTTOM: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = 1; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = 1; + starts[JDIM] = 1 - offset; + starts[KDIM] = 1; + break; + case TOP: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = 1; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = 1; + starts[JDIM] = jmaxLocal + offset; + starts[KDIM] = 1; + break; + case FRONT: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = 1; + starts[IDIM] = 1; + starts[JDIM] = 1; + starts[KDIM] = 1 - offset; + break; + case BACK: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = 1; + starts[IDIM] = 1; + starts[JDIM] = 1; + starts[KDIM] = kmaxLocal + offset; + break; + case NDIRS: + printf("ERROR!\n"); + break; + } + + if (layer == HALO) { + MPI_Type_create_subarray(NDIMS, + sizes, + subSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &c->rbufferTypes[direction]); + MPI_Type_commit(&c->rbufferTypes[direction]); + } else if (layer == BULK) { + MPI_Type_create_subarray(NDIMS, + sizes, + subSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &c->sbufferTypes[direction]); + MPI_Type_commit(&c->sbufferTypes[direction]); + } +} + +static void assembleResult(Comm* c, + double* src, + double* dst, + int imaxLocal[], + int jmaxLocal[], + int kmaxLocal[], + int offset[], + int kmax, + int jmax, + int imax) +{ + int numRequests = 1; + + if (c->rank == 0) { + numRequests = c->size + 1; + } + + MPI_Request requests[numRequests]; + + /* all ranks send their interpolated bulk array */ + MPI_Isend(src, + c->imaxLocal * c->jmaxLocal * c->kmaxLocal, + MPI_DOUBLE, + 0, + 0, + c->comm, + &requests[0]); + + /* rank 0 assembles the subdomains */ + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + MPI_Datatype domainType; + int oldSizes[NDIMS] = { kmax, jmax, imax }; + int newSizes[NDIMS] = { kmaxLocal[i], jmaxLocal[i], imaxLocal[i] }; + int starts[NDIMS] = { offset[i * NDIMS + KDIM], + offset[i * NDIMS + JDIM], + offset[i * NDIMS + IDIM] }; + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} + +static int sum(int* sizes, int position) +{ + int sum = 0; + + for (int i = 0; i < position; i++) { + sum += sizes[i]; + } + + return sum; +} + +// exported subroutines +void commReduction(double* v, int op) +{ + if (op == MAX) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + } else if (op == SUM) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + } +} + +int commIsBoundary(Comm* c, Direction direction) +{ + switch (direction) { + case LEFT: + return c->coords[ICORD] == 0; + break; + case RIGHT: + return c->coords[ICORD] == (c->dims[ICORD] - 1); + break; + case BOTTOM: + return c->coords[JCORD] == 0; + break; + case TOP: + return c->coords[JCORD] == (c->dims[JCORD] - 1); + break; + case FRONT: + return c->coords[KCORD] == 0; + break; + case BACK: + return c->coords[KCORD] == (c->dims[KCORD] - 1); + break; + case NDIRS: + printf("ERROR!\n"); + break; + } + + return 0; +} + +void commExchange(Comm* c, double* grid) +{ + int counts[6] = { 1, 1, 1, 1, 1, 1 }; + MPI_Aint displs[6] = { 0, 0, 0, 0, 0, 0 }; + + MPI_Neighbor_alltoallw(grid, + counts, + displs, + c->sbufferTypes, + grid, + counts, + displs, + c->rbufferTypes, + c->comm); +} + +void commShift(Comm* c, double* f, double* g, double* h) +{ + MPI_Request requests[6] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + MPI_Irecv(g, + 1, + c->rbufferTypes[BOTTOM], + c->neighbours[BOTTOM], + 0, + c->comm, + &requests[0]); + + /* send ghost cells to top neighbor */ + MPI_Isend(g, 1, c->sbufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]); + + /* shift F */ + /* receive ghost cells from left neighbor */ + MPI_Irecv(f, 1, c->rbufferTypes[LEFT], c->neighbours[LEFT], 1, c->comm, &requests[2]); + + /* send ghost cells to right neighbor */ + MPI_Isend(f, + 1, + c->sbufferTypes[RIGHT], + c->neighbours[RIGHT], + 1, + c->comm, + &requests[3]); + + /* shift H */ + /* receive ghost cells from front neighbor */ + MPI_Irecv(h, + 1, + c->rbufferTypes[FRONT], + c->neighbours[FRONT], + 2, + c->comm, + &requests[4]); + + /* send ghost cells to back neighbor */ + MPI_Isend(h, 1, c->sbufferTypes[BACK], c->neighbours[BACK], 2, c->comm, &requests[5]); + + MPI_Waitall(6, requests, MPI_STATUSES_IGNORE); +} + +#define G(v, i, j, k) \ + v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* wg, + double* pg, + double* u, + double* v, + double* w, + double* p, + int kmax, + int jmax, + int imax) +{ + int imaxLocal = c->imaxLocal; + int jmaxLocal = c->jmaxLocal; + int kmaxLocal = c->kmaxLocal; + + int offset[c->size * NDIMS]; + int imaxLocalAll[c->size]; + int jmaxLocalAll[c->size]; + int kmaxLocalAll[c->size]; + + MPI_Gather(&imaxLocal, 1, MPI_INT, imaxLocalAll, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&jmaxLocal, 1, MPI_INT, jmaxLocalAll, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&kmaxLocal, 1, MPI_INT, kmaxLocalAll, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + int coords[NCORDS]; + MPI_Cart_coords(c->comm, i, NDIMS, coords); + offset[i * NDIMS + IDIM] = sum(imaxLocalAll, coords[ICORD]); + offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, coords[JCORD]); + offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, coords[KCORD]); + printf("Rank: %d, Coords(k,j,i): %d %d %d, Size(k,j,i): %d %d %d, " + "Offset(k,j,i): %d %d %d\n", + i, + coords[KCORD], + coords[JCORD], + coords[ICORD], + kmaxLocalAll[i], + jmaxLocalAll[i], + imaxLocalAll[i], + offset[i * NDIMS + KDIM], + offset[i * NDIMS + JDIM], + offset[i * NDIMS + IDIM]); + } + } + + size_t bytesize = imaxLocal * jmaxLocal * kmaxLocal * sizeof(double); + double* tmp = allocate(64, bytesize); + int idx = 0; + + /* collect P */ + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = G(p, i, j, k); + } + } + } + + assembleResult(c, + tmp, + pg, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + /* collect U */ + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(u, i, j, k) + G(u, i - 1, j, k)) / 2.0; + } + } + } + + assembleResult(c, + tmp, + ug, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + /* collect V */ + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(v, i, j, k) + G(v, i, j - 1, k)) / 2.0; + } + } + } + + assembleResult(c, + tmp, + vg, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + /* collect W */ + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(w, i, j, k) + G(w, i, j, k - 1)) / 2.0; + } + } + } + + assembleResult(c, + tmp, + wg, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + free(tmp); +} + +void commPrintConfig(Comm* c) +{ + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + if (commIsMaster(c)) { + printf("Communication setup:\n"); + } + + for (int i = 0; i < c->size; i++) { + if (i == c->rank) { + printf("\tRank %d of %d\n", c->rank, c->size); + printf("\tNeighbours (front, back, bottom, top, left, right): %d, %d, %d, " + "%d, %d, %d\n", + c->neighbours[FRONT], + c->neighbours[BACK], + c->neighbours[BOTTOM], + c->neighbours[TOP], + c->neighbours[LEFT], + c->neighbours[RIGHT]); + printf("\tCoordinates (k,j,i) %d %d %d\n", + c->coords[KCORD], + c->coords[JCORD], + c->coords[ICORD]); + printf("\tLocal domain size (k,j,i) %dx%dx%d\n", + c->kmaxLocal, + c->jmaxLocal, + c->imaxLocal); + fflush(stdout); + } + } + MPI_Barrier(MPI_COMM_WORLD); +} + +void commInit(Comm* c, int kmax, int jmax, int imax) +{ + /* setup communication */ + MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank)); + MPI_Comm_size(MPI_COMM_WORLD, &(c->size)); + int dims[NDIMS] = { 0, 0, 0 }; + int periods[NDIMS] = { 0, 0, 0 }; + MPI_Dims_create(c->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NCORDS, dims, periods, 0, &c->comm); + MPI_Cart_shift(c->comm, ICORD, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]); + MPI_Cart_shift(c->comm, JCORD, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]); + MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]); + MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords); + + c->imaxLocal = sizeOfRank(c->rank, dims[ICORD], imax); + c->jmaxLocal = sizeOfRank(c->rank, dims[JCORD], jmax); + c->kmaxLocal = sizeOfRank(c->rank, dims[KCORD], kmax); + + // setup buffer types for communication + setupCommunication(c, LEFT, BULK); + setupCommunication(c, LEFT, HALO); + setupCommunication(c, RIGHT, BULK); + setupCommunication(c, RIGHT, HALO); + setupCommunication(c, BOTTOM, BULK); + setupCommunication(c, BOTTOM, HALO); + setupCommunication(c, TOP, BULK); + setupCommunication(c, TOP, HALO); + setupCommunication(c, FRONT, BULK); + setupCommunication(c, FRONT, HALO); + setupCommunication(c, BACK, BULK); + setupCommunication(c, BACK, HALO); +} + +void commFree(Comm* c) +{ + for (int i = 0; i < NDIRS; i++) { + MPI_Type_free(&c->sbufferTypes[i]); + MPI_Type_free(&c->rbufferTypes[i]); + } +} diff --git a/BasicSolver/3D-mpi/src/comm.h b/BasicSolver/3D-mpi/src/comm.h new file mode 100644 index 0000000..243db14 --- /dev/null +++ b/BasicSolver/3D-mpi/src/comm.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __COMM_H_ +#define __COMM_H_ +#include +/* + * Spatial directions: + * ICORD (0) from 0 (LEFT) to imax (RIGHT) + * JCORD (1) from 0 (BOTTOM) to jmax (TOP) + * KCORD (2) from 0 (FRONT) to kmax (BACK) + * All derived Subarray types are in C ordering + * with indices KDIM (0), JDIM(1) and IDIM(2) + * */ +typedef enum direction { LEFT = 0, RIGHT, BOTTOM, TOP, FRONT, BACK, NDIRS } Direction; +typedef enum coordinates { ICORD = 0, JCORD, KCORD, NCORDS } Coordinates; +typedef enum dimension { KDIM = 0, JDIM, IDIM, NDIMS } Dimension; +enum layer { HALO = 0, BULK }; +enum op { MAX = 0, SUM }; + +typedef struct { + int rank; + int size; + MPI_Comm comm; + MPI_Datatype sbufferTypes[NDIRS]; + MPI_Datatype rbufferTypes[NDIRS]; + int neighbours[NDIRS]; + int coords[NDIMS], dims[NDIMS]; + int imaxLocal, jmaxLocal, kmaxLocal; + MPI_File fh; +} Comm; + +extern void commInit(Comm* comm, int kmax, int jmax, int imax); +extern void commFree(Comm* comm); +extern void commPrintConfig(Comm*); +extern void commExchange(Comm*, double*); +extern void commShift(Comm* c, double* f, double* g, double* h); +extern void commReduction(double* v, int op); +extern int commIsBoundary(Comm* c, Direction direction); +extern void commCollectResult(Comm* c, + double* ug, + double* vg, + double* wg, + double* pg, + double* u, + double* v, + double* w, + double* p, + int kmax, + int jmax, + int imax); + +static inline int commIsMaster(Comm* c) { return c->rank == 0; } +#endif // __COMM_H_ diff --git a/BasicSolver/3D-mpi/src/grid.h b/BasicSolver/3D-mpi/src/grid.h new file mode 100644 index 0000000..c963429 --- /dev/null +++ b/BasicSolver/3D-mpi/src/grid.h @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __GRID_H_ +#define __GRID_H_ + +typedef struct { + double dx, dy, dz; + int imax, jmax, kmax; + double xlength, ylength, zlength; +} Grid; + +#endif // __GRID_H_ diff --git a/BasicSolver/3D-mpi/src/likwid-marker.h b/BasicSolver/3D-mpi/src/likwid-marker.h new file mode 100644 index 0000000..c3770c0 --- /dev/null +++ b/BasicSolver/3D-mpi/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/BasicSolver/3D-mpi/src/main.c b/BasicSolver/3D-mpi/src/main.c new file mode 100644 index 0000000..f53e2a1 --- /dev/null +++ b/BasicSolver/3D-mpi/src/main.c @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "parameter.h" +#include "progress.h" +#include "solver.h" +#include "test.h" +#include "timing.h" +#include "vtkWriter.h" + +int main(int argc, char** argv) +{ + int rank; + double timeStart, timeStop; + Parameter params; + Solver solver; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + initParameter(¶ms); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(¶ms, argv[1]); + if (commIsMaster(&solver.comm)) { + printParameter(¶ms); + } + initSolver(&solver, ¶ms); +#ifndef VERBOSE + initProgress(solver.te); +#endif + + double tau = solver.tau; + double te = solver.te; + double t = 0.0; + int nt = 0; + + timeStart = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) computeTimestep(&solver); + setBoundaryConditions(&solver); + setSpecialBoundaryCondition(&solver); + computeFG(&solver); + computeRHS(&solver); + // if (nt % 100 == 0) normalizePressure(&solver); + solve(&solver); + adaptUV(&solver); + t += solver.dt; + nt++; + +#ifdef VERBOSE + if (commIsMaster(&solver.comm)) { + printf("TIME %f , TIMESTEP %f\n", t, solver.dt); + } +#else + printProgress(t); +#endif + } + timeStop = getTimeStamp(); + stopProgress(); + if (commIsMaster(&solver.comm)) { + printf("Solution took %.2fs\n", timeStop - timeStart); + } + + // testInit(&solver); + // commExchange(&solver.comm, solver.p); + // testPrintHalo(&solver, solver.p); + + double *pg, *ug, *vg, *wg; + + if (commIsMaster(&solver.comm)) { + size_t bytesize = solver.grid.imax * solver.grid.jmax * solver.grid.kmax * + sizeof(double); + + pg = allocate(64, bytesize); + ug = allocate(64, bytesize); + vg = allocate(64, bytesize); + wg = allocate(64, bytesize); + } + + commCollectResult(&solver.comm, + ug, + vg, + wg, + pg, + solver.u, + solver.v, + solver.w, + solver.p, + solver.grid.kmax, + solver.grid.jmax, + solver.grid.imax); + + VtkOptions opts = { .grid = solver.grid, .comm = solver.comm }; + vtkOpen(&opts, solver.problem); + vtkScalar(&opts, "pressure", pg); + vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); + vtkClose(&opts); + + commFree(&solver.comm); + MPI_Finalize(); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/3D-mpi/src/parameter.c b/BasicSolver/3D-mpi/src/parameter.c new file mode 100644 index 0000000..d4ea5b6 --- /dev/null +++ b/BasicSolver/3D-mpi/src/parameter.c @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->zlength = 1.0; + param->imax = 100; + param->jmax = 100; + param->kmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_REAL(zlength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(kmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_REAL(gz); + PARSE_STRING(name); + PARSE_INT(bcLeft); + PARSE_INT(bcRight); + PARSE_INT(bcBottom); + PARSE_INT(bcTop); + PARSE_INT(bcFront); + PARSE_INT(bcBack); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(w_init); + PARSE_REAL(p_init); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d Front:%d " + "Back:%d\n", + param->bcLeft, + param->bcRight, + param->bcBottom, + param->bcTop, + param->bcFront, + param->bcBack); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f W:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->w_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + param->xlength, + param->ylength, + param->zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", param->imax, param->jmax, param->kmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); +} diff --git a/BasicSolver/3D-mpi/src/parameter.h b/BasicSolver/3D-mpi/src/parameter.h new file mode 100644 index 0000000..6dddf5f --- /dev/null +++ b/BasicSolver/3D-mpi/src/parameter.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + int imax, jmax, kmax; + double xlength, ylength, zlength; + int itermax; + double eps, omg; + double re, tau, gamma; + double te, dt; + double gx, gy, gz; + char* name; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + double u_init, v_init, w_init, p_init; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/BasicSolver/3D-mpi/src/progress.c b/BasicSolver/3D-mpi/src/progress.c new file mode 100644 index 0000000..a9b82bd --- /dev/null +++ b/BasicSolver/3D-mpi/src/progress.c @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; + +void initProgress(double end) +{ + _end = end; + _current = 0; + + printf("[ ]"); + fflush(stdout); +} + +void printProgress(double current) +{ + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); +} + +void stopProgress() +{ + printf("\n"); + fflush(stdout); +} diff --git a/BasicSolver/3D-mpi/src/progress.h b/BasicSolver/3D-mpi/src/progress.h new file mode 100644 index 0000000..9ef2d96 --- /dev/null +++ b/BasicSolver/3D-mpi/src/progress.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ + +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(); + +#endif diff --git a/BasicSolver/3D-mpi/src/solver.c b/BasicSolver/3D-mpi/src/solver.c new file mode 100644 index 0000000..2a49504 --- /dev/null +++ b/BasicSolver/3D-mpi/src/solver.c @@ -0,0 +1,845 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +#define P(i, j, k) \ + p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define F(i, j, k) \ + f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define G(i, j, k) \ + g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define H(i, j, k) \ + h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define U(i, j, k) \ + u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define V(i, j, k) \ + v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define W(i, j, k) \ + w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define RHS(i, j, k) \ + rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +static void printConfig(Solver* s) +{ + if (commIsMaster(&s->comm)) { + printf("Parameters for #%s#\n", s->problem); + printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n", + s->bcLeft, + s->bcRight, + s->bcBottom, + s->bcTop, + s->bcFront, + s->bcBack); + printf("\tReynolds number: %.2f\n", s->re); + printf("\tGx Gy: %.2f %.2f %.2f\n", s->gx, s->gy, s->gz); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + s->grid.xlength, + s->grid.ylength, + s->grid.zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", + s->grid.imax, + s->grid.jmax, + s->grid.kmax); + printf("\tCell size (dx, dy, dz): %f, %f, %f\n", + s->grid.dx, + s->grid.dy, + s->grid.dz); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te); + printf("\tdt bound: %.6f\n", s->dtBound); + printf("\tTau factor: %.2f\n", s->tau); + printf("Iterative parameters:\n"); + printf("\tMax iterations: %d\n", s->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", s->eps); + printf("\tgamma factor: %f\n", s->gamma); + printf("\tomega (SOR relaxation): %f\n", s->omega); + } + commPrintConfig(&s->comm); +} + +void initSolver(Solver* s, Parameter* params) +{ + s->problem = params->name; + s->bcLeft = params->bcLeft; + s->bcRight = params->bcRight; + s->bcBottom = params->bcBottom; + s->bcTop = params->bcTop; + s->bcFront = params->bcFront; + s->bcBack = params->bcBack; + + s->grid.imax = params->imax; + s->grid.jmax = params->jmax; + s->grid.kmax = params->kmax; + s->grid.xlength = params->xlength; + s->grid.ylength = params->ylength; + s->grid.zlength = params->zlength; + s->grid.dx = params->xlength / params->imax; + s->grid.dy = params->ylength / params->jmax; + s->grid.dz = params->zlength / params->kmax; + + s->eps = params->eps; + s->omega = params->omg; + s->itermax = params->itermax; + s->re = params->re; + s->gx = params->gx; + s->gy = params->gy; + s->gz = params->gz; + s->dt = params->dt; + s->te = params->te; + s->tau = params->tau; + s->gamma = params->gamma; + + commInit(&s->comm, s->grid.kmax, s->grid.jmax, s->grid.imax); + /* allocate arrays */ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + size_t size = (imaxLocal + 2) * (jmaxLocal + 2) * (kmaxLocal + 2); + + s->u = allocate(64, size * sizeof(double)); + s->v = allocate(64, size * sizeof(double)); + s->w = allocate(64, size * sizeof(double)); + s->p = allocate(64, size * sizeof(double)); + s->rhs = allocate(64, size * sizeof(double)); + s->f = allocate(64, size * sizeof(double)); + s->g = allocate(64, size * sizeof(double)); + s->h = allocate(64, size * sizeof(double)); + + for (int i = 0; i < size; i++) { + s->u[i] = params->u_init; + s->v[i] = params->v_init; + s->w[i] = params->w_init; + s->p[i] = params->p_init; + s->rhs[i] = 0.0; + s->f[i] = 0.0; + s->g[i] = 0.0; + s->h[i] = 0.0; + } + + double dx = s->grid.dx; + double dy = s->grid.dy; + double dz = s->grid.dz; + + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz); + s->dtBound = 0.5 * s->re * 1.0 / invSqrSum; + +#ifdef VERBOSE + printConfig(s); +#endif /* VERBOSE */ +} + +void computeRHS(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double idx = 1.0 / s->grid.dx; + double idy = 1.0 / s->grid.dy; + double idz = 1.0 / s->grid.dz; + double idt = 1.0 / s->dt; + + double* rhs = s->rhs; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + commShift(&s->comm, f, g, h); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx + + (G(i, j, k) - G(i, j - 1, k)) * idy + + (H(i, j, k) - H(i, j, k - 1)) * idz) * + idt; + } + } + } +} + +void solve(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid.dx * s->grid.dx; + double dy2 = s->grid.dy * s->grid.dy; + double dz2 = s->grid.dz * s->grid.dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* p = s->p; + double* rhs = s->rhs; + double epssq = eps * eps; + int it = 0; + double res = 1.0; + commExchange(&s->comm, p); + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + + double r = RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * + idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2); + + P(i, j, k) -= (factor * r); + res += (r * r); + } + } + } + + if (commIsBoundary(&s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + } + } + } + + if (commIsBoundary(&s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + } + + if (commIsBoundary(&s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + } + } + } + + if (commIsBoundary(&s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + } + + if (commIsBoundary(&s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + } + } + } + + if (commIsBoundary(&s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } + } + + commReduction(&res, SUM); + res = res / (double)(imax * jmax * kmax); +#ifdef DEBUG + if (commIsMaster(&s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + commExchange(&s->comm, p); + it++; + } + +#ifdef VERBOSE + if (commIsMaster(&s->comm)) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif +} + +static double maxElement(Solver* s, double* m) +{ + int size = (s->comm.imaxLocal + 2) * (s->comm.jmaxLocal + 2) * + (s->comm.kmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + commReduction(&maxval, MAX); + return maxval; +} + +void normalizePressure(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* p = s->p; + double avgP = 0.0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + avgP += P(i, j, k); + } + } + } + commReduction(&avgP, SUM); + avgP /= (s->grid.imax * s->grid.jmax * s->grid.kmax); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, k) = P(i, j, k) - avgP; + } + } + } +} + +void computeTimestep(Solver* s) +{ + double dt = s->dtBound; + double dx = s->grid.dx; + double dy = s->grid.dy; + double dz = s->grid.dz; + + double umax = maxElement(s, s->u); + double vmax = maxElement(s, s->v); + double wmax = maxElement(s, s->w); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + if (wmax > 0) { + dt = (dt > dz / wmax) ? dz / wmax : dt; + } + + s->dt = dt * s->tau; +} + +void setBoundaryConditions(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* u = s->u; + double* v = s->v; + double* w = s->w; + + if (commIsBoundary(&s->comm, TOP)) { + switch (s->bcTop) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = -U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = 0.0; + W(i, jmaxLocal + 1, k) = -W(i, jmaxLocal, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = 0.0; + W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = V(i, jmaxLocal - 1, k); + W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, BOTTOM)) { + switch (s->bcBottom) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = -U(i, 1, k); + V(i, 0, k) = 0.0; + W(i, 0, k) = -W(i, 1, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = 0.0; + W(i, 0, k) = W(i, 1, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = V(i, 1, k); + W(i, 0, k) = W(i, 1, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, LEFT)) { + switch (s->bcLeft) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = -V(1, j, k); + W(0, j, k) = -W(1, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = U(1, j, k); + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, RIGHT)) { + switch (s->bcRight) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = 0.0; + V(imaxLocal + 1, j, k) = -V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = -W(imaxLocal, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = 0.0; + V(imaxLocal + 1, j, k) = V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = W(imaxLocal, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = U(imaxLocal - 1, j, k); + V(imaxLocal + 1, j, k) = V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = W(imaxLocal, j, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, FRONT)) { + switch (s->bcFront) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = -U(i, j, 1); + V(i, j, 0) = -V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = W(i, j, 1); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&s->comm, BACK)) { + switch (s->bcBack) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = -U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = -V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = W(i, j, kmaxLocal - 1); + } + } + break; + case PERIODIC: + break; + } + } +} + +void setSpecialBoundaryCondition(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* u = s->u; + + if (strcmp(s->problem, "dcavity") == 0) { + if (commIsBoundary(&s->comm, TOP)) { + for (int k = 1; k < kmaxLocal; k++) { + for (int i = 1; i < imaxLocal; i++) { + U(i, jmaxLocal + 1, k) = 2.0 - U(i, jmaxLocal, k); + } + } + } + } else if (strcmp(s->problem, "canal") == 0) { + if (commIsBoundary(&s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 2.0; + } + } + } + } +} + +void computeFG(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* u = s->u; + double* v = s->v; + double* w = s->w; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + double gx = s->gx; + double gy = s->gy; + double gz = s->gz; + double dt = s->dt; + + double gamma = s->gamma; + double inverseRe = 1.0 / s->re; + double inverseDx = 1.0 / s->grid.dx; + double inverseDy = 1.0 / s->grid.dy; + double inverseDz = 1.0 / s->grid.dz; + double du2dx, dv2dy, dw2dz; + double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz; + double du2dx2, du2dy2, du2dz2; + double dv2dx2, dv2dy2, dv2dz2; + double dw2dx2, dw2dy2, dw2dz2; + + commExchange(&s->comm, u); + commExchange(&s->comm, v); + commExchange(&s->comm, w); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + du2dx = inverseDx * 0.25 * + ((U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) + U(i + 1, j, k)) - + (U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) + U(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) - U(i + 1, j, k)) + + fabs(U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) - U(i - 1, j, k))); + + duvdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) + U(i, j + 1, k)) - + (V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) + U(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) - U(i, j + 1, k)) + + fabs(V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) - U(i, j - 1, k))); + + duwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) + U(i, j, k + 1)) - + (W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) + U(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) - U(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) - U(i, j, k - 1))); + + du2dx2 = inverseDx * inverseDx * + (U(i + 1, j, k) - 2.0 * U(i, j, k) + U(i - 1, j, k)); + du2dy2 = inverseDy * inverseDy * + (U(i, j + 1, k) - 2.0 * U(i, j, k) + U(i, j - 1, k)); + du2dz2 = inverseDz * inverseDz * + (U(i, j, k + 1) - 2.0 * U(i, j, k) + U(i, j, k - 1)); + F(i, j, k) = U(i, j, k) + dt * (inverseRe * (du2dx2 + du2dy2 + du2dz2) - + du2dx - duvdy - duwdz + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) + V(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) + V(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) - V(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) - V(i - 1, j, k))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) + V(i, j + 1, k)) - + (V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) + V(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) - V(i, j + 1, k)) + + fabs(V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) - V(i, j - 1, k))); + + dvwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) + V(i, j, k + 1)) - + (W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) + V(i, j, k + 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) - V(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) - V(i, j, k + 1))); + + dv2dx2 = inverseDx * inverseDx * + (V(i + 1, j, k) - 2.0 * V(i, j, k) + V(i - 1, j, k)); + dv2dy2 = inverseDy * inverseDy * + (V(i, j + 1, k) - 2.0 * V(i, j, k) + V(i, j - 1, k)); + dv2dz2 = inverseDz * inverseDz * + (V(i, j, k + 1) - 2.0 * V(i, j, k) + V(i, j, k - 1)); + G(i, j, k) = V(i, j, k) + dt * (inverseRe * (dv2dx2 + dv2dy2 + dv2dz2) - + duvdx - dv2dy - dvwdz + gy); + + duwdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) + W(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) + W(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) - W(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) - W(i - 1, j, k))); + + dvwdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) + W(i, j + 1, k)) - + (V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) + W(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) - W(i, j + 1, k)) + + fabs(V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) - W(i, j - 1, k))); + + dw2dz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) + W(i, j, k + 1)) - + (W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) + W(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) - W(i, j, k + 1)) + + fabs(W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) - W(i, j, k - 1))); + + dw2dx2 = inverseDx * inverseDx * + (W(i + 1, j, k) - 2.0 * W(i, j, k) + W(i - 1, j, k)); + dw2dy2 = inverseDy * inverseDy * + (W(i, j + 1, k) - 2.0 * W(i, j, k) + W(i, j - 1, k)); + dw2dz2 = inverseDz * inverseDz * + (W(i, j, k + 1) - 2.0 * W(i, j, k) + W(i, j, k - 1)); + H(i, j, k) = W(i, j, k) + dt * (inverseRe * (dw2dx2 + dw2dy2 + dw2dz2) - + duwdx - dvwdy - dw2dz + gz); + } + } + } + + /* ----------------------------- boundary of F --------------------------- + */ + if (commIsBoundary(&s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(0, j, k) = U(0, j, k); + } + } + } + + if (commIsBoundary(&s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(imaxLocal, j, k) = U(imaxLocal, j, k); + } + } + } + + /* ----------------------------- boundary of G --------------------------- + */ + if (commIsBoundary(&s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, 0, k) = V(i, 0, k); + } + } + } + + if (commIsBoundary(&s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, jmaxLocal, k) = V(i, jmaxLocal, k); + } + } + } + + /* ----------------------------- boundary of H --------------------------- + */ + if (commIsBoundary(&s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + H(i, j, 0) = W(i, j, 0); + } + } + } + + if (commIsBoundary(&s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + H(i, j, kmaxLocal) = W(i, j, kmaxLocal); + } + } + } +} + +void adaptUV(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* p = s->p; + double* u = s->u; + double* v = s->v; + double* w = s->w; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + double factorX = s->dt / s->grid.dx; + double factorY = s->dt / s->grid.dy; + double factorZ = s->dt / s->grid.dz; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, k) = F(i, j, k) - (P(i + 1, j, k) - P(i, j, k)) * factorX; + V(i, j, k) = G(i, j, k) - (P(i, j + 1, k) - P(i, j, k)) * factorY; + W(i, j, k) = H(i, j, k) - (P(i, j, k + 1) - P(i, j, k)) * factorZ; + } + } + } +} diff --git a/BasicSolver/3D-mpi/src/solver.h b/BasicSolver/3D-mpi/src/solver.h new file mode 100644 index 0000000..ac9f450 --- /dev/null +++ b/BasicSolver/3D-mpi/src/solver.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "comm.h" +#include "grid.h" +#include "parameter.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g, *h; + double *u, *v, *w; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy, gz; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + /* communication */ + Comm comm; +} Solver; + +void initSolver(Solver*, Parameter*); +void computeRHS(Solver*); +void solve(Solver*); +void normalizePressure(Solver*); +void computeTimestep(Solver*); +void setBoundaryConditions(Solver*); +void setSpecialBoundaryCondition(Solver*); +void computeFG(Solver*); +void adaptUV(Solver*); +#endif diff --git a/BasicSolver/3D-mpi/src/test.c b/BasicSolver/3D-mpi/src/test.c new file mode 100644 index 0000000..1665ce6 --- /dev/null +++ b/BasicSolver/3D-mpi/src/test.c @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "test.h" + +#define G(v, i, j, k) \ + v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +void testInit(Solver* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + int myrank = s->comm.rank; + double* p = s->p; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + for (int k = 0; k < kmaxLocal + 2; k++) { + for (int j = 0; j < jmaxLocal + 2; j++) { + for (int i = 0; i < imaxLocal + 2; i++) { + G(p, i, j, k) = myrank; + G(f, i, j, k) = myrank; + G(g, i, j, k) = myrank; + G(h, i, j, k) = myrank; + } + } + } +} + +static char* direction2String(Direction dir) +{ + switch (dir) { + case LEFT: + return "left"; + break; + case RIGHT: + return "right"; + break; + case BOTTOM: + return "bottom"; + break; + case TOP: + return "top"; + break; + case FRONT: + return "front"; + break; + case BACK: + return "back"; + break; + case NDIRS: + return "ERROR"; + default: + return "ERROR"; + } +} + +static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + char filename[50]; + snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm.rank); + FILE* fh = fopen(filename, "w"); + + for (int y = 0; y < ymax; y++) { + for (int x = 0; x < xmax; x++) { + switch (dir) { + case LEFT: + fprintf(fh, "%12.8f ", G(a, 0, x, y)); + break; + case RIGHT: + fprintf(fh, "%12.8f ", G(a, imaxLocal + 1, x, y)); + break; + case BOTTOM: + fprintf(fh, "%12.8f ", G(a, x, 0, y)); + break; + case TOP: + fprintf(fh, "%12.8f ", G(a, x, jmaxLocal + 1, y)); + break; + case FRONT: + fprintf(fh, "%12.8f ", G(a, x, y, 0)); + break; + case BACK: + fprintf(fh, "%12.8f ", G(a, x, y, kmaxLocal + 1)); + break; + case NDIRS: + printf("ERROR\n"); + break; + } + } + fprintf(fh, "\n"); + } + fclose(fh); +} + +void testPrintHalo(Solver* s, double* a) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, BOTTOM); + printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, TOP); + printPlane(s, a, kmaxLocal + 2, jmaxLocal + 2, LEFT); + printPlane(s, a, kmaxLocal + 2, jmaxLocal + 2, RIGHT); + printPlane(s, a, jmaxLocal + 2, imaxLocal + 2, FRONT); + printPlane(s, a, jmaxLocal + 2, imaxLocal + 2, BACK); +} diff --git a/BasicSolver/3D-mpi/src/test.h b/BasicSolver/3D-mpi/src/test.h new file mode 100644 index 0000000..ecefe4e --- /dev/null +++ b/BasicSolver/3D-mpi/src/test.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __TEST_H_ +#define __TEST_H_ +#include "solver.h" + +extern void testInit(Solver* s); +extern void testPrintHalo(Solver* s, double* a); +#endif // __TEST_H_ diff --git a/BasicSolver/3D-mpi/src/timing.c b/BasicSolver/3D-mpi/src/timing.c new file mode 100644 index 0000000..c4025a4 --- /dev/null +++ b/BasicSolver/3D-mpi/src/timing.c @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution() +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/3D-mpi/src/timing.h b/BasicSolver/3D-mpi/src/timing.h new file mode 100644 index 0000000..db95329 --- /dev/null +++ b/BasicSolver/3D-mpi/src/timing.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(); +extern double getTimeResolution(); +extern double getTimeStamp_(); + +#endif // __TIMING_H_ diff --git a/BasicSolver/3D-mpi/src/util.h b/BasicSolver/3D-mpi/src/util.h new file mode 100644 index 0000000..657b009 --- /dev/null +++ b/BasicSolver/3D-mpi/src/util.h @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#endif // __UTIL_H_ diff --git a/BasicSolver/3D-mpi/src/vtkWriter.c b/BasicSolver/3D-mpi/src/vtkWriter.c new file mode 100644 index 0000000..5b4d36b --- /dev/null +++ b/BasicSolver/3D-mpi/src/vtkWriter.c @@ -0,0 +1,165 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "vtkWriter.h" +#define G(v, i, j, k) v[(k)*imax * jmax + (j)*imax + (i)] + +static float floatSwap(float f) +{ + union { + float f; + char b[4]; + } dat1, dat2; + + dat1.f = f; + dat2.b[0] = dat1.b[3]; + dat2.b[1] = dat1.b[2]; + dat2.b[2] = dat1.b[1]; + dat2.b[3] = dat1.b[0]; + return dat2.f; +} + +static void writeHeader(VtkOptions* o) +{ + fprintf(o->fh, "# vtk DataFile Version 3.0\n"); + fprintf(o->fh, "PAMPI cfd solver output\n"); + if (o->fmt == ASCII) { + fprintf(o->fh, "ASCII\n"); + } else if (o->fmt == BINARY) { + fprintf(o->fh, "BINARY\n"); + } + + fprintf(o->fh, "DATASET STRUCTURED_POINTS\n"); + fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid.imax, o->grid.jmax, o->grid.kmax); + fprintf(o->fh, + "ORIGIN %f %f %f\n", + o->grid.dx * 0.5, + o->grid.dy * 0.5, + o->grid.dz * 0.5); + fprintf(o->fh, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz); + fprintf(o->fh, "POINT_DATA %d\n", o->grid.imax * o->grid.jmax * o->grid.kmax); +} + +void vtkOpen(VtkOptions* o, char* problem) +{ + char filename[50]; + + if (o->mode == UNIX) { + if (commIsMaster(&o->comm)) { + snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size); + o->fh = fopen(filename, "w"); + writeHeader(o); + } + } else if (o->mode == MPI) { + } + + if (commIsMaster(&o->comm)) printf("Writing VTK output for %s\n", problem); +} + +static void writeScalar(VtkOptions* o, double* s) +{ + int imax = o->grid.imax; + int jmax = o->grid.jmax; + int kmax = o->grid.kmax; + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, "%f\n", G(s, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((float[1]) { floatSwap(G(s, i, j, k)) }, + sizeof(float), + 1, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +static bool isInitialized(FILE* ptr) +{ + if (ptr == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + return false; + } + return true; +} + +void vtkScalar(VtkOptions* o, char* name, double* s) +{ + if (commIsMaster(&o->comm)) printf("Register scalar %s\n", name); + + if (o->mode == UNIX) { + if (commIsMaster(&o->comm)) { + if (!isInitialized(o->fh)) return; + fprintf(o->fh, "SCALARS %s float 1\n", name); + fprintf(o->fh, "LOOKUP_TABLE default\n"); + writeScalar(o, s); + } + } else if (o->mode == MPI) { + } +} + +static void writeVector(VtkOptions* o, VtkVector vec) +{ + int imax = o->grid.imax; + int jmax = o->grid.jmax; + int kmax = o->grid.kmax; + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, + "%f %f %f\n", + G(vec.u, i, j, k), + G(vec.v, i, j, k), + G(vec.w, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((float[3]) { floatSwap(G(vec.u, i, j, k)), + floatSwap(G(vec.v, i, j, k)), + floatSwap(G(vec.w, i, j, k)) }, + sizeof(float), + 3, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +void vtkVector(VtkOptions* o, char* name, VtkVector vec) +{ + if (commIsMaster(&o->comm)) printf("Register vector %s\n", name); + + if (o->mode == UNIX) { + if (commIsMaster(&o->comm)) { + if (!isInitialized(o->fh)) return; + fprintf(o->fh, "VECTORS %s float\n", name); + writeVector(o, vec); + } + } else if (o->mode == MPI) { + } +} + +void vtkClose(VtkOptions* o) +{ + if (o->mode == UNIX) { + if (commIsMaster(&o->comm)) { + fclose(o->fh); + o->fh = NULL; + } + } else if (o->mode == MPI) { + } +} diff --git a/BasicSolver/3D-mpi/src/vtkWriter.h b/BasicSolver/3D-mpi/src/vtkWriter.h new file mode 100644 index 0000000..7a91dbb --- /dev/null +++ b/BasicSolver/3D-mpi/src/vtkWriter.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __VTKWRITER_H_ +#define __VTKWRITER_H_ +#include + +#include "comm.h" +#include "grid.h" + +typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat; +typedef enum VtkMode { UNIX = 0, MPI } VtkMode; + +typedef struct VtkOptions { + VtkFormat fmt; + VtkMode mode; + Grid grid; + FILE* fh; + Comm comm; +} VtkOptions; + +typedef struct VtkVector { + double *u, *v, *w; +} VtkVector; + +extern void vtkOpen(VtkOptions* opts, char* filename); +extern void vtkVector(VtkOptions* opts, char* name, VtkVector vec); +extern void vtkScalar(VtkOptions* opts, char* name, double* p); +extern void vtkClose(VtkOptions* opts); +#endif // __VTKWRITER_H_ diff --git a/BasicSolver/3D-seq/Makefile b/BasicSolver/3D-seq/Makefile new file mode 100644 index 0000000..57f99f4 --- /dev/null +++ b/BasicSolver/3D-seq/Makefile @@ -0,0 +1,71 @@ +#======================================================================================= +# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +SRC = $(wildcard $(SRC_DIR)/*.c) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm format + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +format: + @for src in $(SOURCES) ; do \ + echo "Formatting $$src" ; \ + clang-format -i $$src ; \ + done + @echo "Done" + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/BasicSolver/3D-seq/README.md b/BasicSolver/3D-seq/README.md new file mode 100644 index 0000000..d980b54 --- /dev/null +++ b/BasicSolver/3D-seq/README.md @@ -0,0 +1,78 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about solver, affinity settings, allocation sizes and timer resolution. +For debugging you may want to set the VERBOSE option: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +` + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. + +## Usage + +You have to provide a parameter file describing the problem you want to solve: +``` +./exe-CLANG dcavity.par +``` + +Examples are given in in dcavity (a lid driven cavity test case) and canal (simulating a empty canal). + +You can plot the resulting velocity and pressure fields using gnuplot: +``` +gnuplot vector.plot +``` +and for the pressure: +``` +gnuplot surface.plot +``` diff --git a/BasicSolver/3D-seq/canal.par b/BasicSolver/3D-seq/canal.par new file mode 100644 index 0000000..30e1f41 --- /dev/null +++ b/BasicSolver/3D-seq/canal.par @@ -0,0 +1,52 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 100.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +zlength 4.0 # domain size in z-direction +imax 200 # number of interior cells in x-direction +jmax 50 # number of interior cells in y-direction +kmax 50 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 100.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/3D-seq/config.mk b/BasicSolver/3D-seq/config.mk new file mode 100644 index 0000000..af5f1a0 --- /dev/null +++ b/BasicSolver/3D-seq/config.mk @@ -0,0 +1,12 @@ +# Supported: GCC, CLANG, ICC +TAG ?= CLANG +ENABLE_OPENMP ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DDEBUG +#OPTIONS += -DBOUNDCHECK +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/3D-seq/dcavity.par b/BasicSolver/3D-seq/dcavity.par new file mode 100644 index 0000000..6d1ef6d --- /dev/null +++ b/BasicSolver/3D-seq/dcavity.par @@ -0,0 +1,52 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcLeft 1 # flags for boundary conditions +bcRight 1 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 1000.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +zlength 1.0 # domain size in z-direction +imax 128 # number of interior cells in x-direction +jmax 128 # number of interior cells in y-direction +kmax 128 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 2.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma +#=============================================================================== diff --git a/BasicSolver/3D-seq/include_CLANG.mk b/BasicSolver/3D-seq/include_CLANG.mk new file mode 100644 index 0000000..17b26f3 --- /dev/null +++ b/BasicSolver/3D-seq/include_CLANG.mk @@ -0,0 +1,17 @@ +CC = clang +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +# CFLAGS = -O3 -std=c17 $(OPENMP) +CFLAGS = -Ofast -std=c17 -Weverything +#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG +LFLAGS = $(OPENMP) -lm +DEFINES = -D_GNU_SOURCE# -DDEBUG +INCLUDES = diff --git a/BasicSolver/3D-seq/include_GCC.mk b/BasicSolver/3D-seq/include_GCC.mk new file mode 100644 index 0000000..427e798 --- /dev/null +++ b/BasicSolver/3D-seq/include_GCC.mk @@ -0,0 +1,14 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/3D-seq/include_ICC.mk b/BasicSolver/3D-seq/include_ICC.mk new file mode 100644 index 0000000..94b8e20 --- /dev/null +++ b/BasicSolver/3D-seq/include_ICC.mk @@ -0,0 +1,14 @@ +CC = icc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/BasicSolver/3D-seq/src/allocate.c b/BasicSolver/3D-seq/src/allocate.c new file mode 100644 index 0000000..6af6c7f --- /dev/null +++ b/BasicSolver/3D-seq/src/allocate.c @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/BasicSolver/3D-seq/src/allocate.h b/BasicSolver/3D-seq/src/allocate.h new file mode 100644 index 0000000..1537eb2 --- /dev/null +++ b/BasicSolver/3D-seq/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate(size_t alignment, size_t bytesize); + +#endif diff --git a/BasicSolver/3D-seq/src/grid.h b/BasicSolver/3D-seq/src/grid.h new file mode 100644 index 0000000..c963429 --- /dev/null +++ b/BasicSolver/3D-seq/src/grid.h @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __GRID_H_ +#define __GRID_H_ + +typedef struct { + double dx, dy, dz; + int imax, jmax, kmax; + double xlength, ylength, zlength; +} Grid; + +#endif // __GRID_H_ diff --git a/BasicSolver/3D-seq/src/likwid-marker.h b/BasicSolver/3D-seq/src/likwid-marker.h new file mode 100644 index 0000000..c3770c0 --- /dev/null +++ b/BasicSolver/3D-seq/src/likwid-marker.h @@ -0,0 +1,54 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ + likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/BasicSolver/3D-seq/src/main.c b/BasicSolver/3D-seq/src/main.c new file mode 100644 index 0000000..4aed6bd --- /dev/null +++ b/BasicSolver/3D-seq/src/main.c @@ -0,0 +1,133 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "parameter.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" +#include "vtkWriter.h" + +#define G(v, i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +static void createBulkArrays(Solver* s, double* pg, double* ug, double* vg, double* wg) +{ + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + int idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + pg[idx++] = G(s->p, i, j, k); + } + } + } + + idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + ug[idx++] = (G(s->u, i, j, k) + G(s->u, i - 1, j, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + vg[idx++] = (G(s->v, i, j, k) + G(s->v, i, j - 1, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + wg[idx++] = (G(s->w, i, j, k) + G(s->w, i, j, k - 1)) / 2.0; + } + } + } +} + +int main(int argc, char** argv) +{ + double timeStart, timeStop; + Parameter params; + Solver solver; + initParameter(¶ms); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(¶ms, argv[1]); + printParameter(¶ms); + initSolver(&solver, ¶ms); +#ifndef VERBOSE + initProgress(solver.te); +#endif + + double tau = solver.tau; + double te = solver.te; + double t = 0.0; + int nt = 0; + + timeStart = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) computeTimestep(&solver); + setBoundaryConditions(&solver); + setSpecialBoundaryCondition(&solver); + computeFG(&solver); + computeRHS(&solver); + solve(&solver); + adaptUV(&solver); + t += solver.dt; + nt++; + +#ifdef VERBOSE + printf("TIME %f , TIMESTEP %f\n", t, solver.dt); +#else + printProgress(t); +#endif + } + timeStop = getTimeStamp(); +#ifndef VERBOSE + stopProgress(); +#endif + printf("Solution took %.2fs\n", timeStop - timeStart); + + double *pg, *ug, *vg, *wg; + + size_t bytesize = solver.grid.imax * solver.grid.jmax * solver.grid.kmax * + sizeof(double); + + pg = allocate(64, bytesize); + ug = allocate(64, bytesize); + vg = allocate(64, bytesize); + wg = allocate(64, bytesize); + + createBulkArrays(&s, pg, ug, vg, wg); + VtkOptions opts = { .grid = solver.grid }; + vtkOpen(&opts, solver.problem); + vtkScalar(&opts, "pressure", pg); + vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); + vtkClose(&opts); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/3D-seq/src/parameter.c b/BasicSolver/3D-seq/src/parameter.c new file mode 100644 index 0000000..d4ea5b6 --- /dev/null +++ b/BasicSolver/3D-seq/src/parameter.c @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter* param) +{ + param->xlength = 1.0; + param->ylength = 1.0; + param->zlength = 1.0; + param->imax = 100; + param->jmax = 100; + param->kmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; +} + +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if (!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while (!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; + line[i] = '\0'; + + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); + +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if (tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_REAL(zlength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(kmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + PARSE_REAL(re); + PARSE_REAL(tau); + PARSE_REAL(gamma); + PARSE_REAL(dt); + PARSE_REAL(te); + PARSE_REAL(gx); + PARSE_REAL(gy); + PARSE_REAL(gz); + PARSE_STRING(name); + PARSE_INT(bcLeft); + PARSE_INT(bcRight); + PARSE_INT(bcBottom); + PARSE_INT(bcTop); + PARSE_INT(bcFront); + PARSE_INT(bcBack); + PARSE_REAL(u_init); + PARSE_REAL(v_init); + PARSE_REAL(w_init); + PARSE_REAL(p_init); + } + } + + fclose(fp); +} + +void printParameter(Parameter* param) +{ + printf("Parameters for %s\n", param->name); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d Front:%d " + "Back:%d\n", + param->bcLeft, + param->bcRight, + param->bcBottom, + param->bcTop, + param->bcFront, + param->bcBack); + printf("\tReynolds number: %.2f\n", param->re); + printf("\tInit arrays: U:%.2f V:%.2f W:%.2f P:%.2f\n", + param->u_init, + param->v_init, + param->w_init, + param->p_init); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + param->xlength, + param->ylength, + param->zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", param->imax, param->jmax, param->kmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); + printf("\tTau factor: %.2f\n", param->tau); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", param->eps); + printf("\tgamma (stopping tolerance) : %f\n", param->gamma); + printf("\tomega (SOR relaxation): %f\n", param->omg); +} diff --git a/BasicSolver/3D-seq/src/parameter.h b/BasicSolver/3D-seq/src/parameter.h new file mode 100644 index 0000000..6dddf5f --- /dev/null +++ b/BasicSolver/3D-seq/src/parameter.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + int imax, jmax, kmax; + double xlength, ylength, zlength; + int itermax; + double eps, omg; + double re, tau, gamma; + double te, dt; + double gx, gy, gz; + char* name; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + double u_init, v_init, w_init, p_init; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/BasicSolver/3D-seq/src/progress.c b/BasicSolver/3D-seq/src/progress.c new file mode 100644 index 0000000..a9b82bd --- /dev/null +++ b/BasicSolver/3D-seq/src/progress.c @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; + +void initProgress(double end) +{ + _end = end; + _current = 0; + + printf("[ ]"); + fflush(stdout); +} + +void printProgress(double current) +{ + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); +} + +void stopProgress() +{ + printf("\n"); + fflush(stdout); +} diff --git a/BasicSolver/3D-seq/src/progress.h b/BasicSolver/3D-seq/src/progress.h new file mode 100644 index 0000000..9ef2d96 --- /dev/null +++ b/BasicSolver/3D-seq/src/progress.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __PROGRESS_H_ +#define __PROGRESS_H_ + +extern void initProgress(double); +extern void printProgress(double); +extern void stopProgress(); + +#endif diff --git a/BasicSolver/3D-seq/src/solver.c b/BasicSolver/3D-seq/src/solver.c new file mode 100644 index 0000000..bc3e39f --- /dev/null +++ b/BasicSolver/3D-seq/src/solver.c @@ -0,0 +1,722 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +#define P(i, j, k) p[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define F(i, j, k) f[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define G(i, j, k) g[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define H(i, j, k) h[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define U(i, j, k) u[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define V(i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define W(i, j, k) w[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define RHS(i, j, k) rhs[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +static void printConfig(Solver* s) +{ + printf("Parameters for #%s#\n", s->problem); + printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n", + s->bcLeft, + s->bcRight, + s->bcBottom, + s->bcTop, + s->bcFront, + s->bcBack); + printf("\tReynolds number: %.2f\n", s->re); + printf("\tGx Gy: %.2f %.2f %.2f\n", s->gx, s->gy, s->gz); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + s->grid.xlength, + s->grid.ylength, + s->grid.zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", s->grid.imax, s->grid.jmax, s->grid.kmax); + printf("\tCell size (dx, dy, dz): %f, %f, %f\n", s->grid.dx, s->grid.dy, s->grid.dz); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te); + printf("\tdt bound: %.6f\n", s->dtBound); + printf("\tTau factor: %.2f\n", s->tau); + printf("Iterative s parameters:\n"); + printf("\tMax iterations: %d\n", s->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", s->eps); + printf("\tgamma factor: %f\n", s->gamma); + printf("\tomega (SOR relaxation): %f\n", s->omega); +} + +void initSolver(Solver* s, Parameter* params) +{ + s->problem = params->name; + s->bcLeft = params->bcLeft; + s->bcRight = params->bcRight; + s->bcBottom = params->bcBottom; + s->bcTop = params->bcTop; + s->bcFront = params->bcFront; + s->bcBack = params->bcBack; + s->grid.imax = params->imax; + s->grid.jmax = params->jmax; + s->grid.kmax = params->kmax; + s->grid.xlength = params->xlength; + s->grid.ylength = params->ylength; + s->grid.zlength = params->zlength; + s->grid.dx = params->xlength / params->imax; + s->grid.dy = params->ylength / params->jmax; + s->grid.dz = params->zlength / params->kmax; + s->eps = params->eps; + s->omega = params->omg; + s->itermax = params->itermax; + s->re = params->re; + s->gx = params->gx; + s->gy = params->gy; + s->gz = params->gz; + s->dt = params->dt; + s->te = params->te; + s->tau = params->tau; + s->gamma = params->gamma; + + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + size_t bytesize = (imax + 2) * (jmax + 2) * (kmax + 2) * sizeof(double); + s->u = allocate(64, bytesize); + s->v = allocate(64, bytesize); + s->w = allocate(64, bytesize); + s->p = allocate(64, bytesize); + s->rhs = allocate(64, bytesize); + s->f = allocate(64, bytesize); + s->g = allocate(64, bytesize); + s->h = allocate(64, bytesize); + + for (int i = 0; i < (imax + 2) * (jmax + 2) * (kmax + 2); i++) { + s->u[i] = params->u_init; + s->v[i] = params->v_init; + s->w[i] = params->w_init; + s->p[i] = params->p_init; + s->rhs[i] = 0.0; + s->f[i] = 0.0; + s->g[i] = 0.0; + s->h[i] = 0.0; + } + + double dx = s->grid.dx; + double dy = s->grid.dy; + double dz = s->grid.dz; + + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz); + s->dtBound = 0.5 * s->re * 1.0 / invSqrSum; + +#ifdef VERBOSE + printConfig(s); +#endif /* VERBOSE */ +} + +void computeRHS(Solver* s) +{ + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + double idx = 1.0 / s->grid.dx; + double idy = 1.0 / s->grid.dy; + double idz = 1.0 / s->grid.dz; + double idt = 1.0 / s->dt; + double* rhs = s->rhs; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx + + (G(i, j, k) - G(i, j - 1, k)) * idy + + (H(i, j, k) - H(i, j, k - 1)) * idz) * + idt; + } + } + } +} + +void solve(Solver* s) +{ + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid.dx * s->grid.dx; + double dy2 = s->grid.dy * s->grid.dy; + double dz2 = s->grid.dz * s->grid.dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* p = s->p; + double* rhs = s->rhs; + double epssq = eps * eps; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + + double r = RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * + idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2); + + P(i, j, k) -= (factor * r); + res += (r * r); + } + } + } + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } + + res = res / (double)(imax * jmax * kmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif +} + +static double maxElement(Solver* s, double* m) +{ + int size = (s->grid.imax + 2) * (s->grid.jmax + 2) * (s->grid.kmax + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + return maxval; +} + +void normalizePressure(Solver* s) +{ + int size = (s->grid.imax + 2) * (s->grid.jmax + 2) * (s->grid.kmax + 2); + double* p = s->p; + double avgP = 0.0; + + for (int i = 0; i < size; i++) { + avgP += p[i]; + } + avgP /= size; + + for (int i = 0; i < size; i++) { + p[i] = p[i] - avgP; + } +} + +void computeTimestep(Solver* s) +{ + double dt = s->dtBound; + double dx = s->grid.dx; + double dy = s->grid.dy; + double dz = s->grid.dz; + double umax = maxElement(s, s->u); + double vmax = maxElement(s, s->v); + double wmax = maxElement(s, s->w); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + if (wmax > 0) { + dt = (dt > dz / wmax) ? dz / wmax : dt; + } + + s->dt = dt * s->tau; +} + +void setBoundaryConditions(Solver* s) +{ + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + + double* u = s->u; + double* v = s->v; + double* w = s->w; + + switch (s->bcTop) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, jmax, k) = 0.0; + U(i, jmax + 1, k) = -U(i, jmax, k); + W(i, jmax + 1, k) = -W(i, jmax, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, jmax, k) = 0.0; + U(i, jmax + 1, k) = U(i, jmax, k); + W(i, jmax + 1, k) = W(i, jmax, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + U(i, jmax + 1, k) = U(i, jmax, k); + V(i, jmax, k) = V(i, jmax - 1, k); + W(i, jmax + 1, k) = W(i, jmax, k); + } + } + break; + case PERIODIC: + break; + } + + switch (s->bcBottom) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, 0, k) = 0.0; + U(i, 0, k) = -U(i, 1, k); + W(i, 0, k) = -W(i, 1, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, 0, k) = 0.0; + U(i, 0, k) = U(i, 1, k); + W(i, 0, k) = W(i, 1, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = V(i, 1, k); + W(i, 0, k) = W(i, 1, k); + } + } + break; + case PERIODIC: + break; + } + + switch (s->bcLeft) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = -V(1, j, k); + W(0, j, k) = -W(1, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(0, j, k) = U(1, j, k); + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case PERIODIC: + break; + } + + switch (s->bcRight) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(imax, j, k) = 0.0; + V(imax + 1, j, k) = -V(imax, j, k); + W(imax + 1, j, k) = -W(imax, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(imax, j, k) = 0.0; + V(imax + 1, j, k) = V(imax, j, k); + W(imax + 1, j, k) = W(imax, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(imax, j, k) = U(imax - 1, j, k); + V(imax + 1, j, k) = V(imax, j, k); + W(imax + 1, j, k) = W(imax, j, k); + } + } + break; + case PERIODIC: + break; + } + + switch (s->bcFront) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, 0) = -U(i, j, 1); + V(i, j, 0) = -V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = W(i, j, 1); + } + } + break; + case PERIODIC: + break; + } + + switch (s->bcBack) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, kmax + 1) = -U(i, j, kmax); + V(i, j, kmax + 1) = -V(i, j, kmax); + W(i, j, kmax + 1) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, kmax + 1) = U(i, j, kmax); + V(i, j, kmax + 1) = V(i, j, kmax); + W(i, j, kmax + 1) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, kmax + 1) = U(i, j, kmax); + V(i, j, kmax + 1) = V(i, j, kmax); + W(i, j, kmax) = W(i, j, kmax - 1); + } + } + break; + case PERIODIC: + break; + } +} + +void setSpecialBoundaryCondition(Solver* s) +{ + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + + double mDy = s->grid.dy; + double* u = s->u; + + if (strcmp(s->problem, "dcavity") == 0) { + for (int k = 1; k < kmax; k++) { + for (int i = 1; i < imax; i++) { + U(i, jmax + 1, k) = 2.0 - U(i, jmax, k); + } + } + } else if (strcmp(s->problem, "canal") == 0) { + double ylength = s->grid.ylength; + double y; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + y = mDy * (j - 0.5); + U(0, j, k) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } + } +} + +void computeFG(Solver* s) +{ + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + + double* u = s->u; + double* v = s->v; + double* w = s->w; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + double gx = s->gx; + double gy = s->gy; + double gz = s->gz; + double gamma = s->gamma; + double dt = s->dt; + + double inverseRe = 1.0 / s->re; + double inverseDx = 1.0 / s->grid.dx; + double inverseDy = 1.0 / s->grid.dy; + double inverseDz = 1.0 / s->grid.dz; + double du2dx, dv2dy, dw2dz; + double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz; + double du2dx2, du2dy2, du2dz2; + double dv2dx2, dv2dy2, dv2dz2; + double dw2dx2, dw2dy2, dw2dz2; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + du2dx = inverseDx * 0.25 * + ((U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) + U(i + 1, j, k)) - + (U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) + U(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) - U(i + 1, j, k)) + + fabs(U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) - U(i - 1, j, k))); + + duvdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) + U(i, j + 1, k)) - + (V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) + U(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) - U(i, j + 1, k)) + + fabs(V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) - U(i, j - 1, k))); + + duwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) + U(i, j, k + 1)) - + (W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) + U(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) - U(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) - U(i, j, k - 1))); + + du2dx2 = inverseDx * inverseDx * + (U(i + 1, j, k) - 2.0 * U(i, j, k) + U(i - 1, j, k)); + du2dy2 = inverseDy * inverseDy * + (U(i, j + 1, k) - 2.0 * U(i, j, k) + U(i, j - 1, k)); + du2dz2 = inverseDz * inverseDz * + (U(i, j, k + 1) - 2.0 * U(i, j, k) + U(i, j, k - 1)); + F(i, j, k) = U(i, j, k) + dt * (inverseRe * (du2dx2 + du2dy2 + du2dz2) - + du2dx - duvdy - duwdz + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) + V(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) + V(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) - V(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) - V(i - 1, j, k))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) + V(i, j + 1, k)) - + (V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) + V(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) - V(i, j + 1, k)) + + fabs(V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) - V(i, j - 1, k))); + + dvwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) + V(i, j, k + 1)) - + (W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) + V(i, j, k + 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) - V(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) - V(i, j, k + 1))); + + dv2dx2 = inverseDx * inverseDx * + (V(i + 1, j, k) - 2.0 * V(i, j, k) + V(i - 1, j, k)); + dv2dy2 = inverseDy * inverseDy * + (V(i, j + 1, k) - 2.0 * V(i, j, k) + V(i, j - 1, k)); + dv2dz2 = inverseDz * inverseDz * + (V(i, j, k + 1) - 2.0 * V(i, j, k) + V(i, j, k - 1)); + G(i, j, k) = V(i, j, k) + dt * (inverseRe * (dv2dx2 + dv2dy2 + dv2dz2) - + duvdx - dv2dy - dvwdz + gy); + + duwdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) + W(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) + W(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) - W(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) - W(i - 1, j, k))); + + dvwdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) + W(i, j + 1, k)) - + (V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) + W(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) - W(i, j + 1, k)) + + fabs(V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) - W(i, j - 1, k))); + + dw2dz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) + W(i, j, k + 1)) - + (W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) + W(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) - W(i, j, k + 1)) + + fabs(W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) - W(i, j, k - 1))); + + dw2dx2 = inverseDx * inverseDx * + (W(i + 1, j, k) - 2.0 * W(i, j, k) + W(i - 1, j, k)); + dw2dy2 = inverseDy * inverseDy * + (W(i, j + 1, k) - 2.0 * W(i, j, k) + W(i, j - 1, k)); + dw2dz2 = inverseDz * inverseDz * + (W(i, j, k + 1) - 2.0 * W(i, j, k) + W(i, j, k - 1)); + H(i, j, k) = W(i, j, k) + dt * (inverseRe * (dw2dx2 + dw2dy2 + dw2dz2) - + duwdx - dvwdy - dw2dz + gz); + } + } + } + + /* ----------------------------- boundary of F --------------------------- + */ + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + F(0, j, k) = U(0, j, k); + F(imax, j, k) = U(imax, j, k); + } + } + + /* ----------------------------- boundary of G --------------------------- + */ + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + G(i, 0, k) = V(i, 0, k); + G(i, jmax, k) = V(i, jmax, k); + } + } + + /* ----------------------------- boundary of G --------------------------- + */ + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + H(i, j, 0) = W(i, j, 0); + H(i, j, kmax) = W(i, j, kmax); + } + } +} + +void adaptUV(Solver* s) +{ + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + + double* p = s->p; + double* u = s->u; + double* v = s->v; + double* w = s->w; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + double factorX = s->dt / s->grid.dx; + double factorY = s->dt / s->grid.dy; + double factorZ = s->dt / s->grid.dz; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, k) = F(i, j, k) - (P(i + 1, j, k) - P(i, j, k)) * factorX; + V(i, j, k) = G(i, j, k) - (P(i, j + 1, k) - P(i, j, k)) * factorY; + W(i, j, k) = H(i, j, k) - (P(i, j, k + 1) - P(i, j, k)) * factorZ; + } + } + } +} diff --git a/BasicSolver/3D-seq/src/solver.h b/BasicSolver/3D-seq/src/solver.h new file mode 100644 index 0000000..63505d0 --- /dev/null +++ b/BasicSolver/3D-seq/src/solver.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ + +#include "grid.h" +#include "parameter.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g, *h; + double *u, *v, *w; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy, gz; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; +} Solver; + +extern void initSolver(Solver*, Parameter*); +extern void computeRHS(Solver*); +extern void solve(Solver*); +extern void normalizePressure(Solver*); +extern void computeTimestep(Solver*); +extern void setBoundaryConditions(Solver*); +extern void setSpecialBoundaryCondition(Solver*); +extern void computeFG(Solver*); +extern void adaptUV(Solver*); +extern void writeResult(Solver*); +#endif diff --git a/BasicSolver/3D-seq/src/timing.c b/BasicSolver/3D-seq/src/timing.c new file mode 100644 index 0000000..e578f18 --- /dev/null +++ b/BasicSolver/3D-seq/src/timing.c @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include + +double getTimeStamp(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution(void) +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} diff --git a/BasicSolver/3D-seq/src/timing.h b/BasicSolver/3D-seq/src/timing.h new file mode 100644 index 0000000..58fb5ac --- /dev/null +++ b/BasicSolver/3D-seq/src/timing.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(void); +extern double getTimeResolution(void); + +#endif // __TIMING_H_ diff --git a/BasicSolver/3D-seq/src/util.h b/BasicSolver/3D-seq/src/util.h new file mode 100644 index 0000000..657b009 --- /dev/null +++ b/BasicSolver/3D-seq/src/util.h @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#endif // __UTIL_H_ diff --git a/BasicSolver/3D-seq/src/vtkWriter.c b/BasicSolver/3D-seq/src/vtkWriter.c new file mode 100644 index 0000000..2f7e8c5 --- /dev/null +++ b/BasicSolver/3D-seq/src/vtkWriter.c @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "vtkWriter.h" +#define G(v, i, j, k) v[(k)*imax * jmax + (j)*imax + (i)] + +static float floatSwap(float f) +{ + union { + float f; + char b[4]; + } dat1, dat2; + + dat1.f = f; + dat2.b[0] = dat1.b[3]; + dat2.b[1] = dat1.b[2]; + dat2.b[2] = dat1.b[1]; + dat2.b[3] = dat1.b[0]; + return dat2.f; +} + +static void writeHeader(VtkOptions* o) +{ + fprintf(o->fh, "# vtk DataFile Version 3.0\n"); + fprintf(o->fh, "PAMPI cfd solver output\n"); + if (o->fmt == ASCII) { + fprintf(o->fh, "ASCII\n"); + } else if (o->fmt == BINARY) { + fprintf(o->fh, "BINARY\n"); + } + + fprintf(o->fh, "DATASET STRUCTURED_POINTS\n"); + fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid.imax, o->grid.jmax, o->grid.kmax); + fprintf(o->fh, + "ORIGIN %f %f %f\n", + o->grid.dx * 0.5, + o->grid.dy * 0.5, + o->grid.dz * 0.5); + fprintf(o->fh, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz); + fprintf(o->fh, "POINT_DATA %d\n", o->grid.imax * o->grid.jmax * o->grid.kmax); +} + +void vtkOpen(VtkOptions* o, char* problem) +{ + char filename[50]; + snprintf(filename, 50, "%s.vtk", problem); + o->fh = fopen(filename, "w"); + writeHeader(o); + + printf("Writing VTK output for %s\n", problem); +} + +void vtkScalar(VtkOptions* o, char* name, double* s) +{ + int imax = o->grid.imax; + int jmax = o->grid.jmax; + int kmax = o->grid.kmax; + + printf("Register scalar %s\n", name); + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + fprintf(o->fh, "SCALARS %s float 1\n", name); + fprintf(o->fh, "LOOKUP_TABLE default\n"); + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, "%f\n", G(s, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((float[1]) { floatSwap(G(s, i, j, k)) }, + sizeof(float), + 1, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +void vtkVector(VtkOptions* o, char* name, VtkVector vec) +{ + int imax = o->grid.imax; + int jmax = o->grid.jmax; + int kmax = o->grid.kmax; + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + + fprintf(o->fh, "VECTORS %s float\n", name); + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, + "%f %f %f\n", + G(vec.u, i, j, k), + G(vec.v, i, j, k), + G(vec.w, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((float[3]) { floatSwap(G(vec.u, i, j, k)), + floatSwap(G(vec.v, i, j, k)), + floatSwap(G(vec.w, i, j, k)) }, + sizeof(float), + 3, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +void vtkClose(VtkOptions* o) +{ + fclose(o->fh); + o->fh = NULL; +} diff --git a/BasicSolver/3D-seq/src/vtkWriter.h b/BasicSolver/3D-seq/src/vtkWriter.h new file mode 100644 index 0000000..d166f62 --- /dev/null +++ b/BasicSolver/3D-seq/src/vtkWriter.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __VTKWRITER_H_ +#define __VTKWRITER_H_ +#include + +#include "grid.h" + +typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat; + +typedef struct VtkOptions { + VtkFormat fmt; + Grid grid; + FILE* fh; +} VtkOptions; + +typedef struct VtkVector { + double *u, *v, *w; +} VtkVector; + +extern void vtkOpen(VtkOptions* opts, char* filename); +extern void vtkVector(VtkOptions* opts, char* name, VtkVector vec); +extern void vtkScalar(VtkOptions* opts, char* name, double* p); +extern void vtkClose(VtkOptions* opts); +#endif // __VTKWRITER_H_ diff --git a/PoissonSolver/2D-mpi/Makefile b/PoissonSolver/2D-mpi/Makefile new file mode 100644 index 0000000..53fcb50 --- /dev/null +++ b/PoissonSolver/2D-mpi/Makefile @@ -0,0 +1,62 @@ +#======================================================================================= +# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR)/includes -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s,$(wildcard $(SRC_DIR)/*.c)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o,$(wildcard $(SRC_DIR)/*.c)) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/PoissonSolver/2D-mpi/README.md b/PoissonSolver/2D-mpi/README.md new file mode 100644 index 0000000..b0a80a6 --- /dev/null +++ b/PoissonSolver/2D-mpi/README.md @@ -0,0 +1,48 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution. + + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. diff --git a/PoissonSolver/2D-mpi/config.mk b/PoissonSolver/2D-mpi/config.mk new file mode 100644 index 0000000..2abcf13 --- /dev/null +++ b/PoissonSolver/2D-mpi/config.mk @@ -0,0 +1,9 @@ +# Supported: GCC, CLANG, ICC +TAG ?= CLANG +ENABLE_OPENMP ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/PoissonSolver/2D-mpi/include_CLANG.mk b/PoissonSolver/2D-mpi/include_CLANG.mk new file mode 100644 index 0000000..1d17c52 --- /dev/null +++ b/PoissonSolver/2D-mpi/include_CLANG.mk @@ -0,0 +1,16 @@ +CC = mpicc +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +CFLAGS = -Ofast -std=c99 $(OPENMP) +#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE# -DDEBUG +INCLUDES = -I/usr/local/include diff --git a/PoissonSolver/2D-mpi/include_GCC.mk b/PoissonSolver/2D-mpi/include_GCC.mk new file mode 100644 index 0000000..427e798 --- /dev/null +++ b/PoissonSolver/2D-mpi/include_GCC.mk @@ -0,0 +1,14 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/PoissonSolver/2D-mpi/include_ICC.mk b/PoissonSolver/2D-mpi/include_ICC.mk new file mode 100644 index 0000000..f85d836 --- /dev/null +++ b/PoissonSolver/2D-mpi/include_ICC.mk @@ -0,0 +1,14 @@ +CC = mpiicc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/PoissonSolver/2D-mpi/poisson.par b/PoissonSolver/2D-mpi/poisson.par new file mode 100644 index 0000000..1f23554 --- /dev/null +++ b/PoissonSolver/2D-mpi/poisson.par @@ -0,0 +1,21 @@ +# Problem specific Data: +# --------------------- + +name poisson + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 200 # number of interior cells in y-direction + +# Pressure Iteration Data: +# ----------------------- + +itermax 10000 # maximal number of pressure iteration in one time step +eps 0.000001 # stopping tolerance for pressure iteration +omg 1.9 # relaxation parameter for SOR iteration + +#=============================================================================== diff --git a/PoissonSolver/2D-mpi/src/affinity.c b/PoissonSolver/2D-mpi/src/affinity.c new file mode 100644 index 0000000..895aff8 --- /dev/null +++ b/PoissonSolver/2D-mpi/src/affinity.c @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifdef __linux__ +#ifdef _OPENMP +#include +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_THREADS 128 +#define gettid() syscall(SYS_gettid) + +static int getProcessorID(cpu_set_t* cpu_set) +{ + int processorId; + + for ( processorId = 0; processorId < MAX_NUM_THREADS; processorId++ ) + { + if ( CPU_ISSET(processorId,cpu_set) ) + { + break; + } + } + return processorId; +} + +int affinity_getProcessorId() +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + sched_getaffinity(gettid(),sizeof(cpu_set_t), &cpu_set); + + return getProcessorID(&cpu_set); +} + +void affinity_pinThread(int processorId) +{ + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); +} + +void affinity_pinProcess(int processorId) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} +#endif /*_OPENMP*/ +#endif /*__linux__*/ diff --git a/PoissonSolver/2D-mpi/src/affinity.h b/PoissonSolver/2D-mpi/src/affinity.h new file mode 100644 index 0000000..d844fe5 --- /dev/null +++ b/PoissonSolver/2D-mpi/src/affinity.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef AFFINITY_H +#define AFFINITY_H + +extern int affinity_getProcessorId(); +extern void affinity_pinProcess(int); +extern void affinity_pinThread(int); + +#endif /*AFFINITY_H*/ diff --git a/PoissonSolver/2D-mpi/src/allocate.c b/PoissonSolver/2D-mpi/src/allocate.c new file mode 100644 index 0000000..513a693 --- /dev/null +++ b/PoissonSolver/2D-mpi/src/allocate.c @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +void* allocate (int alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, + "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, + "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/PoissonSolver/2D-mpi/src/allocate.h b/PoissonSolver/2D-mpi/src/allocate.h new file mode 100644 index 0000000..4b5575e --- /dev/null +++ b/PoissonSolver/2D-mpi/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate (int alignment, size_t bytesize); + +#endif diff --git a/PoissonSolver/2D-mpi/src/likwid-marker.h b/PoissonSolver/2D-mpi/src/likwid-marker.h new file mode 100644 index 0000000..a35b495 --- /dev/null +++ b/PoissonSolver/2D-mpi/src/likwid-marker.h @@ -0,0 +1,53 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/PoissonSolver/2D-mpi/src/main.c b/PoissonSolver/2D-mpi/src/main.c new file mode 100644 index 0000000..c797be3 --- /dev/null +++ b/PoissonSolver/2D-mpi/src/main.c @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include +#include + +#include "parameter.h" +#include "solver.h" + + +int main (int argc, char** argv) +{ + int rank; + Parameter params; + Solver solver; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + initParameter(¶ms); + + if ( argc != 2 ) { + printf("Usage: %s \n",argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(¶ms, argv[1]); + if ( rank == 0 ) { + printParameter(¶ms); + } + + initSolver(&solver, ¶ms, 2); + solve(&solver); + getResult(&solver); + + MPI_Finalize(); + return EXIT_SUCCESS; +} diff --git a/PoissonSolver/2D-mpi/src/parameter.c b/PoissonSolver/2D-mpi/src/parameter.c new file mode 100644 index 0000000..6c08af9 --- /dev/null +++ b/PoissonSolver/2D-mpi/src/parameter.c @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +//--- +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter *param) { + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.8; +} + +void readParameter(Parameter *param, const char *filename) { + FILE *fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if(!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while(!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for(i = 0; line[i] != '\0' && line[i] != '#'; i++); + line[i] = '\0'; + + char *tok = strtok(line, " "); + char *val = strtok(NULL, " "); + + #define PARSE_PARAM(p,f) if(strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { param->p = f(val); } + #define PARSE_STRING(p) PARSE_PARAM(p, strdup) + #define PARSE_INT(p) PARSE_PARAM(p, atoi) + #define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if(tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + } + } + + fclose(fp); +} + +void printParameter(Parameter *param) { + printf("Parameters:\n"); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %e, %e\n", param->xlength, param->ylength); + printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %e\n", param->eps); + printf("\tomega (SOR relaxation): %e\n", param->omg); +} diff --git a/PoissonSolver/2D-mpi/src/parameter.h b/PoissonSolver/2D-mpi/src/parameter.h new file mode 100644 index 0000000..d57ce19 --- /dev/null +++ b/PoissonSolver/2D-mpi/src/parameter.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + double xlength, ylength; + int imax, jmax; + int itermax; + double eps, omg, gamma; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/PoissonSolver/2D-mpi/src/solver.c b/PoissonSolver/2D-mpi/src/solver.c new file mode 100644 index 0000000..7264c44 --- /dev/null +++ b/PoissonSolver/2D-mpi/src/solver.c @@ -0,0 +1,278 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "solver.h" +#include "parameter.h" +#include "allocate.h" + +#define PI 3.14159265358979323846 +#define P(i,j) p[(j)*(imax+2) + (i)] +#define RHS(i,j) rhs[(j)*(imax+2) + (i)] + +static int sizeOfRank(int rank, int size, int N) +{ + return N/size + ((N%size>rank) ? 1 : 0); +} + +static void print(Solver* solver) +{ + double* p = solver->p; + int imax = solver->imax; + + printf("### RANK %d #######################################################\n", solver->rank); + for( int j=0; j < solver->jmaxLocal+2; j++ ) { + printf("%02d: ", j); + for( int i=0; i < solver->imax+2; i++ ) { + printf("%12.8f ", P(i, j)); + } + printf("\n"); + } + fflush( stdout ); +} + +static void exchange(Solver* solver) +{ + MPI_Request requests[4] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL, MPI_REQUEST_NULL, MPI_REQUEST_NULL }; + + /* exchange ghost cells with top neighbor */ + if (solver->rank + 1 < solver->size) { + int top = solver->rank + 1; + double* src = solver->p + (solver->jmaxLocal) * (solver->imax+2) + 1; + double* dst = solver->p + (solver->jmaxLocal+1) * (solver->imax+2) + 1; + + MPI_Isend(src, solver->imax, MPI_DOUBLE, top, 1, MPI_COMM_WORLD, &requests[0]); + MPI_Irecv(dst, solver->imax, MPI_DOUBLE, top, 2, MPI_COMM_WORLD, &requests[1]); + } + + /* exchange ghost cells with bottom neighbor */ + if (solver->rank > 0) { + int bottom = solver->rank - 1; + double* src = solver->p + (solver->imax+2) + 1; + double* dst = solver->p + 1; + + MPI_Isend(src, solver->imax, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]); + MPI_Irecv(dst, solver->imax, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]); + } + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +} + +void getResult(Solver *solver) +{ + double* Pall = NULL; + int *rcvCounts, *displs; + + if ( solver->rank == 0 ) { + Pall = allocate(64, (solver->imax+2) * (solver->jmax+2) * sizeof(double)); + rcvCounts = (int*) malloc(solver->size * sizeof(int)); + displs = (int*) malloc(solver->size * sizeof(int)); + rcvCounts[0] = solver->jmaxLocal * (solver->imax+2); + displs[0] = 0; + int cursor = rcvCounts[0]; + + for ( int i=1; i < solver->size; i++ ) { + rcvCounts[i] = sizeOfRank(i, solver->size, solver->jmax) * (solver->imax+2); + displs[i] = cursor; + cursor += rcvCounts[i]; + } + } + + int cnt = solver->jmaxLocal*(solver->imax+2); + double* sendbuffer = solver->p + (solver->imax+2); + MPI_Gatherv(sendbuffer, cnt, MPI_DOUBLE, Pall, + rcvCounts, displs, MPI_DOUBLE, 0, MPI_COMM_WORLD); + + if ( solver->rank == 0 ) { + writeResult(solver, Pall, "p.dat"); + } +} + +void initSolver(Solver *solver, Parameter *params, int problem) +{ + MPI_Comm_rank(MPI_COMM_WORLD, &(solver->rank)); + MPI_Comm_size(MPI_COMM_WORLD, &(solver->size)); + solver->imax = params->imax; + solver->jmax = params->jmax; + solver->jmaxLocal = sizeOfRank(solver->rank, solver->size, solver->jmax); + printf("RANK %d: %d\n", solver->rank, solver->jmaxLocal); + + solver->dx = params->xlength/params->imax; + solver->dy = params->ylength/params->jmax; + solver->ys = solver->rank * solver->jmaxLocal * solver->dy; + solver->eps = params->eps; + solver->omega = params->omg; + solver->itermax = params->itermax; + + int imax = solver->imax; + int jmax = solver->jmax; + int jmaxLocal = solver->jmaxLocal; + solver->p = allocate(64, (imax+2) * (jmaxLocal+2) * sizeof(double)); + solver->rhs = allocate(64, (imax+2) * (jmax+2) * sizeof(double)); + + double dx = solver->dx; + double dy = solver->dy; + double* p = solver->p; + double* rhs = solver->rhs; + + for( int j=0; jys + j * dy; + for( int i=0; iimax; + int rank = solver->rank; + double* p = solver->p; + +/* for( int j=0; j < solver->jmaxLocal+2; j++ ) { */ +/* for( int i=0; i < solver->imax+2; i++ ) { */ +/* P(i, j) = (double) rank; */ +/* } */ +/* } */ + +/* for ( int i=0; i < solver->size; i++) { */ +/* if ( i == rank ) { */ +/* print(solver); */ +/* } */ +/* MPI_Barrier(MPI_COMM_WORLD); */ +/* } */ + +/* if ( rank == 0 ) { */ +/* printf("##########################################################\n"); */ +/* printf("## Exchange ghost layers\n"); */ +/* printf("##########################################################\n"); */ +/* } */ +/* exchange(solver); */ + + for ( int i=0; i < solver->size; i++) { + if ( i == rank ) { + print(solver); + } + MPI_Barrier(MPI_COMM_WORLD); + } +} + +int solve(Solver *solver) +{ + double r; + int it = 0; + double res; + + int imax = solver->imax; + int jmax = solver->jmax; + int jmaxLocal = solver->jmaxLocal; + double eps= solver->eps; + double omega = solver->omega; + int itermax = solver->itermax; + + double dx2 = solver->dx * solver->dx; + double dy2 = solver->dy * solver->dy; + double idx2 = 1.0/dx2; + double idy2 = 1.0/dy2; + double factor = omega * 0.5 * (dx2*dy2) / (dx2+dy2); + double* p = solver->p; + double* rhs = solver->rhs; + + res = eps + 1.0; + + while((res >= eps) && (it < itermax)) { + res = 0.0; + exchange(solver); + + for( int j=1; jrank == 0 ) { + for( int i=1; irank == (solver->size-1) ) { + for( int i=1; irank == 0 ) { + printf("%d Residuum: %e\n",it, res); + } +#endif + it++; + } + + if ( solver->rank == 0 ) { + printf("Solver took %d iterations\n",it); + } + if( res < eps ){ + return 1; + } else{ + return 0; + } +} + +void writeResult(Solver* solver, double* m, char* filename) +{ + int imax = solver->imax; + int jmax = solver->jmax; + double* p = solver->p; + + FILE *fp; + fp= fopen(filename, "w"); + + if (fp== NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for( int j=0; j +#include + +double getTimeStamp() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution() +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeStamp_() +{ + return getTimeStamp(); +} diff --git a/PoissonSolver/2D-mpi/src/timing.h b/PoissonSolver/2D-mpi/src/timing.h new file mode 100644 index 0000000..db95329 --- /dev/null +++ b/PoissonSolver/2D-mpi/src/timing.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(); +extern double getTimeResolution(); +extern double getTimeStamp_(); + +#endif // __TIMING_H_ diff --git a/PoissonSolver/2D-mpi/src/util.h b/PoissonSolver/2D-mpi/src/util.h new file mode 100644 index 0000000..fe275cb --- /dev/null +++ b/PoissonSolver/2D-mpi/src/util.h @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x,y) ((x)<(y)?(x):(y)) +#endif +#ifndef MAX +#define MAX(x,y) ((x)>(y)?(x):(y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#endif // __UTIL_H_ diff --git a/PoissonSolver/2D-mpi/surface.plot b/PoissonSolver/2D-mpi/surface.plot new file mode 100644 index 0000000..7d56c7d --- /dev/null +++ b/PoissonSolver/2D-mpi/surface.plot @@ -0,0 +1,7 @@ +set terminal png size 1024,768 enhanced font ,12 +set output 'p.png' +set datafile separator whitespace + +set grid +set hidden3d +splot 'p.dat' matrix using 1:2:3 with lines diff --git a/PoissonSolver/2D-seq/Makefile b/PoissonSolver/2D-seq/Makefile new file mode 100644 index 0000000..53fcb50 --- /dev/null +++ b/PoissonSolver/2D-seq/Makefile @@ -0,0 +1,62 @@ +#======================================================================================= +# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# All rights reserved. +# Use of this source code is governed by a MIT-style +# license that can be found in the LICENSE file. +#======================================================================================= + +#CONFIGURE BUILD SYSTEM +TARGET = exe-$(TAG) +BUILD_DIR = ./$(TAG) +SRC_DIR = ./src +MAKE_DIR = ./ +Q ?= @ + +#DO NOT EDIT BELOW +include $(MAKE_DIR)/config.mk +include $(MAKE_DIR)/include_$(TAG).mk +INCLUDES += -I$(SRC_DIR)/includes -I$(BUILD_DIR) + +VPATH = $(SRC_DIR) +ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s,$(wildcard $(SRC_DIR)/*.c)) +OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o,$(wildcard $(SRC_DIR)/*.c)) +CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) + +${TARGET}: $(BUILD_DIR) $(OBJ) + $(info ===> LINKING $(TARGET)) + $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) + +$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk + $(info ===> COMPILE $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d + +$(BUILD_DIR)/%.s: %.c + $(info ===> GENERATE ASM $@) + $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ + +.PHONY: clean distclean tags info asm + +clean: + $(info ===> CLEAN) + @rm -rf $(BUILD_DIR) + @rm -f tags + +distclean: clean + $(info ===> DIST CLEAN) + @rm -f $(TARGET) + +info: + $(info $(CFLAGS)) + $(Q)$(CC) $(VERSION) + +asm: $(BUILD_DIR) $(ASM) + +tags: + $(info ===> GENERATE TAGS) + $(Q)ctags -R + +$(BUILD_DIR): + @mkdir $(BUILD_DIR) + +-include $(OBJ:.o=.d) diff --git a/PoissonSolver/2D-seq/README.md b/PoissonSolver/2D-seq/README.md new file mode 100644 index 0000000..b0a80a6 --- /dev/null +++ b/PoissonSolver/2D-seq/README.md @@ -0,0 +1,48 @@ +# C source skeleton + +## Build + +1. Configure the toolchain and additional options in `config.mk`: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false + +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +``` + +The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution. + + +2. Build with: +``` +make +``` + +You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +Intermediate build results are located in the `` directory. + +To output the executed commands use: +``` +make Q= +``` + +3. Clean up with: +``` +make clean +``` +to clean intermediate build results. + +``` +make distclean +``` +to clean intermediate build results and binary. + +4. (Optional) Generate assembler: +``` +make asm +``` +The assembler files will also be located in the `` directory. diff --git a/PoissonSolver/2D-seq/config.mk b/PoissonSolver/2D-seq/config.mk new file mode 100644 index 0000000..2abcf13 --- /dev/null +++ b/PoissonSolver/2D-seq/config.mk @@ -0,0 +1,9 @@ +# Supported: GCC, CLANG, ICC +TAG ?= CLANG +ENABLE_OPENMP ?= false + +#Feature options +OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER diff --git a/PoissonSolver/2D-seq/include_CLANG.mk b/PoissonSolver/2D-seq/include_CLANG.mk new file mode 100644 index 0000000..d3995f7 --- /dev/null +++ b/PoissonSolver/2D-seq/include_CLANG.mk @@ -0,0 +1,16 @@ +CC = clang +GCC = cc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif + +VERSION = --version +CFLAGS = -Ofast -std=c99 $(OPENMP) +#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG +LFLAGS = $(OPENMP) -lm +DEFINES = -D_GNU_SOURCE# -DDEBUG +INCLUDES = diff --git a/PoissonSolver/2D-seq/include_GCC.mk b/PoissonSolver/2D-seq/include_GCC.mk new file mode 100644 index 0000000..427e798 --- /dev/null +++ b/PoissonSolver/2D-seq/include_GCC.mk @@ -0,0 +1,14 @@ +CC = gcc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -fopenmp +endif + +VERSION = --version +CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/PoissonSolver/2D-seq/include_ICC.mk b/PoissonSolver/2D-seq/include_ICC.mk new file mode 100644 index 0000000..94b8e20 --- /dev/null +++ b/PoissonSolver/2D-seq/include_ICC.mk @@ -0,0 +1,14 @@ +CC = icc +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) +LFLAGS = $(OPENMP) +DEFINES = -D_GNU_SOURCE +INCLUDES = +LIBS = diff --git a/PoissonSolver/2D-seq/poisson.par b/PoissonSolver/2D-seq/poisson.par new file mode 100644 index 0000000..1f23554 --- /dev/null +++ b/PoissonSolver/2D-seq/poisson.par @@ -0,0 +1,21 @@ +# Problem specific Data: +# --------------------- + +name poisson + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 200 # number of interior cells in y-direction + +# Pressure Iteration Data: +# ----------------------- + +itermax 10000 # maximal number of pressure iteration in one time step +eps 0.000001 # stopping tolerance for pressure iteration +omg 1.9 # relaxation parameter for SOR iteration + +#=============================================================================== diff --git a/PoissonSolver/2D-seq/src/affinity.c b/PoissonSolver/2D-seq/src/affinity.c new file mode 100644 index 0000000..895aff8 --- /dev/null +++ b/PoissonSolver/2D-seq/src/affinity.c @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifdef __linux__ +#ifdef _OPENMP +#include +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_THREADS 128 +#define gettid() syscall(SYS_gettid) + +static int getProcessorID(cpu_set_t* cpu_set) +{ + int processorId; + + for ( processorId = 0; processorId < MAX_NUM_THREADS; processorId++ ) + { + if ( CPU_ISSET(processorId,cpu_set) ) + { + break; + } + } + return processorId; +} + +int affinity_getProcessorId() +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + sched_getaffinity(gettid(),sizeof(cpu_set_t), &cpu_set); + + return getProcessorID(&cpu_set); +} + +void affinity_pinThread(int processorId) +{ + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); +} + +void affinity_pinProcess(int processorId) +{ + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(processorId, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} +#endif /*_OPENMP*/ +#endif /*__linux__*/ diff --git a/PoissonSolver/2D-seq/src/affinity.h b/PoissonSolver/2D-seq/src/affinity.h new file mode 100644 index 0000000..d844fe5 --- /dev/null +++ b/PoissonSolver/2D-seq/src/affinity.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef AFFINITY_H +#define AFFINITY_H + +extern int affinity_getProcessorId(); +extern void affinity_pinProcess(int); +extern void affinity_pinThread(int); + +#endif /*AFFINITY_H*/ diff --git a/PoissonSolver/2D-seq/src/allocate.c b/PoissonSolver/2D-seq/src/allocate.c new file mode 100644 index 0000000..513a693 --- /dev/null +++ b/PoissonSolver/2D-seq/src/allocate.c @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +void* allocate (int alignment, size_t bytesize) +{ + int errorCode; + void* ptr; + + errorCode = posix_memalign(&ptr, alignment, bytesize); + + if (errorCode) { + if (errorCode == EINVAL) { + fprintf(stderr, + "Error: Alignment parameter is not a power of two\n"); + exit(EXIT_FAILURE); + } + if (errorCode == ENOMEM) { + fprintf(stderr, + "Error: Insufficient memory to fulfill the request\n"); + exit(EXIT_FAILURE); + } + } + + if (ptr == NULL) { + fprintf(stderr, "Error: posix_memalign failed!\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} diff --git a/PoissonSolver/2D-seq/src/allocate.h b/PoissonSolver/2D-seq/src/allocate.h new file mode 100644 index 0000000..4b5575e --- /dev/null +++ b/PoissonSolver/2D-seq/src/allocate.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __ALLOCATE_H_ +#define __ALLOCATE_H_ +#include + +extern void* allocate (int alignment, size_t bytesize); + +#endif diff --git a/PoissonSolver/2D-seq/src/likwid-marker.h b/PoissonSolver/2D-seq/src/likwid-marker.h new file mode 100644 index 0000000..a35b495 --- /dev/null +++ b/PoissonSolver/2D-seq/src/likwid-marker.h @@ -0,0 +1,53 @@ +/* + * ======================================================================================= + * + * Author: Jan Eitzinger (je), jan.eitzinger@fau.de + * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * ======================================================================================= + */ +#ifndef LIKWID_MARKERS_H +#define LIKWID_MARKERS_H + +#ifdef LIKWID_PERFMON +#include +#define LIKWID_MARKER_INIT likwid_markerInit() +#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() +#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() +#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) +#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) +#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) +#define LIKWID_MARKER_CLOSE likwid_markerClose() +#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) likwid_markerGetRegion(regionTag, nevents, events, time, count) +#else /* LIKWID_PERFMON */ +#define LIKWID_MARKER_INIT +#define LIKWID_MARKER_THREADINIT +#define LIKWID_MARKER_SWITCH +#define LIKWID_MARKER_REGISTER(regionTag) +#define LIKWID_MARKER_START(regionTag) +#define LIKWID_MARKER_STOP(regionTag) +#define LIKWID_MARKER_CLOSE +#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) +#define LIKWID_MARKER_RESET(regionTag) +#endif /* LIKWID_PERFMON */ + +#endif /*LIKWID_MARKERS_H*/ diff --git a/PoissonSolver/2D-seq/src/main.c b/PoissonSolver/2D-seq/src/main.c new file mode 100644 index 0000000..588a848 --- /dev/null +++ b/PoissonSolver/2D-seq/src/main.c @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "parameter.h" +#include "solver.h" + + +int main (int argc, char** argv) +{ + Parameter params; + Solver solver; + initParameter(¶ms); + + if ( argc != 2 ) { + printf("Usage: %s \n",argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(¶ms, argv[1]); + printParameter(¶ms); + + initSolver(&solver, ¶ms, 2); + solve(&solver); + writeResult(&solver); + + return EXIT_SUCCESS; +} diff --git a/PoissonSolver/2D-seq/src/parameter.c b/PoissonSolver/2D-seq/src/parameter.c new file mode 100644 index 0000000..6c08af9 --- /dev/null +++ b/PoissonSolver/2D-seq/src/parameter.c @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +//--- +#include "parameter.h" +#include "util.h" +#define MAXLINE 4096 + +void initParameter(Parameter *param) { + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.8; +} + +void readParameter(Parameter *param, const char *filename) { + FILE *fp = fopen(filename, "r"); + char line[MAXLINE]; + int i; + + if(!fp) { + fprintf(stderr, "Could not open parameter file: %s\n", filename); + exit(EXIT_FAILURE); + } + + while(!feof(fp)) { + line[0] = '\0'; + fgets(line, MAXLINE, fp); + for(i = 0; line[i] != '\0' && line[i] != '#'; i++); + line[i] = '\0'; + + char *tok = strtok(line, " "); + char *val = strtok(NULL, " "); + + #define PARSE_PARAM(p,f) if(strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { param->p = f(val); } + #define PARSE_STRING(p) PARSE_PARAM(p, strdup) + #define PARSE_INT(p) PARSE_PARAM(p, atoi) + #define PARSE_REAL(p) PARSE_PARAM(p, atof) + + if(tok != NULL && val != NULL) { + PARSE_REAL(xlength); + PARSE_REAL(ylength); + PARSE_INT(imax); + PARSE_INT(jmax); + PARSE_INT(itermax); + PARSE_REAL(eps); + PARSE_REAL(omg); + } + } + + fclose(fp); +} + +void printParameter(Parameter *param) { + printf("Parameters:\n"); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %e, %e\n", param->xlength, param->ylength); + printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); + printf("Iterative solver parameters:\n"); + printf("\tMax iterations: %d\n", param->itermax); + printf("\tepsilon (stopping tolerance) : %e\n", param->eps); + printf("\tomega (SOR relaxation): %e\n", param->omg); +} diff --git a/PoissonSolver/2D-seq/src/parameter.h b/PoissonSolver/2D-seq/src/parameter.h new file mode 100644 index 0000000..d57ce19 --- /dev/null +++ b/PoissonSolver/2D-seq/src/parameter.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + double xlength, ylength; + int imax, jmax; + int itermax; + double eps, omg, gamma; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/PoissonSolver/2D-seq/src/solver.c b/PoissonSolver/2D-seq/src/solver.c new file mode 100644 index 0000000..3a3accd --- /dev/null +++ b/PoissonSolver/2D-seq/src/solver.c @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "stdlib.h" +#include "stdio.h" +#include "math.h" + +#include "solver.h" +#include "parameter.h" +#include "allocate.h" + +#define PI 3.14159265358979323846 +#define P(i,j) p[(j)*(imax+2) + (i)] +#define RHS(i,j) rhs[(j)*(imax+2) + (i)] + +void initSolver(Solver *solver, Parameter *params, int problem) +{ + solver->imax = params->imax; + solver->jmax = params->jmax; + solver->dx = params->xlength/params->imax; + solver->dy = params->ylength/params->jmax; + solver->eps = params->eps; + solver->omega = params->omg; + solver->itermax = params->itermax; + + int imax = solver->imax; + int jmax = solver->jmax; + size_t bytesize = (imax+2) * (jmax+2) * sizeof(double); + solver->p = allocate(64, bytesize); + solver->rhs = allocate(64, bytesize); + + double dx = solver->dx; + double dy = solver->dy; + double* p = solver->p; + double* rhs = solver->rhs; + + for( int j=0; jimax; + int jmax = solver->jmax; + double eps= solver->eps; + double omega = solver->omega; + int itermax = solver->itermax; + + double dx2 = solver->dx * solver->dx; + double dy2 = solver->dy * solver->dy; + double idx2 = 1.0/dx2; + double idy2 = 1.0/dy2; + double factor = omega * 0.5 * (dx2*dy2) / (dx2+dy2); + double* p = solver->p; + double* rhs = solver->rhs; + + res = eps + 1.0; + + while((res >= eps) && (it < itermax)) { + res = 0.0; + + for( int j=1; jimax; + int jmax = solver->jmax; + double* p = solver->p; + + FILE *fp; + fp= fopen("p.dat", "w"); + + if (fp== NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for( int j=0; j +#include + +double getTimeStamp() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeResolution() +{ + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; +} + +double getTimeStamp_() +{ + return getTimeStamp(); +} diff --git a/PoissonSolver/2D-seq/src/timing.h b/PoissonSolver/2D-seq/src/timing.h new file mode 100644 index 0000000..db95329 --- /dev/null +++ b/PoissonSolver/2D-seq/src/timing.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __TIMING_H_ +#define __TIMING_H_ + +extern double getTimeStamp(); +extern double getTimeResolution(); +extern double getTimeStamp_(); + +#endif // __TIMING_H_ diff --git a/PoissonSolver/2D-seq/src/util.h b/PoissonSolver/2D-seq/src/util.h new file mode 100644 index 0000000..fe275cb --- /dev/null +++ b/PoissonSolver/2D-seq/src/util.h @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x,y) ((x)<(y)?(x):(y)) +#endif +#ifndef MAX +#define MAX(x,y) ((x)>(y)?(x):(y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#endif // __UTIL_H_ diff --git a/PoissonSolver/2D-seq/surface.plot b/PoissonSolver/2D-seq/surface.plot new file mode 100644 index 0000000..7d56c7d --- /dev/null +++ b/PoissonSolver/2D-seq/surface.plot @@ -0,0 +1,7 @@ +set terminal png size 1024,768 enhanced font ,12 +set output 'p.png' +set datafile separator whitespace + +set grid +set hidden3d +splot 'p.dat' matrix using 1:2:3 with lines