diff --git a/.clangd b/.clangd index 1ad7985..f226d9f 100644 --- a/.clangd +++ b/.clangd @@ -1,3 +1,3 @@ -CompileFlags: # Tweak the parse settings - Add: [-I/usr/local/include] # treat all files as C++, enable more warnings +CompileFlags: + Add: [-I/usr/local/include, -I/opt/homebrew/include, -D_MPI] Compiler: clang diff --git a/BasicSolver/2D-mpi-v1/canal.par b/BasicSolver/2D-mpi-v1/canal.par deleted file mode 100644 index c90294c..0000000 --- a/BasicSolver/2D-mpi-v1/canal.par +++ /dev/null @@ -1,46 +0,0 @@ -#============================================================================== -# Laminar Canal Flow -#============================================================================== - -# Problem specific Data: -# --------------------- - -name canal # name of flow setup - -bcLeft 3 # flags for boundary conditions -bcRight 3 # 1 = no-slip 3 = outflow -bcBottom 1 # 2 = free-slip 4 = periodic -bcTop 1 # - -gx 0.0 # Body forces (e.g. gravity) -gy 0.0 # - -re 100.0 # Reynolds number - -u_init 1.0 # initial value for velocity in x-direction -v_init 0.0 # initial value for velocity in y-direction -p_init 0.0 # initial value for pressure - -# Geometry Data: -# ------------- - -xlength 30.0 # domain size in x-direction -ylength 4.0 # domain size in y-direction -imax 200 # number of interior cells in x-direction -jmax 50 # number of interior cells in y-direction - -# Time Data: -# --------- - -te 100.0 # final time -dt 0.02 # time stepsize -tau 0.5 # safety factor for time stepsize control (<0 constant delt) - -# Pressure Iteration Data: -# ----------------------- - -itermax 500 # maximal number of pressure iteration in one time step -eps 0.00001 # stopping tolerance for pressure iteration -omg 1.8 # relaxation parameter for SOR iteration -gamma 0.9 # upwind differencing factor gamma -#=============================================================================== diff --git a/BasicSolver/2D-mpi-v1/dcavity.par b/BasicSolver/2D-mpi-v1/dcavity.par deleted file mode 100644 index b5c22ce..0000000 --- a/BasicSolver/2D-mpi-v1/dcavity.par +++ /dev/null @@ -1,46 +0,0 @@ -#============================================================================== -# Driven Cavity -#============================================================================== - -# Problem specific Data: -# --------------------- - -name dcavity # name of flow setup - -bcLeft 1 # flags for boundary conditions -bcRight 1 # 1 = no-slip 3 = outflow -bcBottom 1 # 2 = free-slip 4 = periodic -bcTop 1 # - -gx 0.0 # Body forces (e.g. gravity) -gy 0.0 # - -re 500.0 # Reynolds number - -u_init 0.0 # initial value for velocity in x-direction -v_init 0.0 # initial value for velocity in y-direction -p_init 0.0 # initial value for pressure - -# Geometry Data: -# ------------- - -xlength 1.0 # domain size in x-direction -ylength 1.0 # domain size in y-direction -imax 100 # number of interior cells in x-direction -jmax 100 # number of interior cells in y-direction - -# Time Data: -# --------- - -te 25.0 # final time -dt 0.02 # time stepsize -tau 0.5 # safety factor for time stepsize control (<0 constant delt) - -# Pressure Iteration Data: -# ----------------------- - -itermax 1000 # maximal number of pressure iteration in one time step -eps 0.001 # stopping tolerance for pressure iteration -omg 1.7 # relaxation parameter for SOR iteration -gamma 0.9 # upwind differencing factor gamma -#=============================================================================== diff --git a/BasicSolver/2D-mpi-v1/include_CLANG.mk b/BasicSolver/2D-mpi-v1/include_CLANG.mk deleted file mode 100644 index 1d17c52..0000000 --- a/BasicSolver/2D-mpi-v1/include_CLANG.mk +++ /dev/null @@ -1,16 +0,0 @@ -CC = mpicc -GCC = cc -LINKER = $(CC) - -ifeq ($(ENABLE_OPENMP),true) -OPENMP = -fopenmp -#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp -LIBS = # -lomp -endif - -VERSION = --version -CFLAGS = -Ofast -std=c99 $(OPENMP) -#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG -LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE# -DDEBUG -INCLUDES = -I/usr/local/include diff --git a/BasicSolver/2D-mpi-v1/src/affinity.c b/BasicSolver/2D-mpi-v1/src/affinity.c deleted file mode 100644 index b501665..0000000 --- a/BasicSolver/2D-mpi-v1/src/affinity.c +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifdef __linux__ -#ifdef _OPENMP -#include -#include -#include -#include -#include -#include -#include - -#define MAX_NUM_THREADS 128 -#define gettid() syscall(SYS_gettid) - -static int getProcessorID(cpu_set_t* cpu_set) -{ - int processorId; - - for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { - if (CPU_ISSET(processorId, cpu_set)) { - break; - } - } - return processorId; -} - -int affinity_getProcessorId() -{ - cpu_set_t cpu_set; - CPU_ZERO(&cpu_set); - sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); - - return getProcessorID(&cpu_set); -} - -void affinity_pinThread(int processorId) -{ - cpu_set_t cpuset; - pthread_t thread; - - thread = pthread_self(); - CPU_ZERO(&cpuset); - CPU_SET(processorId, &cpuset); - pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); -} - -void affinity_pinProcess(int processorId) -{ - cpu_set_t cpuset; - - CPU_ZERO(&cpuset); - CPU_SET(processorId, &cpuset); - sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); -} -#endif /*_OPENMP*/ -#endif /*__linux__*/ diff --git a/BasicSolver/2D-mpi-v1/src/affinity.h b/BasicSolver/2D-mpi-v1/src/affinity.h deleted file mode 100644 index d844fe5..0000000 --- a/BasicSolver/2D-mpi-v1/src/affinity.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifndef AFFINITY_H -#define AFFINITY_H - -extern int affinity_getProcessorId(); -extern void affinity_pinProcess(int); -extern void affinity_pinThread(int); - -#endif /*AFFINITY_H*/ diff --git a/BasicSolver/2D-mpi-v1/src/main.c b/BasicSolver/2D-mpi-v1/src/main.c deleted file mode 100644 index d685367..0000000 --- a/BasicSolver/2D-mpi-v1/src/main.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include -#include - -#include "parameter.h" -#include "progress.h" -#include "solver.h" -#include "timing.h" -#include - -int main(int argc, char** argv) -{ - int rank; - double S, E; - Parameter params; - Solver solver; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - initParameter(¶ms); - - if (argc != 2) { - printf("Usage: %s \n", argv[0]); - exit(EXIT_SUCCESS); - } - - readParameter(¶ms, argv[1]); - if (rank == 0) { - printParameter(¶ms); - } - initSolver(&solver, ¶ms); - initProgress(solver.te); - - double tau = solver.tau; - double te = solver.te; - double t = 0.0; - - S = getTimeStamp(); - while (t <= te) { - if (tau > 0.0) { - computeTimestep(&solver); - } - - setBoundaryConditions(&solver); - setSpecialBoundaryCondition(&solver); - computeFG(&solver); - computeRHS(&solver); - solve(&solver); - adaptUV(&solver); - /* exit(EXIT_SUCCESS); */ - t += solver.dt; - -#ifdef VERBOSE - if (rank == 0) { - printf("TIME %f , TIMESTEP %f\n", t, solver.dt); - } -#else - printProgress(t); -#endif - } - E = getTimeStamp(); - stopProgress(); - if (rank == 0) { - printf("Solution took %.2fs\n", E - S); - } - collectResult(&solver); - - MPI_Finalize(); - return EXIT_SUCCESS; -} diff --git a/BasicSolver/2D-mpi-v1/src/parameter.h b/BasicSolver/2D-mpi-v1/src/parameter.h deleted file mode 100644 index f4c331a..0000000 --- a/BasicSolver/2D-mpi-v1/src/parameter.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#ifndef __PARAMETER_H_ -#define __PARAMETER_H_ - -typedef struct { - double xlength, ylength; - int imax, jmax; - int itermax; - double eps, omg; - double re, tau, gamma; - double te, dt; - double gx, gy; - char* name; - int bcLeft, bcRight, bcBottom, bcTop; - double u_init, v_init, p_init; -} Parameter; - -void initParameter(Parameter*); -void readParameter(Parameter*, const char*); -void printParameter(Parameter*); -#endif diff --git a/BasicSolver/2D-mpi-v1/src/progress.c b/BasicSolver/2D-mpi-v1/src/progress.c deleted file mode 100644 index 31a8a90..0000000 --- a/BasicSolver/2D-mpi-v1/src/progress.c +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include - -#include "progress.h" - -static double _end; -static int _current; -static int _rank = -1; - -void initProgress(double end) -{ - MPI_Comm_rank(MPI_COMM_WORLD, &_rank); - _end = end; - _current = 0; - - if (_rank == 0) { - printf("[ ]"); - fflush(stdout); - } -} - -void printProgress(double current) -{ - if (_rank == 0) { - int new = (int)rint((current / _end) * 10.0); - - if (new > _current) { - char progress[11]; - _current = new; - progress[0] = 0; - - for (int i = 0; i < 10; i++) { - if (i < _current) { - sprintf(progress + strlen(progress), "#"); - } else { - sprintf(progress + strlen(progress), " "); - } - } - printf("\r[%s]", progress); - } - fflush(stdout); - } -} - -void stopProgress() -{ - if (_rank == 0) { - printf("\n"); - fflush(stdout); - } -} diff --git a/BasicSolver/2D-mpi-v1/src/solver.c b/BasicSolver/2D-mpi-v1/src/solver.c deleted file mode 100644 index 6bc575d..0000000 --- a/BasicSolver/2D-mpi-v1/src/solver.c +++ /dev/null @@ -1,689 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include -#include - -#include "allocate.h" -#include "parameter.h" -#include "solver.h" -#include "util.h" - -#define P(i, j) p[(j) * (imax + 2) + (i)] -#define F(i, j) f[(j) * (imax + 2) + (i)] -#define G(i, j) g[(j) * (imax + 2) + (i)] -#define U(i, j) u[(j) * (imax + 2) + (i)] -#define V(i, j) v[(j) * (imax + 2) + (i)] -#define RHS(i, j) rhs[(j) * (imax + 2) + (i)] - -static int sizeOfRank(int rank, int size, int N) -{ - return N / size + ((N % size > rank) ? 1 : 0); -} - -static void print(Solver* solver, double* grid) -{ - int imax = solver->imax; - - for (int i = 0; i < solver->size; i++) { - if (i == solver->rank) { - printf("### RANK %d " - "#######################################################\n", - solver->rank); - for (int j = 0; j < solver->jmaxLocal + 2; j++) { - printf("%02d: ", j); - for (int i = 0; i < solver->imax + 2; i++) { - printf("%12.8f ", grid[j * (imax + 2) + i]); - } - printf("\n"); - } - fflush(stdout); - } - MPI_Barrier(MPI_COMM_WORLD); - } -} - -static void exchange(Solver* solver, double* grid) -{ - MPI_Request requests[4] = { MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL }; - - /* exchange ghost cells with top neighbor */ - if (solver->rank + 1 < solver->size) { - int top = solver->rank + 1; - double* src = grid + (solver->jmaxLocal) * (solver->imax + 2) + 1; - double* dst = grid + (solver->jmaxLocal + 1) * (solver->imax + 2) + 1; - - MPI_Isend(src, solver->imax, MPI_DOUBLE, top, 1, MPI_COMM_WORLD, &requests[0]); - MPI_Irecv(dst, solver->imax, MPI_DOUBLE, top, 2, MPI_COMM_WORLD, &requests[1]); - } - - /* exchange ghost cells with bottom neighbor */ - if (solver->rank > 0) { - int bottom = solver->rank - 1; - double* src = grid + (solver->imax + 2) + 1; - double* dst = grid + 1; - - MPI_Isend(src, solver->imax, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]); - MPI_Irecv(dst, solver->imax, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]); - } - - MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); -} - -static void shift(Solver* solver) -{ - MPI_Request requests[2] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL }; - double* g = solver->g; - - /* shift G */ - /* receive ghost cells from bottom neighbor */ - if (solver->rank > 0) { - int bottom = solver->rank - 1; - MPI_Irecv(g + 1, - solver->imax, - MPI_DOUBLE, - bottom, - 0, - MPI_COMM_WORLD, - &requests[0]); - } - - if (solver->rank + 1 < solver->size) { - int top = solver->rank + 1; - double* buf = g + (solver->jmaxLocal) * (solver->imax + 2) + 1; - /* send ghost cells to top neighbor */ - MPI_Isend(buf, solver->imax, MPI_DOUBLE, top, 0, MPI_COMM_WORLD, &requests[1]); - } - - MPI_Waitall(2, requests, MPI_STATUSES_IGNORE); -} - -void collectResult(Solver* solver) -{ - double* Pall = NULL; - double* Uall = NULL; - double* Vall = NULL; - int *rcvCounts, *displs; - - if (solver->rank == 0) { - Pall = allocate(64, (solver->imax + 2) * (solver->jmax + 2) * sizeof(double)); - Uall = allocate(64, (solver->imax + 2) * (solver->jmax + 2) * sizeof(double)); - Vall = allocate(64, (solver->imax + 2) * (solver->jmax + 2) * sizeof(double)); - rcvCounts = (int*)malloc(solver->size * sizeof(int)); - displs = (int*)malloc(solver->size * sizeof(int)); - rcvCounts[0] = solver->jmaxLocal * (solver->imax + 2); - displs[0] = 0; - int cursor = rcvCounts[0]; - - for (int i = 1; i < solver->size; i++) { - rcvCounts[i] = sizeOfRank(i, solver->size, solver->jmax) * (solver->imax + 2); - displs[i] = cursor; - cursor += rcvCounts[i]; - } - } - - int cnt = solver->jmaxLocal * (solver->imax + 2); - double* sendbuffer = solver->p + (solver->imax + 2); - MPI_Gatherv(sendbuffer, - cnt, - MPI_DOUBLE, - Pall, - rcvCounts, - displs, - MPI_DOUBLE, - 0, - MPI_COMM_WORLD); - sendbuffer = solver->u + (solver->imax + 2); - MPI_Gatherv(sendbuffer, - cnt, - MPI_DOUBLE, - Uall, - rcvCounts, - displs, - MPI_DOUBLE, - 0, - MPI_COMM_WORLD); - sendbuffer = solver->v + (solver->imax + 2); - MPI_Gatherv(sendbuffer, - cnt, - MPI_DOUBLE, - Vall, - rcvCounts, - displs, - MPI_DOUBLE, - 0, - MPI_COMM_WORLD); - - if (solver->rank == 0) { - writeResult(solver, Pall, Uall, Vall); - } -} - -static void printConfig(Solver* solver) -{ - if (solver->rank == 0) { - printf("Parameters for #%s#\n", solver->problem); - printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", - solver->bcLeft, - solver->bcRight, - solver->bcBottom, - solver->bcTop); - printf("\tReynolds number: %.2f\n", solver->re); - printf("\tGx Gy: %.2f %.2f\n", solver->gx, solver->gy); - printf("Geometry data:\n"); - printf("\tDomain box size (x, y): %.2f, %.2f\n", - solver->xlength, - solver->ylength); - printf("\tCells (x, y): %d, %d\n", solver->imax, solver->jmax); - printf("Timestep parameters:\n"); - printf("\tDefault stepsize: %.2f, Final time %.2f\n", solver->dt, solver->te); - printf("\tdt bound: %.6f\n", solver->dtBound); - printf("\tTau factor: %.2f\n", solver->tau); - printf("Iterative solver parameters:\n"); - printf("\tMax iterations: %d\n", solver->itermax); - printf("\tepsilon (stopping tolerance) : %f\n", solver->eps); - printf("\tgamma factor: %f\n", solver->gamma); - printf("\tomega (SOR relaxation): %f\n", solver->omega); - printf("Communication parameters:\n"); - } - for (int i = 0; i < solver->size; i++) { - if (i == solver->rank) { - printf("\tRank %d of %d\n", solver->rank, solver->size); - printf("\tLocal domain size: %dx%d\n", solver->imax, solver->jmaxLocal); - fflush(stdout); - } - } -} - -void initSolver(Solver* solver, Parameter* params) -{ - MPI_Comm_rank(MPI_COMM_WORLD, &(solver->rank)); - MPI_Comm_size(MPI_COMM_WORLD, &(solver->size)); - solver->problem = params->name; - solver->bcLeft = params->bcLeft; - solver->bcRight = params->bcRight; - solver->bcBottom = params->bcBottom; - solver->bcTop = params->bcTop; - solver->imax = params->imax; - solver->jmax = params->jmax; - solver->jmaxLocal = sizeOfRank(solver->rank, solver->size, solver->jmax); - solver->xlength = params->xlength; - solver->ylength = params->ylength; - solver->dx = params->xlength / params->imax; - solver->dy = params->ylength / params->jmax; - solver->eps = params->eps; - solver->omega = params->omg; - solver->itermax = params->itermax; - solver->re = params->re; - solver->gx = params->gx; - solver->gy = params->gy; - solver->dt = params->dt; - solver->te = params->te; - solver->tau = params->tau; - solver->gamma = params->gamma; - - int imax = solver->imax; - int jmaxLocal = solver->jmaxLocal; - size_t bytesize = (imax + 2) * (jmaxLocal + 2) * sizeof(double); - solver->u = allocate(64, bytesize); - solver->v = allocate(64, bytesize); - solver->p = allocate(64, bytesize); - solver->rhs = allocate(64, bytesize); - solver->f = allocate(64, bytesize); - solver->g = allocate(64, bytesize); - - for (int i = 0; i < (imax + 2) * (jmaxLocal + 2); i++) { - solver->u[i] = params->u_init; - solver->v[i] = params->v_init; - solver->p[i] = params->p_init; - solver->rhs[i] = 0.0; - solver->f[i] = 0.0; - solver->g[i] = 0.0; - } - - double dx = solver->dx; - double dy = solver->dy; - double inv_sqr_sum = 1.0 / (dx * dx) + 1.0 / (dy * dy); - solver->dtBound = 0.5 * solver->re * 1.0 / inv_sqr_sum; -#ifdef VERBOSE - printConfig(solver); -#endif -} - -void computeRHS(Solver* solver) -{ - int imax = solver->imax; - int jmaxLocal = solver->jmaxLocal; - double idx = 1.0 / solver->dx; - double idy = 1.0 / solver->dy; - double idt = 1.0 / solver->dt; - double* rhs = solver->rhs; - double* f = solver->f; - double* g = solver->g; - - shift(solver); - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) * - idt; - } - } -} - -void solve(Solver* solver) -{ - int imax = solver->imax; - int jmax = solver->jmax; - int jmaxLocal = solver->jmaxLocal; - double eps = solver->eps; - int itermax = solver->itermax; - double dx2 = solver->dx * solver->dx; - double dy2 = solver->dy * solver->dy; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); - double* p = solver->p; - double* rhs = solver->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - exchange(solver, p); - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - - double r = RHS(i, j) - - ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + - (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - - P(i, j) -= (factor * r); - res += (r * r); - } - } - - if (solver->rank == 0) { - for (int i = 1; i < imax + 1; i++) { - P(i, 0) = P(i, 1); - } - } - - if (solver->rank == (solver->size - 1)) { - for (int i = 1; i < imax + 1; i++) { - P(i, jmaxLocal + 1) = P(i, jmaxLocal); - } - } - - for (int j = 1; j < jmaxLocal + 1; j++) { - P(0, j) = P(1, j); - P(imax + 1, j) = P(imax, j); - } - - MPI_Allreduce(MPI_IN_PLACE, &res, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - res = res / (double)(imax * jmax); -#ifdef DEBUG - if (solver->rank == 0) { - printf("%d Residuum: %e\n", it, res); - } -#endif - it++; - } - -#ifdef VERBOSE - if (solver->rank == 0) { - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); - } -#endif -} - -static double maxElement(Solver* solver, double* m) -{ - int size = (solver->imax + 2) * (solver->jmaxLocal + 2); - double maxval = DBL_MIN; - - for (int i = 0; i < size; i++) { - maxval = MAX(maxval, fabs(m[i])); - } - - MPI_Allreduce(MPI_IN_PLACE, &maxval, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - return maxval; -} - -void normalizePressure(Solver* solver) -{ - int size = (solver->imax + 2) * (solver->jmaxLocal + 2); - double* p = solver->p; - double avgP = 0.0; - - for (int i = 0; i < size; i++) { - avgP += p[i]; - } - MPI_Allreduce(MPI_IN_PLACE, &avgP, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - avgP /= (solver->imax + 2) * (solver->jmax + 2); - - for (int i = 0; i < size; i++) { - p[i] = p[i] - avgP; - } -} - -void computeTimestep(Solver* solver) -{ - double dt = solver->dtBound; - double dx = solver->dx; - double dy = solver->dy; - double umax = maxElement(solver, solver->u); - double vmax = maxElement(solver, solver->v); - - if (umax > 0) { - dt = (dt > dx / umax) ? dx / umax : dt; - } - if (vmax > 0) { - dt = (dt > dy / vmax) ? dy / vmax : dt; - } - - solver->dt = dt * solver->tau; -} - -void setBoundaryConditions(Solver* solver) -{ - int imax = solver->imax; - int jmaxLocal = solver->jmaxLocal; - double* u = solver->u; - double* v = solver->v; - - // Left boundary - switch (solver->bcLeft) { - case NOSLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j) = 0.0; - V(0, j) = -V(1, j); - } - break; - case SLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j) = 0.0; - V(0, j) = V(1, j); - } - break; - case OUTFLOW: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j) = U(1, j); - V(0, j) = V(1, j); - } - break; - case PERIODIC: - break; - } - - // Right boundary - switch (solver->bcRight) { - case NOSLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imax, j) = 0.0; - V(imax + 1, j) = -V(imax, j); - } - break; - case SLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imax, j) = 0.0; - V(imax + 1, j) = V(imax, j); - } - break; - case OUTFLOW: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imax, j) = U(imax - 1, j); - V(imax + 1, j) = V(imax, j); - } - break; - case PERIODIC: - break; - } - - // Bottom boundary - if (solver->rank == 0) { - switch (solver->bcBottom) { - case NOSLIP: - for (int i = 1; i < imax + 1; i++) { - V(i, 0) = 0.0; - U(i, 0) = -U(i, 1); - } - break; - case SLIP: - for (int i = 1; i < imax + 1; i++) { - V(i, 0) = 0.0; - U(i, 0) = U(i, 1); - } - break; - case OUTFLOW: - for (int i = 1; i < imax + 1; i++) { - U(i, 0) = U(i, 1); - V(i, 0) = V(i, 1); - } - break; - case PERIODIC: - break; - } - } - - // Top boundary - if (solver->rank == (solver->size - 1)) { - switch (solver->bcTop) { - case NOSLIP: - for (int i = 1; i < imax + 1; i++) { - V(i, jmaxLocal) = 0.0; - U(i, jmaxLocal + 1) = -U(i, jmaxLocal); - } - break; - case SLIP: - for (int i = 1; i < imax + 1; i++) { - V(i, jmaxLocal) = 0.0; - U(i, jmaxLocal + 1) = U(i, jmaxLocal); - } - break; - case OUTFLOW: - for (int i = 1; i < imax + 1; i++) { - U(i, jmaxLocal + 1) = U(i, jmaxLocal); - V(i, jmaxLocal) = V(i, jmaxLocal - 1); - } - break; - case PERIODIC: - break; - } - } -} - -void setSpecialBoundaryCondition(Solver* solver) -{ - int imax = solver->imax; - int jmaxLocal = solver->jmaxLocal; - double* u = solver->u; - - if (strcmp(solver->problem, "dcavity") == 0) { - if (solver->rank == (solver->size - 1)) { - for (int i = 1; i < imax; i++) { - U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal); - } - } - } else if (strcmp(solver->problem, "canal") == 0) { - double ylength = solver->ylength; - double dy = solver->dy; - int rest = solver->jmax % solver->size; - int yc = solver->rank * (solver->jmax / solver->size) + MIN(rest, solver->rank); - double ys = dy * (yc + 0.5); - double y; - - /* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */ - - for (int j = 1; j < jmaxLocal + 1; j++) { - y = ys + dy * (j - 0.5); - U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); - } - } - /* print(solver, solver->u); */ -} - -void computeFG(Solver* solver) -{ - double* u = solver->u; - double* v = solver->v; - double* f = solver->f; - double* g = solver->g; - int imax = solver->imax; - int jmaxLocal = solver->jmaxLocal; - double gx = solver->gx; - double gy = solver->gy; - double gamma = solver->gamma; - double dt = solver->dt; - double inverseRe = 1.0 / solver->re; - double inverseDx = 1.0 / solver->dx; - double inverseDy = 1.0 / solver->dy; - double du2dx, dv2dy, duvdx, duvdy; - double du2dx2, du2dy2, dv2dx2, dv2dy2; - - exchange(solver, u); - exchange(solver, v); - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - du2dx = inverseDx * 0.25 * - ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - - (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + - fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); - - duvdy = inverseDy * 0.25 * - ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - - (V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + - fabs(V(i, j - 1) + V(i + 1, j - 1)) * - (U(i, j) - U(i, j - 1))); - - du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); - du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); - F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); - - duvdx = inverseDx * 0.25 * - ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - - (U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + - fabs(U(i - 1, j) + U(i - 1, j + 1)) * - (V(i, j) - V(i - 1, j))); - - dv2dy = inverseDy * 0.25 * - ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - - (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + - fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); - - dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); - dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); - G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); - } - } - - /* ----------------------------- boundary of F --------------------------- - */ - for (int j = 1; j < jmaxLocal + 1; j++) { - F(0, j) = U(0, j); - F(imax, j) = U(imax, j); - } - - /* ----------------------------- boundary of G --------------------------- - */ - if (solver->rank == 0) { - for (int i = 1; i < imax + 1; i++) { - G(i, 0) = V(i, 0); - } - } - - if (solver->rank == (solver->size - 1)) { - for (int i = 1; i < imax + 1; i++) { - G(i, jmaxLocal) = V(i, jmaxLocal); - } - } -} - -void adaptUV(Solver* solver) -{ - int imax = solver->imax; - int jmaxLocal = solver->jmaxLocal; - double* p = solver->p; - double* u = solver->u; - double* v = solver->v; - double* f = solver->f; - double* g = solver->g; - double factorX = solver->dt / solver->dx; - double factorY = solver->dt / solver->dy; - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; - V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; - } - } -} - -void writeResult(Solver* solver, double* p, double* u, double* v) -{ - int imax = solver->imax; - int jmax = solver->jmax; - double dx = solver->dx; - double dy = solver->dy; - double x = 0.0, y = 0.0; - - FILE* fp; - fp = fopen("pressure.dat", "w"); - - if (fp == NULL) { - printf("Error!\n"); - exit(EXIT_FAILURE); - } - - for (int j = 1; j < jmax + 1; j++) { - y = (double)(j - 0.5) * dy; - for (int i = 1; i < imax + 1; i++) { - x = (double)(i - 0.5) * dx; - fprintf(fp, "%.2f %.2f %f\n", x, y, P(i, j)); - } - fprintf(fp, "\n"); - } - - fclose(fp); - - fp = fopen("velocity.dat", "w"); - - if (fp == NULL) { - printf("Error!\n"); - exit(EXIT_FAILURE); - } - - for (int j = 1; j < jmax + 1; j++) { - y = dy * (j - 0.5); - for (int i = 1; i < imax + 1; i++) { - x = dx * (i - 0.5); - double vel_u = (U(i, j) + U(i - 1, j)) / 2.0; - double vel_v = (V(i, j) + V(i, j - 1)) / 2.0; - double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); - fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); - } - } - - fclose(fp); -} diff --git a/BasicSolver/2D-mpi-v1/src/solver.h b/BasicSolver/2D-mpi-v1/src/solver.h deleted file mode 100644 index 00f8fd1..0000000 --- a/BasicSolver/2D-mpi-v1/src/solver.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#ifndef __SOLVER_H_ -#define __SOLVER_H_ -#include "parameter.h" - -enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; - -typedef struct { - /* geometry and grid information */ - double dx, dy; - int imax, jmax; - int jmaxLocal; - double xlength, ylength; - /* arrays */ - double *p, *rhs; - double *f, *g; - double *u, *v; - /* parameters */ - double eps, omega; - double re, tau, gamma; - double gx, gy; - /* time stepping */ - int itermax; - double dt, te; - double dtBound; - char* problem; - int bcLeft, bcRight, bcBottom, bcTop; - /* mpi */ - int rank; - int size; -} Solver; - -void initSolver(Solver*, Parameter*); -void computeRHS(Solver*); -void solve(Solver*); -void normalizePressure(Solver*); -void computeTimestep(Solver*); -void setBoundaryConditions(Solver*); -void setSpecialBoundaryCondition(Solver*); -void computeFG(Solver*); -void adaptUV(Solver*); -void collectResult(Solver*); -void writeResult(Solver*, double*, double*, double*); -#endif diff --git a/BasicSolver/2D-mpi-v1/src/util.h b/BasicSolver/2D-mpi-v1/src/util.h deleted file mode 100644 index 61a1dff..0000000 --- a/BasicSolver/2D-mpi-v1/src/util.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifndef __UTIL_H_ -#define __UTIL_H_ -#define HLINE \ - "------------------------------------------------------------------------" \ - "----\n" - -#ifndef MIN -#define MIN(x, y) ((x) < (y) ? (x) : (y)) -#endif -#ifndef MAX -#define MAX(x, y) ((x) > (y) ? (x) : (y)) -#endif -#ifndef ABS -#define ABS(a) ((a) >= 0 ? (a) : -(a)) -#endif - -#endif // __UTIL_H_ diff --git a/BasicSolver/2D-mpi-v1/vector.plot b/BasicSolver/2D-mpi-v1/vector.plot deleted file mode 100644 index 0934ab2..0000000 --- a/BasicSolver/2D-mpi-v1/vector.plot +++ /dev/null @@ -1,5 +0,0 @@ -set terminal png size 1800,768 enhanced font ,12 -set output 'velocity.png' -set datafile separator whitespace - -plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/BasicSolver/2D-mpi-v2/README.md b/BasicSolver/2D-mpi-v2/README.md deleted file mode 100644 index b0a80a6..0000000 --- a/BasicSolver/2D-mpi-v2/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# C source skeleton - -## Build - -1. Configure the toolchain and additional options in `config.mk`: -``` -# Supported: GCC, CLANG, ICC -TAG ?= GCC -ENABLE_OPENMP ?= false - -OPTIONS += -DARRAY_ALIGNMENT=64 -#OPTIONS += -DVERBOSE_AFFINITY -#OPTIONS += -DVERBOSE_DATASIZE -#OPTIONS += -DVERBOSE_TIMER -``` - -The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution. - - -2. Build with: -``` -make -``` - -You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. -Intermediate build results are located in the `` directory. - -To output the executed commands use: -``` -make Q= -``` - -3. Clean up with: -``` -make clean -``` -to clean intermediate build results. - -``` -make distclean -``` -to clean intermediate build results and binary. - -4. (Optional) Generate assembler: -``` -make asm -``` -The assembler files will also be located in the `` directory. diff --git a/BasicSolver/2D-mpi-v2/config.mk b/BasicSolver/2D-mpi-v2/config.mk deleted file mode 100644 index e187c7d..0000000 --- a/BasicSolver/2D-mpi-v2/config.mk +++ /dev/null @@ -1,10 +0,0 @@ -# Supported: GCC, CLANG, ICC -TAG ?= CLANG -ENABLE_OPENMP ?= false - -#Feature options -OPTIONS += -DARRAY_ALIGNMENT=64 -# OPTIONS += -DVERBOSE -#OPTIONS += -DVERBOSE_AFFINITY -#OPTIONS += -DVERBOSE_DATASIZE -#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/2D-mpi-v2/dcavity.par b/BasicSolver/2D-mpi-v2/dcavity.par deleted file mode 100644 index d20d83d..0000000 --- a/BasicSolver/2D-mpi-v2/dcavity.par +++ /dev/null @@ -1,46 +0,0 @@ -#============================================================================== -# Driven Cavity -#============================================================================== - -# Problem specific Data: -# --------------------- - -name dcavity # name of flow setup - -bcN 1 # flags for boundary conditions -bcE 1 # 1 = no-slip 3 = outflow -bcS 1 # 2 = free-slip 4 = periodic -bcW 1 # - -gx 0.0 # Body forces (e.g. gravity) -gy 0.0 # - -re 1000.0 # Reynolds number - -u_init 0.0 # initial value for velocity in x-direction -v_init 0.0 # initial value for velocity in y-direction -p_init 0.0 # initial value for pressure - -# Geometry Data: -# ------------- - -xlength 1.0 # domain size in x-direction -ylength 1.0 # domain size in y-direction -imax 100 # number of interior cells in x-direction -jmax 100 # number of interior cells in y-direction - -# Time Data: -# --------- - -te 10.0 # final time -dt 0.02 # time stepsize -tau 0.5 # safety factor for time stepsize control (<0 constant delt) - -# Pressure Iteration Data: -# ----------------------- - -itermax 1000 # maximal number of pressure iteration in one time step -eps 0.001 # stopping tolerance for pressure iteration -omg 1.7 # relaxation parameter for SOR iteration -gamma 0.9 # upwind differencing factor gamma -#=============================================================================== diff --git a/BasicSolver/2D-mpi-v2/include_CLANG.mk b/BasicSolver/2D-mpi-v2/include_CLANG.mk deleted file mode 100644 index 1d17c52..0000000 --- a/BasicSolver/2D-mpi-v2/include_CLANG.mk +++ /dev/null @@ -1,16 +0,0 @@ -CC = mpicc -GCC = cc -LINKER = $(CC) - -ifeq ($(ENABLE_OPENMP),true) -OPENMP = -fopenmp -#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp -LIBS = # -lomp -endif - -VERSION = --version -CFLAGS = -Ofast -std=c99 $(OPENMP) -#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG -LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE# -DDEBUG -INCLUDES = -I/usr/local/include diff --git a/BasicSolver/2D-mpi-v2/include_ICC.mk b/BasicSolver/2D-mpi-v2/include_ICC.mk deleted file mode 100644 index f85d836..0000000 --- a/BasicSolver/2D-mpi-v2/include_ICC.mk +++ /dev/null @@ -1,14 +0,0 @@ -CC = mpiicc -GCC = gcc -LINKER = $(CC) - -ifeq ($(ENABLE_OPENMP),true) -OPENMP = -qopenmp -endif - -VERSION = --version -CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) -LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE -INCLUDES = -LIBS = diff --git a/BasicSolver/2D-mpi-v2/src/affinity.c b/BasicSolver/2D-mpi-v2/src/affinity.c deleted file mode 100644 index b501665..0000000 --- a/BasicSolver/2D-mpi-v2/src/affinity.c +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifdef __linux__ -#ifdef _OPENMP -#include -#include -#include -#include -#include -#include -#include - -#define MAX_NUM_THREADS 128 -#define gettid() syscall(SYS_gettid) - -static int getProcessorID(cpu_set_t* cpu_set) -{ - int processorId; - - for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { - if (CPU_ISSET(processorId, cpu_set)) { - break; - } - } - return processorId; -} - -int affinity_getProcessorId() -{ - cpu_set_t cpu_set; - CPU_ZERO(&cpu_set); - sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); - - return getProcessorID(&cpu_set); -} - -void affinity_pinThread(int processorId) -{ - cpu_set_t cpuset; - pthread_t thread; - - thread = pthread_self(); - CPU_ZERO(&cpuset); - CPU_SET(processorId, &cpuset); - pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); -} - -void affinity_pinProcess(int processorId) -{ - cpu_set_t cpuset; - - CPU_ZERO(&cpuset); - CPU_SET(processorId, &cpuset); - sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); -} -#endif /*_OPENMP*/ -#endif /*__linux__*/ diff --git a/BasicSolver/2D-mpi-v2/src/main.c b/BasicSolver/2D-mpi-v2/src/main.c deleted file mode 100644 index c600033..0000000 --- a/BasicSolver/2D-mpi-v2/src/main.c +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include -#include - -#include "parameter.h" -#include "progress.h" -#include "solver.h" -#include "timing.h" - -int main(int argc, char** argv) -{ - int rank; - double S, E; - Parameter params; - Solver solver; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - initParameter(¶ms); - - if (argc != 2) { - printf("Usage: %s \n", argv[0]); - exit(EXIT_SUCCESS); - } - - readParameter(¶ms, argv[1]); - if (rank == 0) { - printParameter(¶ms); - } - initSolver(&solver, ¶ms); - /* debugExchange(&solver); */ - /* debugBC(&solver); */ - /* exit(EXIT_SUCCESS); */ - initProgress(solver.te); - - double tau = solver.tau; - double te = solver.te; - double t = 0.0; - - S = getTimeStamp(); - while (t <= te) { - if (tau > 0.0) { - computeTimestep(&solver); - } - - setBoundaryConditions(&solver); - setSpecialBoundaryCondition(&solver); - computeFG(&solver); - computeRHS(&solver); - solve(&solver); - adaptUV(&solver); - t += solver.dt; - -#ifdef VERBOSE - if (rank == 0) { - printf("TIME %f , TIMESTEP %f\n", t, solver.dt); - } -#else - printProgress(t); -#endif - } - E = getTimeStamp(); - stopProgress(); - if (rank == 0) { - printf("Solution took %.2fs\n", E - S); - } - collectResult(&solver); - - MPI_Finalize(); - return EXIT_SUCCESS; -} diff --git a/BasicSolver/2D-mpi-v2/src/parameter.c b/BasicSolver/2D-mpi-v2/src/parameter.c deleted file mode 100644 index 8f8a7e3..0000000 --- a/BasicSolver/2D-mpi-v2/src/parameter.c +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include - -#include "parameter.h" -#include "util.h" -#define MAXLINE 4096 - -void initParameter(Parameter* param) -{ - param->xlength = 1.0; - param->ylength = 1.0; - param->imax = 100; - param->jmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.8; -} - -void readParameter(Parameter* param, const char* filename) -{ - FILE* fp = fopen(filename, "r"); - char line[MAXLINE]; - int i; - - if (!fp) { - fprintf(stderr, "Could not open parameter file: %s\n", filename); - exit(EXIT_FAILURE); - } - - while (!feof(fp)) { - line[0] = '\0'; - fgets(line, MAXLINE, fp); - for (i = 0; line[i] != '\0' && line[i] != '#'; i++) - ; - line[i] = '\0'; - - char* tok = strtok(line, " "); - char* val = strtok(NULL, " "); - -#define PARSE_PARAM(p, f) \ - if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ - param->p = f(val); \ - } -#define PARSE_STRING(p) PARSE_PARAM(p, strdup) -#define PARSE_INT(p) PARSE_PARAM(p, atoi) -#define PARSE_REAL(p) PARSE_PARAM(p, atof) - - if (tok != NULL && val != NULL) { - PARSE_REAL(xlength); - PARSE_REAL(ylength); - PARSE_INT(imax); - PARSE_INT(jmax); - PARSE_INT(itermax); - PARSE_REAL(eps); - PARSE_REAL(omg); - PARSE_REAL(re); - PARSE_REAL(tau); - PARSE_REAL(gamma); - PARSE_REAL(dt); - PARSE_REAL(te); - PARSE_REAL(gx); - PARSE_REAL(gy); - PARSE_STRING(name); - PARSE_INT(bcN); - PARSE_INT(bcS); - PARSE_INT(bcE); - PARSE_INT(bcW); - PARSE_REAL(u_init); - PARSE_REAL(v_init); - PARSE_REAL(p_init); - } - } - - fclose(fp); -} - -void printParameter(Parameter* param) -{ - printf("Parameters for %s\n", param->name); - printf("Boundary conditions N:%d E:%d S:%d W:%d\n", - param->bcN, - param->bcE, - param->bcS, - param->bcW); - printf("\tReynolds number: %.2f\n", param->re); - printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n", - param->u_init, - param->v_init, - param->p_init); - printf("Geometry data:\n"); - printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength); - printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); - printf("Timestep parameters:\n"); - printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); - printf("\tTau factor: %.2f\n", param->tau); - printf("Iterative solver parameters:\n"); - printf("\tMax iterations: %d\n", param->itermax); - printf("\tepsilon (stopping tolerance) : %f\n", param->eps); - printf("\tgamma (stopping tolerance) : %f\n", param->gamma); - printf("\tomega (SOR relaxation): %f\n", param->omg); -} diff --git a/BasicSolver/2D-mpi-v2/src/parameter.h b/BasicSolver/2D-mpi-v2/src/parameter.h deleted file mode 100644 index a6c6756..0000000 --- a/BasicSolver/2D-mpi-v2/src/parameter.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#ifndef __PARAMETER_H_ -#define __PARAMETER_H_ - -typedef struct { - double xlength, ylength; - int imax, jmax; - int itermax; - double eps, omg; - double re, tau, gamma; - double te, dt; - double gx, gy; - char* name; - int bcN, bcS, bcE, bcW; - double u_init, v_init, p_init; -} Parameter; - -void initParameter(Parameter*); -void readParameter(Parameter*, const char*); -void printParameter(Parameter*); -#endif diff --git a/BasicSolver/2D-mpi-v2/src/progress.c b/BasicSolver/2D-mpi-v2/src/progress.c deleted file mode 100644 index 31a8a90..0000000 --- a/BasicSolver/2D-mpi-v2/src/progress.c +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include - -#include "progress.h" - -static double _end; -static int _current; -static int _rank = -1; - -void initProgress(double end) -{ - MPI_Comm_rank(MPI_COMM_WORLD, &_rank); - _end = end; - _current = 0; - - if (_rank == 0) { - printf("[ ]"); - fflush(stdout); - } -} - -void printProgress(double current) -{ - if (_rank == 0) { - int new = (int)rint((current / _end) * 10.0); - - if (new > _current) { - char progress[11]; - _current = new; - progress[0] = 0; - - for (int i = 0; i < 10; i++) { - if (i < _current) { - sprintf(progress + strlen(progress), "#"); - } else { - sprintf(progress + strlen(progress), " "); - } - } - printf("\r[%s]", progress); - } - fflush(stdout); - } -} - -void stopProgress() -{ - if (_rank == 0) { - printf("\n"); - fflush(stdout); - } -} diff --git a/BasicSolver/2D-mpi-v2/src/solver.c b/BasicSolver/2D-mpi-v2/src/solver.c deleted file mode 100644 index 28f266e..0000000 --- a/BasicSolver/2D-mpi-v2/src/solver.c +++ /dev/null @@ -1,900 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include -#include - -#include "allocate.h" -#include "parameter.h" -#include "solver.h" -#include "util.h" - -#define P(i, j) p[(j) * (imaxLocal + 2) + (i)] -#define F(i, j) f[(j) * (imaxLocal + 2) + (i)] -#define G(i, j) g[(j) * (imaxLocal + 2) + (i)] -#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] -#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] -#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)] - -#define NDIMS 2 -#define IDIM 0 -#define JDIM 1 - -static int sizeOfRank(int rank, int size, int N) -{ - return N / size + ((N % size > rank) ? 1 : 0); -} - -void print(Solver* solver, double* grid) -{ - int imaxLocal = solver->imaxLocal; - - for (int i = 0; i < solver->size; i++) { - if (i == solver->rank) { - printf( - "### RANK %d #######################################################\n", - solver->rank); - for (int j = 0; j < solver->jmaxLocal + 2; j++) { - printf("%02d: ", j); - for (int i = 0; i < solver->imaxLocal + 2; i++) { - printf("%12.8f ", grid[j * (imaxLocal + 2) + i]); - } - printf("\n"); - } - fflush(stdout); - } - MPI_Barrier(MPI_COMM_WORLD); - } -} - -static void exchange(Solver* solver, double* grid) -{ - double* buf[8]; - MPI_Request requests[8]; - for (int i = 0; i < 8; i++) - requests[i] = MPI_REQUEST_NULL; - - buf[0] = grid + 1; // recv bottom - buf[1] = grid + (solver->imaxLocal + 2) + 1; // send bottom - buf[2] = grid + (solver->jmaxLocal + 1) * (solver->imaxLocal + 2) + 1; // recv top - buf[3] = grid + (solver->jmaxLocal) * (solver->imaxLocal + 2) + 1; // send top - buf[4] = grid + (solver->imaxLocal + 2); // recv left - buf[5] = grid + (solver->imaxLocal + 2) + 1; // send left - buf[6] = grid + (solver->imaxLocal + 2) + (solver->imaxLocal + 1); // recv right - buf[7] = grid + (solver->imaxLocal + 2) + (solver->imaxLocal); // send right - - for (int i = 0; i < 2; i++) { - int tag = 0; - if (solver->jNeighbours[i] != MPI_PROC_NULL) { - tag = solver->jNeighbours[i]; - } - /* exchange ghost cells with bottom/top neighbor */ - MPI_Irecv(buf[i * 2], - 1, - solver->jBufferType, - solver->jNeighbours[i], - tag, - solver->comm, - &requests[i * 2]); - MPI_Isend(buf[(i * 2) + 1], - 1, - solver->jBufferType, - solver->jNeighbours[i], - solver->rank, - solver->comm, - &requests[i * 2 + 1]); - - tag = 0; - if (solver->iNeighbours[i] != MPI_PROC_NULL) { - tag = solver->iNeighbours[i]; - } - /* exchange ghost cells with left/right neighbor */ - MPI_Irecv(buf[i * 2 + 4], - 1, - solver->iBufferType, - solver->iNeighbours[i], - tag, - solver->comm, - &requests[i * 2 + 4]); - MPI_Isend(buf[i * 2 + 5], - 1, - solver->iBufferType, - solver->iNeighbours[i], - solver->rank, - solver->comm, - &requests[(i * 2) + 5]); - } - - MPI_Waitall(8, requests, MPI_STATUSES_IGNORE); -} - -static void shift(Solver* solver) -{ - MPI_Request requests[4] = { MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL }; - double* f = solver->f; - double* g = solver->g; - - /* shift G */ - double* buf = g + 1; - /* receive ghost cells from bottom neighbor */ - MPI_Irecv(buf, - 1, - solver->jBufferType, - solver->jNeighbours[0], - 0, - solver->comm, - &requests[0]); - - buf = g + (solver->jmaxLocal) * (solver->imaxLocal + 2) + 1; - /* send ghost cells to top neighbor */ - MPI_Isend(buf, - 1, - solver->jBufferType, - solver->jNeighbours[1], - 0, - solver->comm, - &requests[1]); - - /* shift F */ - buf = f + (solver->imaxLocal + 2); - /* receive ghost cells from left neighbor */ - MPI_Irecv(buf, - 1, - solver->iBufferType, - solver->iNeighbours[0], - 1, - solver->comm, - &requests[2]); - - buf = f + (solver->imaxLocal + 2) + (solver->imaxLocal); - /* send ghost cells to right neighbor */ - MPI_Isend(buf, - 1, - solver->iBufferType, - solver->iNeighbours[1], - 1, - solver->comm, - &requests[3]); - - MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); -} - -void debugExchange(Solver* solver) -{ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - - for (int j = 0; j < jmaxLocal + 2; j++) { - for (int i = 0; i < solver->imaxLocal + 2; i++) { - solver->p[j * (imaxLocal + 2) + i] = solver->rank + 0.01 * i + 0.0001 * j; - } - } - collectResult(solver); - /* print(solver, solver->p); */ -} - -void debugBC(Solver* solver) -{ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double* v = solver->v; - - // Northern boundary - if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc - for (int i = 1; i < imaxLocal + 1; i++) { - V(i, jmaxLocal + 1) = 10.0 + solver->rank; - } - } - - // Eastern boundary - if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc - for (int j = 1; j < jmaxLocal + 1; j++) { - V(imaxLocal + 1, j) = 20.0 + solver->rank; - } - } - - // Southern boundary - if (solver->coords[JDIM] == 0) { // set bottom bc - for (int i = 1; i < imaxLocal + 1; i++) { - V(i, 0) = 30.0 + solver->rank; - } - } - - // Western boundary - if (solver->coords[IDIM] == 0) { // set left bc - for (int j = 1; j < jmaxLocal + 1; j++) { - V(0, j) = 40.0 + solver->rank; - } - } - print(solver, solver->v); -} - -static void assembleResult(Solver* solver, - double* src, - double* dst, - int imaxLocal[], - int jmaxLocal[], - int offset[]) -{ - MPI_Request* requests; - int numRequests = 1; - - if (solver->rank == 0) { - numRequests = solver->size + 1; - } else { - numRequests = 1; - } - - requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); - - /* all ranks send their bulk array */ - MPI_Datatype bulkType; - const int ndims = 2; - int oldSizes[ndims] = { solver->jmaxLocal + 2, solver->imaxLocal + 2 }; - int newSizes[ndims] = { solver->jmaxLocal, solver->imaxLocal }; - int starts[ndims] = { 1, 1 }; - MPI_Type_create_subarray(2, - oldSizes, - newSizes, - starts, - MPI_ORDER_C, - MPI_DOUBLE, - &bulkType); - MPI_Type_commit(&bulkType); - - MPI_Isend(src, 1, bulkType, 0, 0, solver->comm, &requests[0]); - - /* rank 0 assembles the subdomains */ - if (solver->rank == 0) { - for (int i = 0; i < solver->size; i++) { - MPI_Datatype domainType; - MPI_Type_vector(jmaxLocal[i], - imaxLocal[i], - solver->imax, - MPI_DOUBLE, - &domainType); - MPI_Type_commit(&domainType); - - MPI_Irecv(dst + offset[i], - 1, - domainType, - i, - 0, - solver->comm, - &requests[i + 1]); - } - } - - MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); -} - -static int sum(int* sizes, int position) -{ - int sum = 0; - - for (int i = 0; i < position; i++) { - sum += sizes[i]; - } - - return sum; -} - -void collectResult(Solver* solver) -{ - double* Pall = NULL; - double* Uall = NULL; - double* Vall = NULL; - int offset[solver->size]; - int imaxLocal[solver->size]; - int jmaxLocal[solver->size]; - - MPI_Gather(&solver->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Gather(&solver->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); - - if (solver->rank == 0) { - Pall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); - Uall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); - Vall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); - - for (int i = 0; i < solver->size; i++) { - int coords[2]; - MPI_Cart_coords(solver->comm, i, 2, coords); - int ioffset = sum(imaxLocal, coords[0]); - int joffset = sum(jmaxLocal, coords[1]); - offset[i] = (joffset * solver->imax) + ioffset; - printf("Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Offset(i,j): %d %d\n", - i, - coords[0], - coords[1], - imaxLocal[i], - jmaxLocal[i], - ioffset, - joffset); - } - } - - /* collect P */ - assembleResult(solver, solver->p, Pall, imaxLocal, jmaxLocal, offset); - - /* collect U */ - assembleResult(solver, solver->u, Uall, imaxLocal, jmaxLocal, offset); - - /* collect V */ - assembleResult(solver, solver->v, Vall, imaxLocal, jmaxLocal, offset); - - /* write to disk */ - if (solver->rank == 0) writeResult(solver, Pall, Uall, Vall); -} - -static void printConfig(Solver* solver) -{ - if (solver->rank == 0) { - printf("Parameters for #%s#\n", solver->problem); - printf("Boundary conditions N:%d E:%d S:%d W:%d\n", - solver->bcN, - solver->bcE, - solver->bcS, - solver->bcW); - printf("\tReynolds number: %.2f\n", solver->re); - printf("\tGx Gy: %.2f %.2f\n", solver->gx, solver->gy); - printf("Geometry data:\n"); - printf("\tDomain box size (x, y): %.2f, %.2f\n", - solver->xlength, - solver->ylength); - printf("\tCells (x, y): %d, %d\n", solver->imax, solver->jmax); - printf("Timestep parameters:\n"); - printf("\tDefault stepsize: %.2f, Final time %.2f\n", solver->dt, solver->te); - printf("\tdt bound: %.6f\n", solver->dtBound); - printf("\tTau factor: %.2f\n", solver->tau); - printf("Iterative solver parameters:\n"); - printf("\tMax iterations: %d\n", solver->itermax); - printf("\tepsilon (stopping tolerance) : %f\n", solver->eps); - printf("\tgamma factor: %f\n", solver->gamma); - printf("\tomega (SOR relaxation): %f\n", solver->omega); - printf("Communication parameters:\n"); - } - for (int i = 0; i < solver->size; i++) { - if (i == solver->rank) { - printf("\tRank %d of %d\n", solver->rank, solver->size); - printf("\tNeighbours (b, t, l, r): %d, %d, %d, %d\n", - solver->jNeighbours[0], - solver->jNeighbours[1], - solver->iNeighbours[0], - solver->iNeighbours[1]); - printf("\tCoordinates %d,%d\n", solver->coords[0], solver->coords[1]); - printf("\tLocal domain size: %dx%d\n", solver->imaxLocal, solver->jmaxLocal); - fflush(stdout); - } - } -} - -void initSolver(Solver* solver, Parameter* params) -{ - solver->problem = params->name; - solver->bcN = params->bcN; - solver->bcS = params->bcS; - solver->bcW = params->bcW; - solver->bcE = params->bcE; - solver->imax = params->imax; - solver->jmax = params->jmax; - solver->xlength = params->xlength; - solver->ylength = params->ylength; - solver->dx = params->xlength / params->imax; - solver->dy = params->ylength / params->jmax; - solver->eps = params->eps; - solver->omega = params->omg; - solver->itermax = params->itermax; - solver->re = params->re; - solver->gx = params->gx; - solver->gy = params->gy; - solver->dt = params->dt; - solver->te = params->te; - solver->tau = params->tau; - solver->gamma = params->gamma; - - /* setup communication */ - MPI_Comm_rank(MPI_COMM_WORLD, &(solver->rank)); - MPI_Comm_size(MPI_COMM_WORLD, &(solver->size)); - int dims[NDIMS] = { 0, 0 }; - int periods[NDIMS] = { 0, 0 }; - MPI_Dims_create(solver->size, NDIMS, dims); - MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &solver->comm); - MPI_Cart_shift(solver->comm, - IDIM, - 1, - &solver->iNeighbours[0], - &solver->iNeighbours[1]); - MPI_Cart_shift(solver->comm, - JDIM, - 1, - &solver->jNeighbours[0], - &solver->jNeighbours[1]); - MPI_Cart_get(solver->comm, NDIMS, solver->dims, periods, solver->coords); - - solver->imaxLocal = sizeOfRank(solver->rank, dims[IDIM], solver->imax); - solver->jmaxLocal = sizeOfRank(solver->rank, dims[JDIM], solver->jmax); - - MPI_Type_contiguous(solver->imaxLocal, MPI_DOUBLE, &solver->jBufferType); - MPI_Type_commit(&solver->jBufferType); - - MPI_Type_vector(solver->jmaxLocal, - 1, - solver->imaxLocal + 2, - MPI_DOUBLE, - &solver->iBufferType); - MPI_Type_commit(&solver->iBufferType); - - /* allocate arrays */ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - size_t bytesize = (imaxLocal + 2) * (jmaxLocal + 2) * sizeof(double); - solver->u = allocate(64, bytesize); - solver->v = allocate(64, bytesize); - solver->p = allocate(64, bytesize); - solver->rhs = allocate(64, bytesize); - solver->f = allocate(64, bytesize); - solver->g = allocate(64, bytesize); - - for (int i = 0; i < (imaxLocal + 2) * (jmaxLocal + 2); i++) { - solver->u[i] = params->u_init; - solver->v[i] = params->v_init; - solver->p[i] = params->p_init; - solver->rhs[i] = 0.0; - solver->f[i] = 0.0; - solver->g[i] = 0.0; - } - - double dx = solver->dx; - double dy = solver->dy; - double inv_sqr_sum = 1.0 / (dx * dx) + 1.0 / (dy * dy); - solver->dtBound = 0.5 * solver->re * 1.0 / inv_sqr_sum; -#ifdef VERBOSE - printConfig(solver); -#endif -} - -void computeRHS(Solver* solver) -{ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double idx = 1.0 / solver->dx; - double idy = 1.0 / solver->dy; - double idt = 1.0 / solver->dt; - double* rhs = solver->rhs; - double* f = solver->f; - double* g = solver->g; - - shift(solver); - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) * - idt; - } - } -} - -int solve(Solver* solver) -{ - int imax = solver->imax; - int jmax = solver->jmax; - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double eps = solver->eps; - int itermax = solver->itermax; - double dx2 = solver->dx * solver->dx; - double dy2 = solver->dy * solver->dy; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); - double* p = solver->p; - double* rhs = solver->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - exchange(solver, p); - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - - double r = RHS(i, j) - - ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + - (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - - P(i, j) -= (factor * r); - res += (r * r); - } - } - - if (solver->coords[JDIM] == 0) { // set bottom bc - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, 0) = P(i, 1); - } - } - - if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, jmaxLocal + 1) = P(i, jmaxLocal); - } - } - - if (solver->coords[IDIM] == 0) { // set left bc - for (int j = 1; j < jmaxLocal + 1; j++) { - P(0, j) = P(1, j); - } - } - - if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc - for (int j = 1; j < jmaxLocal + 1; j++) { - P(imaxLocal + 1, j) = P(imaxLocal, j); - } - } - - MPI_Allreduce(MPI_IN_PLACE, &res, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - res = res / (double)(imax * jmax); -#ifdef DEBUG - if (solver->rank == 0) { - printf("%d Residuum: %e\n", it, res); - } -#endif - it++; - } - -#ifdef VERBOSE - if (solver->rank == 0) { - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); - } -#endif - if (res < eps) { - return 0; - } else { - return 1; - } -} - -static double maxElement(Solver* solver, double* m) -{ - int size = (solver->imaxLocal + 2) * (solver->jmaxLocal + 2); - double maxval = DBL_MIN; - - for (int i = 0; i < size; i++) { - maxval = MAX(maxval, fabs(m[i])); - } - - MPI_Allreduce(MPI_IN_PLACE, &maxval, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - return maxval; -} - -void computeTimestep(Solver* solver) -{ - double dt = solver->dtBound; - double dx = solver->dx; - double dy = solver->dy; - double umax = maxElement(solver, solver->u); - double vmax = maxElement(solver, solver->v); - - if (umax > 0) { - dt = (dt > dx / umax) ? dx / umax : dt; - } - if (vmax > 0) { - dt = (dt > dy / vmax) ? dy / vmax : dt; - } - - solver->dt = dt * solver->tau; -} - -void setBoundaryConditions(Solver* solver) -{ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double* u = solver->u; - double* v = solver->v; - - // Northern boundary - if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc - switch (solver->bcN) { - case NOSLIP: - for (int i = 1; i < imaxLocal + 1; i++) { - V(i, jmaxLocal) = 0.0; - U(i, jmaxLocal + 1) = -U(i, jmaxLocal); - } - break; - case SLIP: - for (int i = 1; i < imaxLocal + 1; i++) { - V(i, jmaxLocal) = 0.0; - U(i, jmaxLocal + 1) = U(i, jmaxLocal); - } - break; - case OUTFLOW: - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, jmaxLocal + 1) = U(i, jmaxLocal); - V(i, jmaxLocal) = V(i, jmaxLocal - 1); - } - break; - case PERIODIC: - break; - } - } - - // Southern boundary - if (solver->coords[JDIM] == 0) { // set bottom bc - switch (solver->bcS) { - case NOSLIP: - for (int i = 1; i < imaxLocal + 1; i++) { - V(i, 0) = 0.0; - U(i, 0) = -U(i, 1); - } - break; - case SLIP: - for (int i = 1; i < imaxLocal + 1; i++) { - V(i, 0) = 0.0; - U(i, 0) = U(i, 1); - } - break; - case OUTFLOW: - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, 0) = U(i, 1); - V(i, 0) = V(i, 1); - } - break; - case PERIODIC: - break; - } - } - - // Eastern boundary - if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc - switch (solver->bcE) { - case NOSLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imaxLocal, j) = 0.0; - V(imaxLocal + 1, j) = -V(imaxLocal, j); - } - break; - case SLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imaxLocal, j) = 0.0; - V(imaxLocal + 1, j) = V(imaxLocal, j); - } - break; - case OUTFLOW: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imaxLocal, j) = U(imaxLocal - 1, j); - V(imaxLocal + 1, j) = V(imaxLocal, j); - } - break; - case PERIODIC: - break; - } - } - - // Western boundary - if (solver->coords[IDIM] == 0) { // set left bc - switch (solver->bcW) { - case NOSLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j) = 0.0; - V(0, j) = -V(1, j); - } - break; - case SLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j) = 0.0; - V(0, j) = V(1, j); - } - break; - case OUTFLOW: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j) = U(1, j); - V(0, j) = V(1, j); - } - break; - case PERIODIC: - break; - } - } -} - -void setSpecialBoundaryCondition(Solver* solver) -{ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double* u = solver->u; - - if (strcmp(solver->problem, "dcavity") == 0) { - if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal); - } - } - } else if (strcmp(solver->problem, "canal") == 0) { - if (solver->coords[IDIM] == 0) { // set left bc - double ylength = solver->ylength; - double dy = solver->dy; - int rest = solver->jmax % solver->size; - int yc = solver->rank * (solver->jmax / solver->size) + - MIN(rest, solver->rank); - double ys = dy * (yc + 0.5); - double y; - - /* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */ - - for (int j = 1; j < jmaxLocal + 1; j++) { - y = ys + dy * (j - 0.5); - U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); - } - } - } - /* print(solver, solver->u); */ -} - -void computeFG(Solver* solver) -{ - double* u = solver->u; - double* v = solver->v; - double* f = solver->f; - double* g = solver->g; - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double gx = solver->gx; - double gy = solver->gy; - double gamma = solver->gamma; - double dt = solver->dt; - double inverseRe = 1.0 / solver->re; - double inverseDx = 1.0 / solver->dx; - double inverseDy = 1.0 / solver->dy; - double du2dx, dv2dy, duvdx, duvdy; - double du2dx2, du2dy2, dv2dx2, dv2dy2; - - exchange(solver, u); - exchange(solver, v); - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - du2dx = inverseDx * 0.25 * - ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - - (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + - fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); - - duvdy = inverseDy * 0.25 * - ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - - (V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + - fabs(V(i, j - 1) + V(i + 1, j - 1)) * - (U(i, j) - U(i, j - 1))); - - du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); - du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); - F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); - - duvdx = inverseDx * 0.25 * - ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - - (U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + - fabs(U(i - 1, j) + U(i - 1, j + 1)) * - (V(i, j) - V(i - 1, j))); - - dv2dy = inverseDy * 0.25 * - ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - - (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + - fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); - - dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); - dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); - G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); - } - } - - /* ----------------------------- boundary of F --------------------------- */ - if (solver->coords[IDIM] == 0) { // set left bc - for (int j = 1; j < jmaxLocal + 1; j++) { - F(0, j) = U(0, j); - } - } - - if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc - for (int j = 1; j < jmaxLocal + 1; j++) { - F(imaxLocal, j) = U(imaxLocal, j); - } - } - - /* ----------------------------- boundary of G --------------------------- */ - if (solver->coords[JDIM] == 0) { // set bottom bc - for (int i = 1; i < imaxLocal + 1; i++) { - G(i, 0) = V(i, 0); - } - } - - if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc - for (int i = 1; i < imaxLocal + 1; i++) { - G(i, jmaxLocal) = V(i, jmaxLocal); - } - } -} - -void adaptUV(Solver* solver) -{ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double* p = solver->p; - double* u = solver->u; - double* v = solver->v; - double* f = solver->f; - double* g = solver->g; - double factorX = solver->dt / solver->dx; - double factorY = solver->dt / solver->dy; - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; - V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; - } - } -} - -void writeResult(Solver* solver, double* p, double* u, double* v) -{ - int imax = solver->imax; - int jmax = solver->jmax; - double dx = solver->dx; - double dy = solver->dy; - double x = 0.0, y = 0.0; - - FILE* fp; - fp = fopen("pressure.dat", "w"); - - if (fp == NULL) { - printf("Error!\n"); - exit(EXIT_FAILURE); - } - - for (int j = 1; j < jmax; j++) { - y = (double)(j - 0.5) * dy; - for (int i = 1; i < imax; i++) { - x = (double)(i - 0.5) * dx; - fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]); - } - fprintf(fp, "\n"); - } - - fclose(fp); - - fp = fopen("velocity.dat", "w"); - - if (fp == NULL) { - printf("Error!\n"); - exit(EXIT_FAILURE); - } - - for (int j = 1; j < jmax; j++) { - y = dy * (j - 0.5); - for (int i = 1; i < imax; i++) { - x = dx * (i - 0.5); - double vel_u = (u[j * (imax) + i] + u[j * (imax) + (i - 1)]) / 2.0; - double vel_v = (v[j * (imax) + i] + v[(j - 1) * (imax) + i]) / 2.0; - double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); - fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); - } - } - - fclose(fp); -} diff --git a/BasicSolver/2D-mpi-v2/src/solver.h b/BasicSolver/2D-mpi-v2/src/solver.h deleted file mode 100644 index 1d3f7c9..0000000 --- a/BasicSolver/2D-mpi-v2/src/solver.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#ifndef __SOLVER_H_ -#define __SOLVER_H_ -#include "parameter.h" -#include - -enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; - -typedef struct { - /* geometry and grid information */ - double dx, dy; - int imax, jmax; - double xlength, ylength; - /* arrays */ - double *p, *rhs; - double *f, *g; - double *u, *v; - /* parameters */ - double eps, omega; - double re, tau, gamma; - double gx, gy; - /* time stepping */ - int itermax; - double dt, te; - double dtBound; - char* problem; - int bcN, bcS, bcW, bcE; - /* mpi */ - int rank; - int size; - MPI_Comm comm; - MPI_Datatype iBufferType, jBufferType; - int iNeighbours[2], jNeighbours[2]; - int coords[2], dims[2]; - int imaxLocal, jmaxLocal; -} Solver; - -void initSolver(Solver*, Parameter*); -void computeRHS(Solver*); -int solve(Solver*); -void computeTimestep(Solver*); -void setBoundaryConditions(Solver*); -void setSpecialBoundaryCondition(Solver*); -void computeFG(Solver*); -void adaptUV(Solver*); -void collectResult(Solver*); -void writeResult(Solver*, double*, double*, double*); -void debugExchange(Solver*); -void debugBC(Solver*); -void print(Solver*, double*); -#endif diff --git a/BasicSolver/2D-mpi-v2/vector.plot b/BasicSolver/2D-mpi-v2/vector.plot deleted file mode 100644 index 0934ab2..0000000 --- a/BasicSolver/2D-mpi-v2/vector.plot +++ /dev/null @@ -1,5 +0,0 @@ -set terminal png size 1800,768 enhanced font ,12 -set output 'velocity.png' -set datafile separator whitespace - -plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/BasicSolver/2D-mpi-v3/canal.par b/BasicSolver/2D-mpi-v3/canal.par deleted file mode 100644 index 80dfa19..0000000 --- a/BasicSolver/2D-mpi-v3/canal.par +++ /dev/null @@ -1,46 +0,0 @@ -#============================================================================== -# Laminar Canal Flow -#============================================================================== - -# Problem specific Data: -# --------------------- - -name canal # name of flow setup - -bcN 1 # flags for boundary conditions -bcE 3 # 1 = no-slip 3 = outflow -bcS 1 # 2 = free-slip 4 = periodic -bcW 3 # - -gx 0.0 # Body forces (e.g. gravity) -gy 0.0 # - -re 100.0 # Reynolds number - -u_init 1.0 # initial value for velocity in x-direction -v_init 0.0 # initial value for velocity in y-direction -p_init 0.0 # initial value for pressure - -# Geometry Data: -# ------------- - -xlength 30.0 # domain size in x-direction -ylength 4.0 # domain size in y-direction -imax 200 # number of interior cells in x-direction -jmax 50 # number of interior cells in y-direction - -# Time Data: -# --------- - -te 100.0 # final time -dt 0.02 # time stepsize -tau 0.5 # safety factor for time stepsize control (<0 constant delt) - -# Pressure Iteration Data: -# ----------------------- - -itermax 500 # maximal number of pressure iteration in one time step -eps 0.00001 # stopping tolerance for pressure iteration -omg 1.8 # relaxation parameter for SOR iteration -gamma 0.9 # upwind differencing factor gamma -#=============================================================================== diff --git a/BasicSolver/2D-mpi-v3/include_CLANG.mk b/BasicSolver/2D-mpi-v3/include_CLANG.mk deleted file mode 100644 index 1d17c52..0000000 --- a/BasicSolver/2D-mpi-v3/include_CLANG.mk +++ /dev/null @@ -1,16 +0,0 @@ -CC = mpicc -GCC = cc -LINKER = $(CC) - -ifeq ($(ENABLE_OPENMP),true) -OPENMP = -fopenmp -#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp -LIBS = # -lomp -endif - -VERSION = --version -CFLAGS = -Ofast -std=c99 $(OPENMP) -#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG -LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE# -DDEBUG -INCLUDES = -I/usr/local/include diff --git a/BasicSolver/2D-mpi-v3/include_ICC.mk b/BasicSolver/2D-mpi-v3/include_ICC.mk deleted file mode 100644 index f85d836..0000000 --- a/BasicSolver/2D-mpi-v3/include_ICC.mk +++ /dev/null @@ -1,14 +0,0 @@ -CC = mpiicc -GCC = gcc -LINKER = $(CC) - -ifeq ($(ENABLE_OPENMP),true) -OPENMP = -qopenmp -endif - -VERSION = --version -CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) -LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE -INCLUDES = -LIBS = diff --git a/BasicSolver/2D-mpi-v3/src/affinity.h b/BasicSolver/2D-mpi-v3/src/affinity.h deleted file mode 100644 index d844fe5..0000000 --- a/BasicSolver/2D-mpi-v3/src/affinity.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifndef AFFINITY_H -#define AFFINITY_H - -extern int affinity_getProcessorId(); -extern void affinity_pinProcess(int); -extern void affinity_pinThread(int); - -#endif /*AFFINITY_H*/ diff --git a/BasicSolver/2D-mpi-v3/src/main.c b/BasicSolver/2D-mpi-v3/src/main.c deleted file mode 100644 index 34de302..0000000 --- a/BasicSolver/2D-mpi-v3/src/main.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include - -#include "parameter.h" -#include "progress.h" -#include "solver.h" -#include "timing.h" -#include - -int main(int argc, char** argv) -{ - int rank; - double S, E; - Parameter params; - Solver solver; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - initParameter(¶ms); - - if (argc != 2) { - printf("Usage: %s \n", argv[0]); - exit(EXIT_SUCCESS); - } - - readParameter(¶ms, argv[1]); - if (rank == 0) { - printParameter(¶ms); - } - initSolver(&solver, ¶ms); - initProgress(solver.te); - - double tau = solver.tau; - double te = solver.te; - double t = 0.0; - - S = getTimeStamp(); - while (t <= te) { - if (tau > 0.0) { - computeTimestep(&solver); - } - - setBoundaryConditions(&solver); - setSpecialBoundaryCondition(&solver); - computeFG(&solver); - computeRHS(&solver); - solve(&solver); - adaptUV(&solver); - t += solver.dt; - -#ifdef VERBOSE - if (rank == 0) { - printf("TIME %f , TIMESTEP %f\n", t, solver.dt); - } -#else - printProgress(t); -#endif - } - E = getTimeStamp(); - stopProgress(); - if (rank == 0) { - printf("Solution took %.2fs\n", E - S); - } - collectResult(&solver); - - MPI_Finalize(); - return EXIT_SUCCESS; -} diff --git a/BasicSolver/2D-mpi-v3/src/progress.c b/BasicSolver/2D-mpi-v3/src/progress.c deleted file mode 100644 index 31a8a90..0000000 --- a/BasicSolver/2D-mpi-v3/src/progress.c +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include - -#include "progress.h" - -static double _end; -static int _current; -static int _rank = -1; - -void initProgress(double end) -{ - MPI_Comm_rank(MPI_COMM_WORLD, &_rank); - _end = end; - _current = 0; - - if (_rank == 0) { - printf("[ ]"); - fflush(stdout); - } -} - -void printProgress(double current) -{ - if (_rank == 0) { - int new = (int)rint((current / _end) * 10.0); - - if (new > _current) { - char progress[11]; - _current = new; - progress[0] = 0; - - for (int i = 0; i < 10; i++) { - if (i < _current) { - sprintf(progress + strlen(progress), "#"); - } else { - sprintf(progress + strlen(progress), " "); - } - } - printf("\r[%s]", progress); - } - fflush(stdout); - } -} - -void stopProgress() -{ - if (_rank == 0) { - printf("\n"); - fflush(stdout); - } -} diff --git a/BasicSolver/2D-mpi-v3/src/solver.c b/BasicSolver/2D-mpi-v3/src/solver.c deleted file mode 100644 index 4721a95..0000000 --- a/BasicSolver/2D-mpi-v3/src/solver.c +++ /dev/null @@ -1,833 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include -#include - -#include "allocate.h" -#include "parameter.h" -#include "solver.h" -#include "util.h" - -#define P(i, j) p[(j) * (imaxLocal + 2) + (i)] -#define F(i, j) f[(j) * (imaxLocal + 2) + (i)] -#define G(i, j) g[(j) * (imaxLocal + 2) + (i)] -#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] -#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] -#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)] - -#define IDIM 0 -#define JDIM 1 - -static int sizeOfRank(int rank, int size, int N) -{ - return N / size + ((N % size > rank) ? 1 : 0); -} - -void print(Solver* solver, double* grid) -{ - int imaxLocal = solver->imaxLocal; - - for (int i = 0; i < solver->size; i++) { - if (i == solver->rank) { - printf( - "### RANK %d #######################################################\n", - solver->rank); - for (int j = 0; j < solver->jmaxLocal + 2; j++) { - printf("%02d: ", j); - for (int i = 0; i < solver->imaxLocal + 2; i++) { - printf("%12.8f ", grid[j * (imaxLocal + 2) + i]); - } - printf("\n"); - } - fflush(stdout); - } - MPI_Barrier(MPI_COMM_WORLD); - } -} - -static void exchange(Solver* solver, double* grid) -{ - int counts[4] = { 1, 1, 1, 1 }; - - MPI_Neighbor_alltoallw(grid, - counts, - solver->sdispls, - solver->bufferTypes, - grid, - counts, - solver->rdispls, - solver->bufferTypes, - solver->comm); -} - -static void shift(Solver* solver) -{ - MPI_Request requests[4] = { MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL }; - double* f = solver->f; - double* g = solver->g; - - /* shift G */ - double* buf = g + 1; - /* receive ghost cells from bottom neighbor */ - MPI_Irecv(buf, - 1, - solver->bufferTypes[2], - solver->jNeighbours[0], - 0, - solver->comm, - &requests[0]); - - buf = g + (solver->jmaxLocal) * (solver->imaxLocal + 2) + 1; - /* send ghost cells to top neighbor */ - MPI_Isend(buf, - 1, - solver->bufferTypes[2], - solver->jNeighbours[1], - 0, - solver->comm, - &requests[1]); - - /* shift F */ - buf = f + (solver->imaxLocal + 2); - /* receive ghost cells from left neighbor */ - MPI_Irecv(buf, - 1, - solver->bufferTypes[0], - solver->iNeighbours[0], - 1, - solver->comm, - &requests[2]); - - buf = f + (solver->imaxLocal + 2) + (solver->imaxLocal); - /* send ghost cells to right neighbor */ - MPI_Isend(buf, - 1, - solver->bufferTypes[0], - solver->iNeighbours[1], - 1, - solver->comm, - &requests[3]); - - MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); -} - -void debugExchange(Solver* solver) -{ - for (int i = 0; i < (solver->imaxLocal + 2) * (solver->jmaxLocal + 2); i++) { - solver->p[i] = solver->rank; - } - exchange(solver, solver->p); - print(solver, solver->p); -} - -static void assembleResult(Solver* solver, - double* src, - double* dst, - int imaxLocal[], - int jmaxLocal[], - int offset[]) -{ - MPI_Request* requests; - int numRequests = 1; - - if (solver->rank == 0) { - numRequests = solver->size + 1; - } else { - numRequests = 1; - } - - requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); - - /* all ranks send their bulk array */ - MPI_Datatype bulkType; - const int ndims = 2; - int oldSizes[ndims] = { solver->jmaxLocal + 2, solver->imaxLocal + 2 }; - int newSizes[ndims] = { solver->jmaxLocal, solver->imaxLocal }; - int starts[ndims] = { 1, 1 }; - MPI_Type_create_subarray(2, - oldSizes, - newSizes, - starts, - MPI_ORDER_C, - MPI_DOUBLE, - &bulkType); - MPI_Type_commit(&bulkType); - - MPI_Isend(src, 1, bulkType, 0, 0, solver->comm, &requests[0]); - - /* rank 0 assembles the subdomains */ - if (solver->rank == 0) { - for (int i = 0; i < solver->size; i++) { - MPI_Datatype domainType; - MPI_Type_vector(jmaxLocal[i], - imaxLocal[i], - solver->imax, - MPI_DOUBLE, - &domainType); - MPI_Type_commit(&domainType); - - MPI_Irecv(dst + offset[i], - 1, - domainType, - i, - 0, - solver->comm, - &requests[i + 1]); - } - } - - MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); -} - -static int sum(int* sizes, int position) -{ - int sum = 0; - - for (int i = 0; i < position; i++) { - sum += sizes[i]; - } - - return sum; -} - -void collectResult(Solver* solver) -{ - double* Pall = NULL; - double* Uall = NULL; - double* Vall = NULL; - int offset[solver->size]; - int imaxLocal[solver->size]; - int jmaxLocal[solver->size]; - - MPI_Gather(&solver->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Gather(&solver->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); - - if (solver->rank == 0) { - Pall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); - Uall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); - Vall = allocate(64, (solver->imax) * (solver->jmax) * sizeof(double)); - - for (int i = 0; i < solver->size; i++) { - int coords[2]; - MPI_Cart_coords(solver->comm, i, 2, coords); - int ioffset = sum(imaxLocal, coords[0]); - int joffset = sum(jmaxLocal, coords[1]); - offset[i] = (joffset * solver->imax) + ioffset; - printf("Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Offset(i,j): %d %d\n", - i, - coords[0], - coords[1], - imaxLocal[i], - jmaxLocal[i], - ioffset, - joffset); - } - } - - /* collect P */ - assembleResult(solver, solver->p, Pall, imaxLocal, jmaxLocal, offset); - - /* collect U */ - assembleResult(solver, solver->u, Uall, imaxLocal, jmaxLocal, offset); - - /* collect V */ - assembleResult(solver, solver->v, Vall, imaxLocal, jmaxLocal, offset); - - /* write to disk */ - if (solver->rank == 0) writeResult(solver, Pall, Uall, Vall); -} - -static void printConfig(Solver* solver) -{ - if (solver->rank == 0) { - printf("Parameters for #%s#\n", solver->problem); - printf("Boundary conditions Top:%d Bottom:%d Left:%d Right:%d\n", - solver->bcTop, - solver->bcBottom, - solver->bcLeft, - solver->bcRight); - printf("\tReynolds number: %.2f\n", solver->re); - printf("\tGx Gy: %.2f %.2f\n", solver->gx, solver->gy); - printf("Geometry data:\n"); - printf("\tDomain box size (x, y): %.2f, %.2f\n", - solver->xlength, - solver->ylength); - printf("\tCells (x, y): %d, %d\n", solver->imax, solver->jmax); - printf("Timestep parameters:\n"); - printf("\tDefault stepsize: %.2f, Final time %.2f\n", solver->dt, solver->te); - printf("\tdt bound: %.6f\n", solver->dtBound); - printf("\tTau factor: %.2f\n", solver->tau); - printf("Iterative solver parameters:\n"); - printf("\tMax iterations: %d\n", solver->itermax); - printf("\tepsilon (stopping tolerance) : %f\n", solver->eps); - printf("\tgamma factor: %f\n", solver->gamma); - printf("\tomega (SOR relaxation): %f\n", solver->omega); - printf("Communication parameters:\n"); - } - for (int i = 0; i < solver->size; i++) { - if (i == solver->rank) { - printf("\tRank %d of %d\n", solver->rank, solver->size); - printf("\tNeighbours (b, t, l, r): %d, %d, %d, %d\n", - solver->jNeighbours[0], - solver->jNeighbours[1], - solver->iNeighbours[0], - solver->iNeighbours[1]); - printf("\tCoordinates %d,%d\n", solver->coords[0], solver->coords[1]); - printf("\tLocal domain size: %dx%d\n", solver->imaxLocal, solver->jmaxLocal); - fflush(stdout); - } - } -} - -void initSolver(Solver* solver, Parameter* params) -{ - solver->problem = params->name; - solver->bcTop = params->bcTop; - solver->bcBottom = params->bcBottom; - solver->bcLeft = params->bcLeft; - solver->bcRight = params->bcRight; - solver->imax = params->imax; - solver->jmax = params->jmax; - solver->xlength = params->xlength; - solver->ylength = params->ylength; - solver->dx = params->xlength / params->imax; - solver->dy = params->ylength / params->jmax; - solver->eps = params->eps; - solver->omega = params->omg; - solver->itermax = params->itermax; - solver->re = params->re; - solver->gx = params->gx; - solver->gy = params->gy; - solver->dt = params->dt; - solver->te = params->te; - solver->tau = params->tau; - solver->gamma = params->gamma; - - /* setup communication */ - MPI_Comm_rank(MPI_COMM_WORLD, &(solver->rank)); - MPI_Comm_size(MPI_COMM_WORLD, &(solver->size)); - int dims[NDIMS] = { 0, 0 }; - int periods[NDIMS] = { 0, 0 }; - MPI_Dims_create(solver->size, NDIMS, dims); - MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &solver->comm); - MPI_Cart_shift(solver->comm, - IDIM, - 1, - &solver->iNeighbours[0], - &solver->iNeighbours[1]); - MPI_Cart_shift(solver->comm, - JDIM, - 1, - &solver->jNeighbours[0], - &solver->jNeighbours[1]); - MPI_Cart_get(solver->comm, NDIMS, solver->dims, periods, solver->coords); - - solver->imaxLocal = sizeOfRank(solver->coords[IDIM], dims[IDIM], solver->imax); - solver->jmaxLocal = sizeOfRank(solver->coords[JDIM], dims[JDIM], solver->jmax); - - MPI_Datatype jBufferType; - MPI_Type_contiguous(solver->imaxLocal, MPI_DOUBLE, &jBufferType); - MPI_Type_commit(&jBufferType); - - MPI_Datatype iBufferType; - MPI_Type_vector(solver->jmaxLocal, - 1, - solver->imaxLocal + 2, - MPI_DOUBLE, - &iBufferType); - MPI_Type_commit(&iBufferType); - - // in the order of the dimensions i->0, j->1 - // first negative direction, then positive direction - size_t dblsize = sizeof(double); - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - solver->bufferTypes[0] = iBufferType; // left - solver->bufferTypes[1] = iBufferType; // right - solver->bufferTypes[2] = jBufferType; // bottom - solver->bufferTypes[3] = jBufferType; // top - - solver->sdispls[0] = ((imaxLocal + 2) + 1) * dblsize; // send left - solver->sdispls[1] = ((imaxLocal + 2) + imaxLocal) * dblsize; // send right - solver->sdispls[2] = ((imaxLocal + 2) + 1) * dblsize; // send bottom - solver->sdispls[3] = ((jmaxLocal) * (imaxLocal + 2) + 1) * dblsize; // send top - - solver->rdispls[0] = (imaxLocal + 2) * dblsize; // recv left - solver->rdispls[1] = ((imaxLocal + 2) + (imaxLocal + 1)) * dblsize; // recv right - solver->rdispls[2] = 1 * dblsize; // recv bottom - solver->rdispls[3] = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * dblsize; // recv top - - /* allocate arrays */ - size_t bytesize = (imaxLocal + 2) * (jmaxLocal + 2) * sizeof(double); - solver->u = allocate(64, bytesize); - solver->v = allocate(64, bytesize); - solver->p = allocate(64, bytesize); - solver->rhs = allocate(64, bytesize); - solver->f = allocate(64, bytesize); - solver->g = allocate(64, bytesize); - - for (int i = 0; i < (imaxLocal + 2) * (jmaxLocal + 2); i++) { - solver->u[i] = params->u_init; - solver->v[i] = params->v_init; - solver->p[i] = params->p_init; - solver->rhs[i] = 0.0; - solver->f[i] = 0.0; - solver->g[i] = 0.0; - } - - double dx = solver->dx; - double dy = solver->dy; - double inv_sqr_sum = 1.0 / (dx * dx) + 1.0 / (dy * dy); - solver->dtBound = 0.5 * solver->re * 1.0 / inv_sqr_sum; -#ifdef VERBOSE - printConfig(solver); -#endif -} - -void computeRHS(Solver* solver) -{ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double idx = 1.0 / solver->dx; - double idy = 1.0 / solver->dy; - double idt = 1.0 / solver->dt; - double* rhs = solver->rhs; - double* f = solver->f; - double* g = solver->g; - - shift(solver); - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) * - idt; - } - } -} - -int solve(Solver* solver) -{ - int imax = solver->imax; - int jmax = solver->jmax; - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double eps = solver->eps; - int itermax = solver->itermax; - double dx2 = solver->dx * solver->dx; - double dy2 = solver->dy * solver->dy; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - // identical to 1/((2/dx2)+(2/dy2)) - double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); - double* p = solver->p; - double* rhs = solver->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - exchange(solver, p); - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - - double r = RHS(i, j) - - ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + - (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - - P(i, j) -= (factor * r); - res += (r * r); - } - } - - if (solver->coords[JDIM] == 0) { // set bottom bc - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, 0) = P(i, 1); - } - } - - if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, jmaxLocal + 1) = P(i, jmaxLocal); - } - } - - if (solver->coords[IDIM] == 0) { // set left bc - for (int j = 1; j < jmaxLocal + 1; j++) { - P(0, j) = P(1, j); - } - } - - if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc - for (int j = 1; j < jmaxLocal + 1; j++) { - P(imaxLocal + 1, j) = P(imaxLocal, j); - } - } - - MPI_Allreduce(MPI_IN_PLACE, &res, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - res = res / (double)(imax * jmax); -#ifdef DEBUG - if (solver->rank == 0) { - printf("%d Residuum: %e\n", it, res); - } -#endif - it++; - } - -#ifdef VERBOSE - if (solver->rank == 0) { - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); - } -#endif - if (res < eps) { - return 0; - } else { - return 1; - } -} - -static double maxElement(Solver* solver, double* m) -{ - int size = (solver->imaxLocal + 2) * (solver->jmaxLocal + 2); - double maxval = DBL_MIN; - - for (int i = 0; i < size; i++) { - maxval = MAX(maxval, fabs(m[i])); - } - - MPI_Allreduce(MPI_IN_PLACE, &maxval, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - return maxval; -} - -void computeTimestep(Solver* solver) -{ - double dt = solver->dtBound; - double dx = solver->dx; - double dy = solver->dy; - double umax = maxElement(solver, solver->u); - double vmax = maxElement(solver, solver->v); - - if (umax > 0) { - dt = (dt > dx / umax) ? dx / umax : dt; - } - if (vmax > 0) { - dt = (dt > dy / vmax) ? dy / vmax : dt; - } - - solver->dt = dt * solver->tau; -} - -void setBoundaryConditions(Solver* solver) -{ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double* u = solver->u; - double* v = solver->v; - - // Northern boundary - if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc - switch (solver->bcTop) { - case NOSLIP: - for (int i = 1; i < imaxLocal + 1; i++) { - V(i, jmaxLocal) = 0.0; - U(i, jmaxLocal + 1) = -U(i, jmaxLocal); - } - break; - case SLIP: - for (int i = 1; i < imaxLocal + 1; i++) { - V(i, jmaxLocal) = 0.0; - U(i, jmaxLocal + 1) = U(i, jmaxLocal); - } - break; - case OUTFLOW: - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, jmaxLocal + 1) = U(i, jmaxLocal); - V(i, jmaxLocal) = V(i, jmaxLocal - 1); - } - break; - case PERIODIC: - break; - } - } - - // Southern boundary - if (solver->coords[JDIM] == 0) { // set bottom bc - switch (solver->bcBottom) { - case NOSLIP: - for (int i = 1; i < imaxLocal + 1; i++) { - V(i, 0) = 0.0; - U(i, 0) = -U(i, 1); - } - break; - case SLIP: - for (int i = 1; i < imaxLocal + 1; i++) { - V(i, 0) = 0.0; - U(i, 0) = U(i, 1); - } - break; - case OUTFLOW: - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, 0) = U(i, 1); - V(i, 0) = V(i, 1); - } - break; - case PERIODIC: - break; - } - } - - // Eastern boundary - if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc - switch (solver->bcRight) { - case NOSLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imaxLocal, j) = 0.0; - V(imaxLocal + 1, j) = -V(imaxLocal, j); - } - break; - case SLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imaxLocal, j) = 0.0; - V(imaxLocal + 1, j) = V(imaxLocal, j); - } - break; - case OUTFLOW: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imaxLocal, j) = U(imaxLocal - 1, j); - V(imaxLocal + 1, j) = V(imaxLocal, j); - } - break; - case PERIODIC: - break; - } - } - - // Western boundary - if (solver->coords[IDIM] == 0) { // set left bc - switch (solver->bcLeft) { - case NOSLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j) = 0.0; - V(0, j) = -V(1, j); - } - break; - case SLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j) = 0.0; - V(0, j) = V(1, j); - } - break; - case OUTFLOW: - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j) = U(1, j); - V(0, j) = V(1, j); - } - break; - case PERIODIC: - break; - } - } -} - -void setSpecialBoundaryCondition(Solver* solver) -{ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double* u = solver->u; - - if (strcmp(solver->problem, "dcavity") == 0) { - if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal); - } - } - } else if (strcmp(solver->problem, "canal") == 0) { - if (solver->coords[IDIM] == 0) { // set left bc - double ylength = solver->ylength; - double dy = solver->dy; - int rest = solver->jmax % solver->size; - int yc = solver->rank * (solver->jmax / solver->size) + - MIN(rest, solver->rank); - double ys = dy * (yc + 0.5); - double y; - - /* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */ - - for (int j = 1; j < jmaxLocal + 1; j++) { - y = ys + dy * (j - 0.5); - U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); - } - } - } - /* print(solver, solver->u); */ -} - -void computeFG(Solver* solver) -{ - double* u = solver->u; - double* v = solver->v; - double* f = solver->f; - double* g = solver->g; - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double gx = solver->gx; - double gy = solver->gy; - double gamma = solver->gamma; - double dt = solver->dt; - double inverseRe = 1.0 / solver->re; - double inverseDx = 1.0 / solver->dx; - double inverseDy = 1.0 / solver->dy; - double du2dx, dv2dy, duvdx, duvdy; - double du2dx2, du2dy2, dv2dx2, dv2dy2; - - exchange(solver, u); - exchange(solver, v); - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - du2dx = inverseDx * 0.25 * - ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - - (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + - fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); - - duvdy = inverseDy * 0.25 * - ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - - (V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + - fabs(V(i, j - 1) + V(i + 1, j - 1)) * - (U(i, j) - U(i, j - 1))); - - du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); - du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); - F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); - - duvdx = inverseDx * 0.25 * - ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - - (U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + - fabs(U(i - 1, j) + U(i - 1, j + 1)) * - (V(i, j) - V(i - 1, j))); - - dv2dy = inverseDy * 0.25 * - ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - - (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + - fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); - - dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); - dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); - G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); - } - } - - /* ----------------------------- boundary of F --------------------------- */ - if (solver->coords[IDIM] == 0) { // set left bc - for (int j = 1; j < jmaxLocal + 1; j++) { - F(0, j) = U(0, j); - } - } - - if (solver->coords[IDIM] == (solver->dims[IDIM] - 1)) { // set right bc - for (int j = 1; j < jmaxLocal + 1; j++) { - F(imaxLocal, j) = U(imaxLocal, j); - } - } - - /* ----------------------------- boundary of G --------------------------- */ - if (solver->coords[JDIM] == 0) { // set bottom bc - for (int i = 1; i < imaxLocal + 1; i++) { - G(i, 0) = V(i, 0); - } - } - - if (solver->coords[JDIM] == (solver->dims[JDIM] - 1)) { // set top bc - for (int i = 1; i < imaxLocal + 1; i++) { - G(i, jmaxLocal) = V(i, jmaxLocal); - } - } -} - -void adaptUV(Solver* solver) -{ - int imaxLocal = solver->imaxLocal; - int jmaxLocal = solver->jmaxLocal; - double* p = solver->p; - double* u = solver->u; - double* v = solver->v; - double* f = solver->f; - double* g = solver->g; - double factorX = solver->dt / solver->dx; - double factorY = solver->dt / solver->dy; - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; - V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; - } - } -} - -void writeResult(Solver* solver, double* p, double* u, double* v) -{ - int imax = solver->imax; - int jmax = solver->jmax; - double dx = solver->dx; - double dy = solver->dy; - double x = 0.0, y = 0.0; - - FILE* fp; - fp = fopen("pressure.dat", "w"); - - if (fp == NULL) { - printf("Error!\n"); - exit(EXIT_FAILURE); - } - - for (int j = 1; j < jmax; j++) { - y = (double)(j - 0.5) * dy; - for (int i = 1; i < imax; i++) { - x = (double)(i - 0.5) * dx; - fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]); - } - fprintf(fp, "\n"); - } - - fclose(fp); - - fp = fopen("velocity.dat", "w"); - - if (fp == NULL) { - printf("Error!\n"); - exit(EXIT_FAILURE); - } - - for (int j = 1; j < jmax; j++) { - y = dy * (j - 0.5); - for (int i = 1; i < imax; i++) { - x = dx * (i - 0.5); - double vel_u = (u[j * (imax) + i] + u[j * (imax) + (i - 1)]) / 2.0; - double vel_v = (v[j * (imax) + i] + v[(j - 1) * (imax) + i]) / 2.0; - double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); - fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); - } - } - - fclose(fp); -} diff --git a/BasicSolver/2D-mpi-v3/src/solver.h b/BasicSolver/2D-mpi-v3/src/solver.h deleted file mode 100644 index 76ccce0..0000000 --- a/BasicSolver/2D-mpi-v3/src/solver.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#ifndef __SOLVER_H_ -#define __SOLVER_H_ -#include "parameter.h" -#include - -#define NDIMS 2 - -enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; - -typedef struct { - /* geometry and grid information */ - double dx, dy; - int imax, jmax; - double xlength, ylength; - /* arrays */ - double *p, *rhs; - double *f, *g; - double *u, *v; - /* parameters */ - double eps, omega; - double re, tau, gamma; - double gx, gy; - /* time stepping */ - int itermax; - double dt, te; - double dtBound; - char* problem; - int bcLeft, bcRight, bcBottom, bcTop; - /* mpi */ - int rank; - int size; - MPI_Comm comm; - MPI_Datatype bufferTypes[NDIMS * 2]; - MPI_Aint sdispls[NDIMS * 2], rdispls[NDIMS * 2]; - int iNeighbours[NDIMS], jNeighbours[NDIMS]; - int coords[NDIMS], dims[NDIMS]; - int imaxLocal, jmaxLocal; -} Solver; - -void initSolver(Solver*, Parameter*); -void computeRHS(Solver*); -int solve(Solver*); -void computeTimestep(Solver*); -void setBoundaryConditions(Solver*); -void setSpecialBoundaryCondition(Solver*); -void computeFG(Solver*); -void adaptUV(Solver*); -void collectResult(Solver*); -void writeResult(Solver*, double*, double*, double*); -void debugExchange(Solver*); -void print(Solver*, double*); -#endif diff --git a/BasicSolver/2D-mpi-v3/surface.plot b/BasicSolver/2D-mpi-v3/surface.plot deleted file mode 100644 index 4f7ccd9..0000000 --- a/BasicSolver/2D-mpi-v3/surface.plot +++ /dev/null @@ -1,7 +0,0 @@ -set terminal png size 1024,768 enhanced font ,12 -set output 'p.png' -set datafile separator whitespace - -set grid -set hidden3d -splot 'pressure.dat' using 1:2:3 with lines diff --git a/BasicSolver/2D-mpi/Makefile b/BasicSolver/2D-mpi/Makefile index 57f99f4..eb5349c 100644 --- a/BasicSolver/2D-mpi/Makefile +++ b/BasicSolver/2D-mpi/Makefile @@ -1,5 +1,5 @@ #======================================================================================= -# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. # All rights reserved. # Use of this source code is governed by a MIT-style # license that can be found in the LICENSE file. @@ -18,9 +18,11 @@ include $(MAKE_DIR)/include_$(TAG).mk INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) VPATH = $(SRC_DIR) -SRC = $(wildcard $(SRC_DIR)/*.c) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/comm-$(COMM_TYPE).o +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) @@ -37,9 +39,20 @@ $(BUILD_DIR)/%.s: %.c $(info ===> GENERATE ASM $@) $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ -.PHONY: clean distclean tags info asm format +.PHONY: clean distclean vis vis_clean tags info asm format -clean: +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='pressure.dat'" ./surface.plot + @gnuplot -e "filename='velocity.dat'" ./vector.plot + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.png + +clean: vis_clean $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @rm -f tags @@ -47,6 +60,8 @@ clean: distclean: clean $(info ===> DIST CLEAN) @rm -f $(TARGET) + @rm -f *.dat + @rm -f *.png info: $(info $(CFLAGS)) diff --git a/BasicSolver/2D-mpi/canal.par b/BasicSolver/2D-mpi/canal.par index 80dfa19..37b26fa 100644 --- a/BasicSolver/2D-mpi/canal.par +++ b/BasicSolver/2D-mpi/canal.par @@ -7,10 +7,10 @@ name canal # name of flow setup -bcN 1 # flags for boundary conditions -bcE 3 # 1 = no-slip 3 = outflow -bcS 1 # 2 = free-slip 4 = periodic -bcW 3 # +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # gx 0.0 # Body forces (e.g. gravity) gy 0.0 # @@ -27,15 +27,22 @@ p_init 0.0 # initial value for pressure xlength 30.0 # domain size in x-direction ylength 4.0 # domain size in y-direction imax 200 # number of interior cells in x-direction -jmax 50 # number of interior cells in y-direction +jmax 40 # number of interior cells in y-direction # Time Data: # --------- -te 100.0 # final time +te 60.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- diff --git a/BasicSolver/2D-mpi/config.mk b/BasicSolver/2D-mpi/config.mk index 46cef95..eb5546a 100644 --- a/BasicSolver/2D-mpi/config.mk +++ b/BasicSolver/2D-mpi/config.mk @@ -1,10 +1,17 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC +# Supported: true, false +ENABLE_MPI ?= true ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg +# Supported: v1, v2, v3 +COMM_TYPE ?= v3 #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 -#OPTIONS += -DVERBOSE +OPTIONS += -DVERBOSE +# OPTIONS += -DTEST #OPTIONS += -DVERBOSE_AFFINITY #OPTIONS += -DVERBOSE_DATASIZE #OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/2D-mpi/dcavity.par b/BasicSolver/2D-mpi/dcavity.par index b4013d6..8f229ed 100644 --- a/BasicSolver/2D-mpi/dcavity.par +++ b/BasicSolver/2D-mpi/dcavity.par @@ -15,7 +15,7 @@ bcRight 1 # gx 0.0 # Body forces (e.g. gravity) gy 0.0 # -re 10.0 # Reynolds number +re 100.0 # Reynolds number u_init 0.0 # initial value for velocity in x-direction v_init 0.0 # initial value for velocity in y-direction @@ -26,15 +26,22 @@ p_init 0.0 # initial value for pressure xlength 1.0 # domain size in x-direction ylength 1.0 # domain size in y-direction -imax 100 # number of interior cells in x-direction -jmax 100 # number of interior cells in y-direction +imax 128 # number of interior cells in x-direction +jmax 128 # number of interior cells in y-direction # Time Data: # --------- -te 5.0 # final time -dt 0.02 # time stepsize -tau 0.5 # safety factor for time stepsize control (<0 constant delt) +te 10.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations # Pressure Iteration Data: # ----------------------- diff --git a/BasicSolver/2D-mpi/include_CLANG.mk b/BasicSolver/2D-mpi/include_CLANG.mk index 1d17c52..3641cad 100644 --- a/BasicSolver/2D-mpi/include_CLANG.mk +++ b/BasicSolver/2D-mpi/include_CLANG.mk @@ -1,4 +1,10 @@ +ifeq ($(ENABLE_MPI),true) CC = mpicc +DEFINES = -D_MPI +else +CC = cc +endif + GCC = cc LINKER = $(CC) @@ -9,8 +15,7 @@ LIBS = # -lomp endif VERSION = --version -CFLAGS = -Ofast -std=c99 $(OPENMP) -#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG -LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE# -DDEBUG -INCLUDES = -I/usr/local/include +CFLAGS = -Ofast -std=c17 +LFLAGS = $(OPENMP) -lm +DEFINES += -D_GNU_SOURCE# -DDEBUG +INCLUDES = -I/opt/homebrew/include diff --git a/BasicSolver/2D-mpi/include_GCC.mk b/BasicSolver/2D-mpi/include_GCC.mk index 427e798..cf1e2aa 100644 --- a/BasicSolver/2D-mpi/include_GCC.mk +++ b/BasicSolver/2D-mpi/include_GCC.mk @@ -1,4 +1,10 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpicc +DEFINES = -D_MPI +else CC = gcc +endif + GCC = gcc LINKER = $(CC) @@ -9,6 +15,6 @@ endif VERSION = --version CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE +DEFINES += -D_GNU_SOURCE INCLUDES = LIBS = diff --git a/BasicSolver/2D-mpi/include_ICC.mk b/BasicSolver/2D-mpi/include_ICC.mk index f85d836..6bedf55 100644 --- a/BasicSolver/2D-mpi/include_ICC.mk +++ b/BasicSolver/2D-mpi/include_ICC.mk @@ -1,4 +1,10 @@ +ifeq ($(ENABLE_MPI),true) CC = mpiicc +DEFINES = -D_MPI +else +CC = icc +endif + GCC = gcc LINKER = $(CC) @@ -9,6 +15,6 @@ endif VERSION = --version CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE +DEFINES += -D_GNU_SOURCE# -DDEBUG INCLUDES = LIBS = diff --git a/BasicSolver/2D-mpi/residual.plot b/BasicSolver/2D-mpi/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/BasicSolver/2D-mpi/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/2D-mpi/src/affinity.c b/BasicSolver/2D-mpi/src/affinity.c index b501665..ef8355a 100644 --- a/BasicSolver/2D-mpi/src/affinity.c +++ b/BasicSolver/2D-mpi/src/affinity.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. diff --git a/BasicSolver/2D-mpi/src/affinity.h b/BasicSolver/2D-mpi/src/affinity.h index d844fe5..84bf733 100644 --- a/BasicSolver/2D-mpi/src/affinity.h +++ b/BasicSolver/2D-mpi/src/affinity.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. diff --git a/BasicSolver/2D-mpi/src/allocate.c b/BasicSolver/2D-mpi/src/allocate.c index 81e1e9d..cf2efd6 100644 --- a/BasicSolver/2D-mpi/src/allocate.c +++ b/BasicSolver/2D-mpi/src/allocate.c @@ -1,14 +1,17 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ #include +#include #include #include -void* allocate(int alignment, size_t bytesize) +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) { int errorCode; void* ptr; diff --git a/BasicSolver/2D-mpi/src/allocate.h b/BasicSolver/2D-mpi/src/allocate.h index 54cfe06..77f4ba0 100644 --- a/BasicSolver/2D-mpi/src/allocate.h +++ b/BasicSolver/2D-mpi/src/allocate.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -8,6 +8,6 @@ #define __ALLOCATE_H_ #include -extern void* allocate(int alignment, size_t bytesize); +extern void* allocate(size_t alignment, size_t bytesize); #endif diff --git a/BasicSolver/2D-mpi/src/comm-v1.c b/BasicSolver/2D-mpi/src/comm-v1.c new file mode 100644 index 0000000..16a0422 --- /dev/null +++ b/BasicSolver/2D-mpi/src/comm-v1.c @@ -0,0 +1,234 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include + +#include "comm.h" + +#ifdef _MPI +// subroutines local to this module +static int sum(int* sizes, int position) +{ + int sum = 0; + + for (int i = 0; i < position; i += position) { + sum += sizes[i]; + } + + return sum; +} + +static void gatherArray( + Comm* c, int cnt, int* rcvCounts, int* displs, double* src, double* dst) +{ + double* sendbuffer = src + (c->imaxLocal + 2); + + if (c->rank == 0) { + sendbuffer = src; + } + + MPI_Gatherv(sendbuffer, + cnt, + MPI_DOUBLE, + dst, + rcvCounts, + displs, + MPI_DOUBLE, + 0, + MPI_COMM_WORLD); +} +#endif // defined _MPI + +// exported subroutines +int commIsBoundary(Comm* c, int direction) +{ +#ifdef _MPI + switch (direction) { + case LEFT: + return 1; + break; + case RIGHT: + return 1; + break; + case BOTTOM: + return c->rank == 0; + break; + case TOP: + return c->rank == (c->size - 1); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ + // printf("Rank : %d In exchange \n", c->rank); +#ifdef _MPI + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* exchange ghost cells with top neighbor */ + if (c->rank + 1 < c->size) { + int top = c->rank + 1; + double* src = grid + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + double* dst = grid + (c->jmaxLocal + 1) * (c->imaxLocal + 2) + 1; + + MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, top, 1, MPI_COMM_WORLD, &requests[0]); + MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, top, 2, MPI_COMM_WORLD, &requests[1]); + } + + /* exchange ghost cells with bottom neighbor */ + if (c->rank > 0) { + int bottom = c->rank - 1; + double* src = grid + (c->imaxLocal + 2) + 1; + double* dst = grid + 1; + + MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]); + MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]); + } + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commShift(Comm* c, double* f, double* g) +{ +#ifdef _MPI + MPI_Request requests[2] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + if (c->rank > 0) { + int bottom = c->rank - 1; + MPI_Irecv(g + 1, + c->imaxLocal, + MPI_DOUBLE, + bottom, + 0, + MPI_COMM_WORLD, + &requests[0]); + } + + if (c->rank + 1 < c->size) { + int top = c->rank + 1; + double* buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + /* send ghost cells to top neighbor */ + MPI_Isend(buf, c->imaxLocal, MPI_DOUBLE, top, 0, MPI_COMM_WORLD, &requests[1]); + } + + MPI_Waitall(2, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int jmax, + int imax) +{ +#ifdef _MPI + int *rcvCounts, *displs; + int cnt = c->jmaxLocal * (imax + 2); + + if (c->rank == 0) { + rcvCounts = (int*)malloc(c->size * sizeof(int)); + displs = (int*)malloc(c->size * sizeof(int)); + } + + if (c->rank == 0 && c->size == 1) { + cnt = (c->jmaxLocal + 2) * (imax + 2); + } else if (c->rank == 0 || c->rank == (c->size - 1)) { + cnt = (c->jmaxLocal + 1) * (imax + 2); + } + + MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD); + + if (c->rank == 0) { + displs[0] = 0; + int cursor = rcvCounts[0]; + + for (int i = 1; i < c->size; i++) { + displs[i] = cursor; + cursor += rcvCounts[i]; + } + } + + gatherArray(c, cnt, rcvCounts, displs, p, pg); + gatherArray(c, cnt, rcvCounts, displs, u, ug); + gatherArray(c, cnt, rcvCounts, displs, v, vg); +#endif +} + +void commPartition(Comm* c, int jmax, int imax) +{ +#ifdef _MPI + c->imaxLocal = imax; + c->jmaxLocal = sizeOfRank(c->coords[JDIM], c->size, jmax); + + c->neighbours[BOTTOM] = c->rank == 0 ? -1 : c->rank - 1; + c->neighbours[TOP] = c->rank == (c->size - 1) ? -1 : c->rank + 1; + c->neighbours[LEFT] = -1; + c->neighbours[RIGHT] = -1; + + c->coords[IDIM] = 0; + c->coords[JDIM] = c->rank; + + c->dims[IDIM] = 1; + c->dims[JDIM] = c->size; +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; +#endif +} + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ + +#if defined _MPI + newcomm->comm = MPI_COMM_NULL; + int result = MPI_Comm_dup(MPI_COMM_WORLD, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + + newcomm->neighbours[BOTTOM] = newcomm->rank == 0 ? -1 : newcomm->rank - 1; + newcomm->neighbours[TOP] = newcomm->rank == (newcomm->size - 1) ? -1 : newcomm->rank + 1; + newcomm->neighbours[LEFT] = -1; + newcomm->neighbours[RIGHT] = -1; + + newcomm->coords[IDIM] = 0; + newcomm->coords[JDIM] = newcomm->rank; + + newcomm->dims[IDIM] = 1; + newcomm->dims[JDIM] = newcomm->size; + + +#endif + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/BasicSolver/2D-mpi/src/comm-v2.c b/BasicSolver/2D-mpi/src/comm-v2.c new file mode 100644 index 0000000..5ace203 --- /dev/null +++ b/BasicSolver/2D-mpi/src/comm-v2.c @@ -0,0 +1,342 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "comm.h" +#include +#include +#include + +#ifdef _MPI +// subroutines local to this module +static int sum(int* sizes, int init, int offset, int coord) +{ + int sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax) +{ + MPI_Request* requests; + int numRequests = 1; + + if (c->rank == 0) { + numRequests = c->size + 1; + } else { + numRequests = 1; + } + + requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); + + /* all ranks send their bulk array, including the external boundary layer */ + MPI_Datatype bulkType; + int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; + int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; + int starts[NDIMS] = { 1, 1 }; + + if (commIsBoundary(c, LEFT)) { + newSizes[CIDIM] += 1; + starts[CIDIM] = 0; + } + if (commIsBoundary(c, RIGHT)) { + newSizes[CIDIM] += 1; + } + if (commIsBoundary(c, BOTTOM)) { + newSizes[CJDIM] += 1; + starts[CJDIM] = 0; + } + if (commIsBoundary(c, TOP)) { + newSizes[CJDIM] += 1; + } + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]); + + int newSizesI[c->size]; + int newSizesJ[c->size]; + MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD); + + /* rank 0 assembles the subdomains */ + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + MPI_Datatype domainType; + int oldSizes[NDIMS] = { jmax + 2, imax + 2 }; + int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] }; + int coords[NDIMS]; + MPI_Cart_coords(c->comm, i, NDIMS, coords); + int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]), + sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) }; + printf( + "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d " + "Starts(i,j): %d %d\n", + i, + coords[IDIM], + coords[JDIM], + oldSizes[CIDIM], + oldSizes[CJDIM], + newSizes[CIDIM], + newSizes[CJDIM], + starts[CIDIM], + starts[CJDIM]); + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} +#endif // defined _MPI + +// exported subroutines +int commIsBoundary(Comm* c, int direction) +{ +#ifdef _MPI + switch (direction) { + case LEFT: + return c->coords[IDIM] == 0; + break; + case RIGHT: + return c->coords[IDIM] == (c->dims[IDIM] - 1); + break; + case BOTTOM: + return c->coords[JDIM] == 0; + break; + case TOP: + return c->coords[JDIM] == (c->dims[JDIM] - 1); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ +#ifdef _MPI + MPI_Request requests[8]; + for (int i = 0; i < 8; i++) + requests[i] = MPI_REQUEST_NULL; + + for (int i = 0; i < NDIRS; i++) { + double* sbuf = grid + c->sdispls[i]; + double* rbuf = grid + c->rdispls[i]; + + int tag = 0; + if (c->neighbours[i] != MPI_PROC_NULL) { + // printf("DEBUG: Rank %d - SendRecv with %d\n", c->rank, c->neighbours[i]); + tag = c->neighbours[i]; + } + MPI_Irecv(rbuf, + 1, + c->bufferTypes[i], + c->neighbours[i], + tag, + c->comm, + &requests[i * 2]); + MPI_Isend(sbuf, + 1, + c->bufferTypes[i], + c->neighbours[i], + c->rank, + c->comm, + &requests[i * 2 + 1]); + } + + MPI_Waitall(8, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commShift(Comm* c, double* f, double* g) +{ +#ifdef _MPI + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + double* buf = g + 1; + MPI_Irecv(buf, + 1, + c->bufferTypes[BOTTOM], + c->neighbours[BOTTOM], + 0, + c->comm, + &requests[0]); + + /* send ghost cells to top neighbor */ + buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + MPI_Isend(buf, 1, c->bufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]); + + /* shift F */ + /* receive ghost cells from left neighbor */ + buf = f + (c->imaxLocal + 2); + MPI_Irecv(buf, + 1, + c->bufferTypes[LEFT], + c->neighbours[LEFT], + 1, + c->comm, + &requests[2]); + + /* send ghost cells to right neighbor */ + buf = f + (c->imaxLocal + 2) + (c->imaxLocal); + MPI_Isend(buf, + 1, + c->bufferTypes[RIGHT], + c->neighbours[RIGHT], + 1, + c->comm, + &requests[3]); + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int imax, + int jmax) +{ +#ifdef _MPI + /* collect P */ + assembleResult(c, p, pg, imax, jmax); + + /* collect U */ + assembleResult(c, u, ug, imax, jmax); + + /* collect V */ + assembleResult(c, v, vg, imax, jmax); +#endif +} + +void commPartition(Comm* c, int jmax, int imax) +{ +#ifdef _MPI + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + MPI_Dims_create(c->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm); + MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]); + MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]); + MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords); + + int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax); + int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax); + + c->imaxLocal = imaxLocal; + c->jmaxLocal = jmaxLocal; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + c->bufferTypes[LEFT] = iBufferType; + c->bufferTypes[RIGHT] = iBufferType; + c->bufferTypes[BOTTOM] = jBufferType; + c->bufferTypes[TOP] = jBufferType; + + c->sdispls[LEFT] = (imaxLocal + 2) + 1; + c->sdispls[RIGHT] = (imaxLocal + 2) + imaxLocal; + c->sdispls[BOTTOM] = (imaxLocal + 2) + 1; + c->sdispls[TOP] = jmaxLocal * (imaxLocal + 2) + 1; + + c->rdispls[LEFT] = (imaxLocal + 2); + c->rdispls[RIGHT] = (imaxLocal + 2) + (imaxLocal + 1); + c->rdispls[BOTTOM] = 1; + c->rdispls[TOP] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; +#endif +} + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ +#if defined _MPI + newcomm->comm = MPI_COMM_NULL; + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + memcpy(&newcomm->neighbours, &oldcomm->neighbours, sizeof(oldcomm->neighbours)); + memcpy(&newcomm->coords, &oldcomm->coords, sizeof(oldcomm->coords)); + memcpy(&newcomm->dims, &oldcomm->dims, sizeof(oldcomm->dims)); + + newcomm->imaxLocal = imaxLocal/2; + newcomm->jmaxLocal = jmaxLocal/2; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + newcomm->bufferTypes[LEFT] = iBufferType; + newcomm->bufferTypes[RIGHT] = iBufferType; + newcomm->bufferTypes[BOTTOM] = jBufferType; + newcomm->bufferTypes[TOP] = jBufferType; + + newcomm->sdispls[LEFT] = (imaxLocal + 2) + 1; + newcomm->sdispls[RIGHT] = (imaxLocal + 2) + imaxLocal; + newcomm->sdispls[BOTTOM] = (imaxLocal + 2) + 1; + newcomm->sdispls[TOP] = jmaxLocal * (imaxLocal + 2) + 1; + + newcomm->rdispls[LEFT] = (imaxLocal + 2); + newcomm->rdispls[RIGHT] = (imaxLocal + 2) + (imaxLocal + 1); + newcomm->rdispls[BOTTOM] = 1; + newcomm->rdispls[TOP] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/BasicSolver/2D-mpi/src/comm-v3.c b/BasicSolver/2D-mpi/src/comm-v3.c new file mode 100644 index 0000000..a1e6fb2 --- /dev/null +++ b/BasicSolver/2D-mpi/src/comm-v3.c @@ -0,0 +1,320 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "comm.h" + +#ifdef _MPI +// subroutines local to this module +static int sum(int* sizes, int init, int offset, int coord) +{ + int sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax) +{ + MPI_Request* requests; + int numRequests = 1; + + if (c->rank == 0) { + numRequests = c->size + 1; + } else { + numRequests = 1; + } + + requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); + + /* all ranks send their bulk array, including the external boundary layer */ + MPI_Datatype bulkType; + int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; + int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; + int starts[NDIMS] = { 1, 1 }; + + if (commIsBoundary(c, LEFT)) { + newSizes[CIDIM] += 1; + starts[CIDIM] = 0; + } + if (commIsBoundary(c, RIGHT)) { + newSizes[CIDIM] += 1; + } + if (commIsBoundary(c, BOTTOM)) { + newSizes[CJDIM] += 1; + starts[CJDIM] = 0; + } + if (commIsBoundary(c, TOP)) { + newSizes[CJDIM] += 1; + } + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]); + + int newSizesI[c->size]; + int newSizesJ[c->size]; + MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD); + + /* rank 0 assembles the subdomains */ + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + MPI_Datatype domainType; + int oldSizes[NDIMS] = { jmax + 2, imax + 2 }; + int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] }; + int coords[NDIMS]; + MPI_Cart_coords(c->comm, i, NDIMS, coords); + int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]), + sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) }; + printf( + "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d " + "Starts(i,j): %d %d\n", + i, + coords[IDIM], + coords[JDIM], + oldSizes[CIDIM], + oldSizes[CJDIM], + newSizes[CIDIM], + newSizes[CJDIM], + starts[CIDIM], + starts[CJDIM]); + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} +#endif // defined _MPI + +// exported subroutines +int commIsBoundary(Comm* c, int direction) +{ +#ifdef _MPI + switch (direction) { + case LEFT: + return c->coords[IDIM] == 0; + break; + case RIGHT: + return c->coords[IDIM] == (c->dims[IDIM] - 1); + break; + case BOTTOM: + return c->coords[JDIM] == 0; + break; + case TOP: + return c->coords[JDIM] == (c->dims[JDIM] - 1); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ +#ifdef _MPI + int counts[NDIRS] = { 1, 1, 1, 1 }; + MPI_Neighbor_alltoallw(grid, + counts, + c->sdispls, + c->bufferTypes, + grid, + counts, + c->rdispls, + c->bufferTypes, + c->comm); +#endif +} + +void commShift(Comm* c, double* f, double* g) +{ +#ifdef _MPI + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + double* buf = g + 1; + MPI_Irecv(buf, + 1, + c->bufferTypes[BOTTOM], + c->neighbours[BOTTOM], + 0, + c->comm, + &requests[0]); + + /* send ghost cells to top neighbor */ + buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + MPI_Isend(buf, 1, c->bufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]); + + /* shift F */ + /* receive ghost cells from left neighbor */ + buf = f + (c->imaxLocal + 2); + MPI_Irecv(buf, + 1, + c->bufferTypes[LEFT], + c->neighbours[LEFT], + 1, + c->comm, + &requests[2]); + + /* send ghost cells to right neighbor */ + buf = f + (c->imaxLocal + 2) + (c->imaxLocal); + MPI_Isend(buf, + 1, + c->bufferTypes[RIGHT], + c->neighbours[RIGHT], + 1, + c->comm, + &requests[3]); + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int imax, + int jmax) +{ +#ifdef _MPI + /* collect P */ + assembleResult(c, p, pg, imax, jmax); + + /* collect U */ + assembleResult(c, u, ug, imax, jmax); + + /* collect V */ + assembleResult(c, v, vg, imax, jmax); +#endif +} + +void commPartition(Comm* c, int jmax, int imax) +{ +#ifdef _MPI + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + MPI_Dims_create(c->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm); + MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]); + MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]); + MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords); + + int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax); + int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax); + + c->imaxLocal = imaxLocal; + c->jmaxLocal = jmaxLocal; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + c->bufferTypes[LEFT] = iBufferType; + c->bufferTypes[RIGHT] = iBufferType; + c->bufferTypes[BOTTOM] = jBufferType; + c->bufferTypes[TOP] = jBufferType; + + size_t dblsize = sizeof(double); + c->sdispls[LEFT] = ((imaxLocal + 2) + 1) * dblsize; + c->sdispls[RIGHT] = ((imaxLocal + 2) + imaxLocal) * dblsize; + c->sdispls[BOTTOM] = ((imaxLocal + 2) + 1) * dblsize; + c->sdispls[TOP] = (jmaxLocal * (imaxLocal + 2) + 1) * dblsize; + + c->rdispls[LEFT] = (imaxLocal + 2) * dblsize; + c->rdispls[RIGHT] = ((imaxLocal + 2) + (imaxLocal + 1)) * dblsize; + c->rdispls[BOTTOM] = 1 * dblsize; + c->rdispls[TOP] = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * dblsize; +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; +#endif +} + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ +#if defined _MPI + + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + newcomm->bufferTypes[LEFT] = iBufferType; + newcomm->bufferTypes[RIGHT] = iBufferType; + newcomm->bufferTypes[BOTTOM] = jBufferType; + newcomm->bufferTypes[TOP] = jBufferType; + + newcomm->sdispls[LEFT] = (imaxLocal + 2) + 1; + newcomm->sdispls[RIGHT] = (imaxLocal + 2) + imaxLocal; + newcomm->sdispls[BOTTOM] = (imaxLocal + 2) + 1; + newcomm->sdispls[TOP] = jmaxLocal * (imaxLocal + 2) + 1; + + newcomm->rdispls[LEFT] = (imaxLocal + 2); + newcomm->rdispls[RIGHT] = (imaxLocal + 2) + (imaxLocal + 1); + newcomm->rdispls[BOTTOM] = 1; + newcomm->rdispls[TOP] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/BasicSolver/2D-mpi/src/comm.c b/BasicSolver/2D-mpi/src/comm.c index eeab2a7..fab8f98 100644 --- a/BasicSolver/2D-mpi/src/comm.c +++ b/BasicSolver/2D-mpi/src/comm.c @@ -1,281 +1,34 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ -#include #include #include #include "comm.h" // subroutines local to this module -static int sizeOfRank(int rank, int size, int N) +int sizeOfRank(int rank, int size, int N) { return N / size + ((N % size > rank) ? 1 : 0); } -static void setupCommunication(Comm* c, int direction, int layer) -{ - MPI_Datatype type; - size_t dblsize = sizeof(double); - int imaxLocal = c->imaxLocal; - int jmaxLocal = c->jmaxLocal; - int sizes[NDIMS]; - int subSizes[NDIMS]; - int starts[NDIMS]; - int offset = 0; - - sizes[IDIM] = imaxLocal + 2; - sizes[JDIM] = jmaxLocal + 2; - - if (layer == HALO) { - offset = 1; - } - - switch (direction) { - case LEFT: - subSizes[IDIM] = 1; - subSizes[JDIM] = jmaxLocal; - starts[IDIM] = 1 - offset; - starts[JDIM] = 1; - break; - case RIGHT: - subSizes[IDIM] = 1; - subSizes[JDIM] = jmaxLocal; - starts[IDIM] = imaxLocal + offset; - starts[JDIM] = 1; - break; - case BOTTOM: - subSizes[IDIM] = imaxLocal; - subSizes[JDIM] = 1; - starts[IDIM] = 1; - starts[JDIM] = 1 - offset; - break; - case TOP: - subSizes[IDIM] = imaxLocal; - subSizes[JDIM] = 1; - starts[IDIM] = 1; - starts[JDIM] = jmaxLocal + offset; - break; - } - - MPI_Type_create_subarray(NDIMS, - sizes, - subSizes, - starts, - MPI_ORDER_C, - MPI_DOUBLE, - &type); - MPI_Type_commit(&type); - - if (layer == HALO) { - c->rbufferTypes[direction] = type; - } else if (layer == BULK) { - c->sbufferTypes[direction] = type; - } -} - -static void assembleResult(Comm* c, - double* src, - double* dst, - int imaxLocal[], - int jmaxLocal[], - int offset[], - int jmax, - int imax) -{ - MPI_Request* requests; - int numRequests = 1; - - if (c->rank == 0) { - numRequests = c->size + 1; - } else { - numRequests = 1; - } - - requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); - - /* all ranks send their bulk array */ - MPI_Datatype bulkType; - int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; - int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; - int starts[NDIMS] = { 1, 1 }; - MPI_Type_create_subarray(NDIMS, - oldSizes, - newSizes, - starts, - MPI_ORDER_C, - MPI_DOUBLE, - &bulkType); - MPI_Type_commit(&bulkType); - - MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]); - - /* rank 0 assembles the subdomains */ - if (c->rank == 0) { - for (int i = 0; i < c->size; i++) { - MPI_Datatype domainType; - int oldSizes[NDIMS] = { jmax, imax }; - int newSizes[NDIMS] = { jmaxLocal[i], imaxLocal[i] }; - int starts[NDIMS] = { offset[i * NDIMS + JDIM], offset[i * NDIMS + IDIM] }; - MPI_Type_create_subarray(NDIMS, - oldSizes, - newSizes, - starts, - MPI_ORDER_C, - MPI_DOUBLE, - &domainType); - MPI_Type_commit(&domainType); - - MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); - } - } - - MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); -} - -static int sum(int* sizes, int position) -{ - int sum = 0; - - for (int i = 0; i < position; i++) { - sum += sizes[i]; - } - - return sum; -} - -// exported subroutines void commReduction(double* v, int op) { +#ifdef _MPI if (op == MAX) { MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); } else if (op == SUM) { MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); } -} - -int commIsBoundary(Comm* c, int direction) -{ - switch (direction) { - case LEFT: - return c->coords[IDIM] == 0; - break; - case RIGHT: - return c->coords[IDIM] == (c->dims[IDIM] - 1); - break; - case BOTTOM: - return c->coords[JDIM] == 0; - break; - case TOP: - return c->coords[JDIM] == (c->dims[JDIM] - 1); - break; - } - - return 0; -} - -void commExchange(Comm* c, double* grid) -{ - int counts[NDIRS] = { 1, 1, 1, 1 }; - MPI_Aint displs[NDIRS] = { 0, 0, 0, 0 }; - - MPI_Neighbor_alltoallw(grid, - counts, - displs, - c->sbufferTypes, - grid, - counts, - displs, - c->rbufferTypes, - c->comm); -} - -void commShift(Comm* c, double* f, double* g) -{ - MPI_Request requests[4] = { MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL }; - - /* shift G */ - /* receive ghost cells from bottom neighbor */ - MPI_Irecv(g, - 1, - c->rbufferTypes[BOTTOM], - c->neighbours[BOTTOM], - 0, - c->comm, - &requests[0]); - - /* send ghost cells to top neighbor */ - MPI_Isend(g, 1, c->sbufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]); - - /* shift F */ - /* receive ghost cells from left neighbor */ - MPI_Irecv(f, 1, c->rbufferTypes[LEFT], c->neighbours[LEFT], 1, c->comm, &requests[2]); - - /* send ghost cells to right neighbor */ - MPI_Isend(f, - 1, - c->sbufferTypes[RIGHT], - c->neighbours[RIGHT], - 1, - c->comm, - &requests[3]); - - MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); -} - -void commCollectResult(Comm* c, - double* ug, - double* vg, - double* pg, - double* u, - double* v, - double* p, - int jmax, - int imax) -{ - int offset[c->size * NDIMS]; - int imaxLocal[c->size]; - int jmaxLocal[c->size]; - - MPI_Gather(&c->imaxLocal, 1, MPI_INT, imaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Gather(&c->jmaxLocal, 1, MPI_INT, jmaxLocal, 1, MPI_INT, 0, MPI_COMM_WORLD); - - if (c->rank == 0) { - for (int i = 0; i < c->size; i++) { - int coords[NDIMS]; - MPI_Cart_coords(c->comm, i, NDIMS, coords); - offset[i * NDIMS + IDIM] = sum(imaxLocal, coords[IDIM]); - offset[i * NDIMS + JDIM] = sum(jmaxLocal, coords[JDIM]); - printf("Rank: %d, Coords(j,i): %d %d, Size(j,i): %d %d " - "Offset(j,i): %d %d\n", - i, - coords[JDIM], - coords[IDIM], - jmaxLocal[i], - imaxLocal[i], - offset[i * NDIMS + JDIM], - offset[i * NDIMS + IDIM]); - } - } - - /* collect P */ - assembleResult(c, p, pg, imaxLocal, jmaxLocal, offset, jmax, imax); - - /* collect U */ - assembleResult(c, u, ug, imaxLocal, jmaxLocal, offset, jmax, imax); - - /* collect V */ - assembleResult(c, v, vg, imaxLocal, jmaxLocal, offset, jmax, imax); +#endif } void commPrintConfig(Comm* c) { +#ifdef _MPI fflush(stdout); MPI_Barrier(MPI_COMM_WORLD); if (commIsMaster(c)) { @@ -290,37 +43,87 @@ void commPrintConfig(Comm* c) c->neighbours[TOP], c->neighbours[LEFT], c->neighbours[RIGHT]); - printf("\tCoordinates (j,i) %d %d\n", c->coords[JDIM], c->coords[IDIM]); - printf("\tLocal domain size (j,i) %dx%d\n", c->jmaxLocal, c->imaxLocal); + printf("\tIs boundary:\n"); + printf("\t\tLEFT: %d\n", commIsBoundary(c, LEFT)); + printf("\t\tRIGHT: %d\n", commIsBoundary(c, RIGHT)); + printf("\t\tBOTTOM: %d\n", commIsBoundary(c, BOTTOM)); + printf("\t\tTOP: %d\n", commIsBoundary(c, TOP)); + printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]); + printf("\tDims (i,j) %d %d\n", c->dims[IDIM], c->dims[JDIM]); + printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal); fflush(stdout); } + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); +#endif } -void commInit(Comm* c, int jmax, int imax) +void commInit(Comm* c, int argc, char** argv) { - /* setup communication */ +#ifdef _MPI + MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank)); MPI_Comm_size(MPI_COMM_WORLD, &(c->size)); - int dims[NDIMS] = { 0, 0 }; - int periods[NDIMS] = { 0, 0 }; - MPI_Dims_create(c->size, NDIMS, dims); - MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm); - MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]); - MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]); - MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords); - - c->imaxLocal = sizeOfRank(c->rank, dims[IDIM], imax); - c->jmaxLocal = sizeOfRank(c->rank, dims[JDIM], jmax); - - // setup buffer types for communication - setupCommunication(c, LEFT, BULK); - setupCommunication(c, LEFT, HALO); - setupCommunication(c, RIGHT, BULK); - setupCommunication(c, RIGHT, HALO); - setupCommunication(c, BOTTOM, BULK); - setupCommunication(c, BOTTOM, HALO); - setupCommunication(c, TOP, BULK); - setupCommunication(c, TOP, HALO); +#else + c->rank = 0; + c->size = 1; +#endif +} + +void commTestInit(Comm* c, double* p, double* f, double* g) +{ + int imax = c->imaxLocal; + int jmax = c->jmaxLocal; + int rank = c->rank; + + for (int j = 0; j < jmax + 2; j++) { + for (int i = 0; i < imax + 2; i++) { + p[j * (imax + 2) + i] = rank; + f[j * (imax + 2) + i] = rank; + g[j * (imax + 2) + i] = rank; + } + } +} + +static void testWriteFile(char* filename, double* grid, int imax, int jmax) +{ + FILE* fp = fopen(filename, "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 0; j < jmax + 2; j++) { + for (int i = 0; i < imax + 2; i++) { + fprintf(fp, "%.2f ", grid[j * (imax + 2) + i]); + } + fprintf(fp, "\n"); + } + + fclose(fp); +} + +void commTestWrite(Comm* c, double* p, double* f, double* g) +{ + int imax = c->imaxLocal; + int jmax = c->jmaxLocal; + int rank = c->rank; + + char filename[30]; + snprintf(filename, 30, "ptest-%d.dat", rank); + testWriteFile(filename, p, imax, jmax); + + snprintf(filename, 30, "ftest-%d.dat", rank); + testWriteFile(filename, f, imax, jmax); + + snprintf(filename, 30, "gtest-%d.dat", rank); + testWriteFile(filename, g, imax, jmax); +} + +void commFinalize(Comm* c) +{ +#ifdef _MPI + MPI_Finalize(); +#endif } diff --git a/BasicSolver/2D-mpi/src/comm.h b/BasicSolver/2D-mpi/src/comm.h index 11782f1..30eaf7b 100644 --- a/BasicSolver/2D-mpi/src/comm.h +++ b/BasicSolver/2D-mpi/src/comm.h @@ -1,35 +1,48 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ #ifndef __COMM_H_ #define __COMM_H_ +#if defined(_MPI) #include +#endif enum direction { LEFT = 0, RIGHT, BOTTOM, TOP, NDIRS }; -enum dimension { JDIM = 0, IDIM, NDIMS }; +enum dimension { IDIM = 0, JDIM, NDIMS }; +enum cdimension { CJDIM = 0, CIDIM }; enum layer { HALO = 0, BULK }; enum op { MAX = 0, SUM }; typedef struct { int rank; int size; +#if defined(_MPI) MPI_Comm comm; - MPI_Datatype sbufferTypes[NDIRS]; - MPI_Datatype rbufferTypes[NDIRS]; + MPI_Datatype bufferTypes[NDIRS]; + MPI_Aint sdispls[NDIRS]; + MPI_Aint rdispls[NDIRS]; +#endif int neighbours[NDIRS]; int coords[NDIMS], dims[NDIMS]; int imaxLocal, jmaxLocal; } Comm; -extern void commInit(Comm* c, int jmax, int imax); +extern int sizeOfRank(int rank, int size, int N); +extern void commInit(Comm* c, int argc, char** argv); +extern void commTestInit(Comm* c, double* p, double* f, double* g); +extern void commTestWrite(Comm* c, double* p, double* f, double* g); +extern void commFinalize(Comm* c); +extern void commPartition(Comm* c, int jmax, int imax); extern void commPrintConfig(Comm*); extern void commExchange(Comm*, double*); extern void commShift(Comm* c, double* f, double* g); extern void commReduction(double* v, int op); extern int commIsBoundary(Comm* c, int direction); +extern void commUpdateDatatypes(Comm*, Comm*, int, int); +extern void commFreeCommunicator(Comm*); extern void commCollectResult(Comm* c, double* ug, double* vg, diff --git a/BasicSolver/2D-mpi/src/solver.c b/BasicSolver/2D-mpi/src/discretization.c similarity index 67% rename from BasicSolver/2D-mpi/src/solver.c rename to BasicSolver/2D-mpi/src/discretization.c index cbff349..69d3582 100644 --- a/BasicSolver/2D-mpi/src/solver.c +++ b/BasicSolver/2D-mpi/src/discretization.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -12,18 +12,11 @@ #include "allocate.h" #include "comm.h" +#include "discretization.h" #include "parameter.h" -#include "solver.h" #include "util.h" -#define P(i, j) p[(j) * (imaxLocal + 2) + (i)] -#define F(i, j) f[(j) * (imaxLocal + 2) + (i)] -#define G(i, j) g[(j) * (imaxLocal + 2) + (i)] -#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] -#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] -#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)] - -static void printConfig(Solver* s) +static void printConfig(Discretization* s) { if (commIsMaster(&s->comm)) { printf("Parameters for #%s#\n", s->problem); @@ -35,47 +28,41 @@ static void printConfig(Solver* s) printf("\tReynolds number: %.2f\n", s->re); printf("\tGx Gy: %.2f %.2f\n", s->gx, s->gy); printf("Geometry data:\n"); - printf("\tDomain box size (x, y): %.2f, %.2f\n", s->xlength, s->ylength); - printf("\tCells (x, y): %d, %d\n", s->imax, s->jmax); - printf("\tCell size (dx, dy): %f, %f\n", s->dx, s->dy); + printf("\tDomain box size (x, y): %.2f, %.2f\n", + s->grid.xlength, + s->grid.ylength); + printf("\tCells (x, y): %d, %d\n", s->grid.imax, s->grid.jmax); + printf("\tCell size (dx, dy): %f, %f\n", s->grid.dx, s->grid.dy); printf("Timestep parameters:\n"); printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te); printf("\tdt bound: %.6f\n", s->dtBound); printf("\tTau factor: %.2f\n", s->tau); printf("Iterative s parameters:\n"); - printf("\tMax iterations: %d\n", s->itermax); - printf("\tepsilon (stopping tolerance) : %f\n", s->eps); printf("\tgamma factor: %f\n", s->gamma); - printf("\tomega (SOR relaxation): %f\n", s->omega); } commPrintConfig(&s->comm); } -void initSolver(Solver* s, Parameter* params) +void initDiscretiztion(Discretization* s, Parameter* params) { - s->problem = params->name; - s->bcLeft = params->bcLeft; - s->bcRight = params->bcRight; - s->bcBottom = params->bcBottom; - s->bcTop = params->bcTop; - s->imax = params->imax; - s->jmax = params->jmax; - s->xlength = params->xlength; - s->ylength = params->ylength; - s->dx = params->xlength / params->imax; - s->dy = params->ylength / params->jmax; - s->eps = params->eps; - s->omega = params->omg; - s->itermax = params->itermax; - s->re = params->re; - s->gx = params->gx; - s->gy = params->gy; - s->dt = params->dt; - s->te = params->te; - s->tau = params->tau; - s->gamma = params->gamma; - - commInit(&s->comm, s->jmax, s->imax); + s->problem = params->name; + s->bcLeft = params->bcLeft; + s->bcRight = params->bcRight; + s->bcBottom = params->bcBottom; + s->bcTop = params->bcTop; + s->grid.imax = params->imax; + s->grid.jmax = params->jmax; + s->grid.xlength = params->xlength; + s->grid.ylength = params->ylength; + s->grid.dx = params->xlength / params->imax; + s->grid.dy = params->ylength / params->jmax; + s->re = params->re; + s->gx = params->gx; + s->gy = params->gy; + s->dt = params->dt; + s->te = params->te; + s->tau = params->tau; + s->gamma = params->gamma; /* allocate arrays */ int imaxLocal = s->comm.imaxLocal; @@ -98,8 +85,8 @@ void initSolver(Solver* s, Parameter* params) s->g[i] = 0.0; } - double dx = s->dx; - double dy = s->dy; + double dx = s->grid.dx; + double dy = s->grid.dy; double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy); s->dtBound = 0.5 * s->re * 1.0 / invSqrSum; @@ -108,12 +95,12 @@ void initSolver(Solver* s, Parameter* params) #endif } -void computeRHS(Solver* s) +void computeRHS(Discretization* s) { int imaxLocal = s->comm.imaxLocal; int jmaxLocal = s->comm.jmaxLocal; - double idx = 1.0 / s->dx; - double idy = 1.0 / s->dy; + double idx = 1.0 / s->grid.dx; + double idy = 1.0 / s->grid.dy; double idt = 1.0 / s->dt; double* rhs = s->rhs; double* f = s->f; @@ -129,88 +116,7 @@ void computeRHS(Solver* s) } } -int solve(Solver* s) -{ - int imax = s->imax; - int jmax = s->jmax; - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - double eps = s->eps; - int itermax = s->itermax; - double dx2 = s->dx * s->dx; - double dy2 = s->dy * s->dy; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); - double* p = s->p; - double* rhs = s->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - commExchange(&s->comm, p); - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - - double r = RHS(i, j) - - ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + - (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - - P(i, j) -= (factor * r); - res += (r * r); - } - } - - if (commIsBoundary(&s->comm, BOTTOM)) { // set bottom bc - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, 0) = P(i, 1); - } - } - - if (commIsBoundary(&s->comm, TOP)) { // set top bc - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, jmaxLocal + 1) = P(i, jmaxLocal); - } - } - - if (commIsBoundary(&s->comm, LEFT)) { // set left bc - for (int j = 1; j < jmaxLocal + 1; j++) { - P(0, j) = P(1, j); - } - } - - if (commIsBoundary(&s->comm, RIGHT)) { // set right bc - for (int j = 1; j < jmaxLocal + 1; j++) { - P(imaxLocal + 1, j) = P(imaxLocal, j); - } - } - - commReduction(&res, SUM); - res = res / (double)(imax * jmax); -#ifdef DEBUG - if (commIsMaster(&s->comm)) { - printf("%d Residuum: %e\n", it, res); - } -#endif - it++; - } - -#ifdef VERBOSE - if (commIsMaster(&s->comm)) { - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); - } -#endif - if (res < eps) { - return 0; - } else { - return 1; - } -} - -static double maxElement(Solver* s, double* m) +static double maxElement(Discretization* s, double* m) { int imaxLocal = s->comm.imaxLocal; int jmaxLocal = s->comm.jmaxLocal; @@ -225,11 +131,11 @@ static double maxElement(Solver* s, double* m) return maxval; } -void computeTimestep(Solver* s) +void computeTimestep(Discretization* s) { double dt = s->dtBound; - double dx = s->dx; - double dy = s->dy; + double dx = s->grid.dx; + double dy = s->grid.dy; double umax = maxElement(s, s->u); double vmax = maxElement(s, s->v); @@ -243,7 +149,7 @@ void computeTimestep(Solver* s) s->dt = dt * s->tau; } -void setBoundaryConditions(Solver* s) +void setBoundaryConditions(Discretization* s) { int imaxLocal = s->comm.imaxLocal; int jmaxLocal = s->comm.jmaxLocal; @@ -351,7 +257,7 @@ void setBoundaryConditions(Solver* s) } } -void setSpecialBoundaryCondition(Solver* s) +void setSpecialBoundaryCondition(Discretization* s) { int imaxLocal = s->comm.imaxLocal; int jmaxLocal = s->comm.jmaxLocal; @@ -365,25 +271,27 @@ void setSpecialBoundaryCondition(Solver* s) } } else if (strcmp(s->problem, "canal") == 0) { if (commIsBoundary(&s->comm, LEFT)) { - double ylength = s->ylength; - double dy = s->dy; - int rest = s->jmax % s->comm.size; - int yc = s->comm.rank * (s->jmax / s->comm.size) + MIN(rest, s->comm.rank); + double ylength = s->grid.ylength; + double dy = s->grid.dy; + int rest = s->grid.jmax % s->comm.dims[JDIM]; + int yc = s->comm.rank * (s->grid.jmax / s->comm.dims[JDIM]) + + MIN(rest, s->comm.rank); double ys = dy * (yc + 0.5); double y; - /* printf("RANK %d yc: %d ys: %f\n", solver->rank, yc, ys); */ + // printf("RANK %d yc: %d ys: %f\n", s->comm.rank, yc, ys); for (int j = 1; j < jmaxLocal + 1; j++) { y = ys + dy * (j - 0.5); U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } } } /* print(solver, solver->u); */ } -void computeFG(Solver* s) +void computeFG(Discretization* s) { double* u = s->u; double* v = s->v; @@ -398,8 +306,8 @@ void computeFG(Solver* s) double gamma = s->gamma; double dt = s->dt; double inverseRe = 1.0 / s->re; - double inverseDx = 1.0 / s->dx; - double inverseDy = 1.0 / s->dy; + double inverseDx = 1.0 / s->grid.dx; + double inverseDy = 1.0 / s->grid.dy; double du2dx, dv2dy, duvdx, duvdy; double du2dx2, du2dy2, dv2dx2, dv2dy2; @@ -475,7 +383,7 @@ void computeFG(Solver* s) } } -void adaptUV(Solver* s) +void adaptUV(Discretization* s) { int imaxLocal = s->comm.imaxLocal; int jmaxLocal = s->comm.jmaxLocal; @@ -486,8 +394,8 @@ void adaptUV(Solver* s) double* f = s->f; double* g = s->g; - double factorX = s->dt / s->dx; - double factorY = s->dt / s->dy; + double factorX = s->dt / s->grid.dx; + double factorY = s->dt / s->grid.dy; for (int j = 1; j < jmaxLocal + 1; j++) { for (int i = 1; i < imaxLocal + 1; i++) { @@ -497,12 +405,12 @@ void adaptUV(Solver* s) } } -void writeResult(Solver* s, double* u, double* v, double* p) +void writeResult(Discretization* s, double* u, double* v, double* p) { - int imax = s->imax; - int jmax = s->jmax; - double dx = s->dx; - double dy = s->dy; + int imax = s->grid.imax; + int jmax = s->grid.jmax; + double dx = s->grid.dx; + double dy = s->grid.dy; double x = 0.0, y = 0.0; FILE* fp; @@ -513,11 +421,11 @@ void writeResult(Solver* s, double* u, double* v, double* p) exit(EXIT_FAILURE); } - for (int j = 1; j < jmax; j++) { + for (int j = 1; j <= jmax; j++) { y = (double)(j - 0.5) * dy; - for (int i = 1; i < imax; i++) { + for (int i = 1; i <= imax; i++) { x = (double)(i - 0.5) * dx; - fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax) + i]); + fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax + 2) + i]); } fprintf(fp, "\n"); } @@ -531,14 +439,14 @@ void writeResult(Solver* s, double* u, double* v, double* p) exit(EXIT_FAILURE); } - for (int j = 1; j < jmax; j++) { + for (int j = 1; j <= jmax; j++) { y = dy * (j - 0.5); - for (int i = 1; i < imax; i++) { - x = dx * (i - 0.5); - double vel_u = (u[j * (imax) + i] + u[j * (imax) + (i - 1)]) / 2.0; - double vel_v = (v[j * (imax) + i] + v[(j - 1) * (imax) + i]) / 2.0; - double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); - fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); + for (int i = 1; i <= imax; i++) { + x = dx * (i - 0.5); + double velU = (u[j * (imax + 2) + i] + u[j * (imax + 2) + (i - 1)]) / 2.0; + double velV = (v[j * (imax + 2) + i] + v[(j - 1) * (imax + 2) + i]) / 2.0; + double len = sqrt((velU * velU) + (velV * velV)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, velU, velV, len); } } diff --git a/BasicSolver/2D-mpi/src/discretization.h b/BasicSolver/2D-mpi/src/discretization.h new file mode 100644 index 0000000..a20e295 --- /dev/null +++ b/BasicSolver/2D-mpi/src/discretization.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ +#include "comm.h" +#include "grid.h" +#include "parameter.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double re, tau, gamma; + double gx, gy; + /* time stepping */ + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop; + /* communication */ + Comm comm; +} Discretization; + +void initDiscretiztion(Discretization*, Parameter*); +void computeRHS(Discretization*); +void normalizePressure(Discretization*); +void computeTimestep(Discretization*); +void setBoundaryConditions(Discretization*); +void setSpecialBoundaryCondition(Discretization*); +void computeFG(Discretization*); +void adaptUV(Discretization*); +void writeResult(Discretization* s, double* u, double* v, double* p); +#endif diff --git a/BasicSolver/2D-mpi/src/grid.h b/BasicSolver/2D-mpi/src/grid.h new file mode 100644 index 0000000..7e58d1a --- /dev/null +++ b/BasicSolver/2D-mpi/src/grid.h @@ -0,0 +1,16 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __GRID_H_ +#define __GRID_H_ + +typedef struct { + double dx, dy; + int imax, jmax; + double xlength, ylength; +} Grid; + +#endif // __GRID_H_ diff --git a/BasicSolver/2D-mpi/src/main.c b/BasicSolver/2D-mpi/src/main.c index 41bb084..8f93767 100644 --- a/BasicSolver/2D-mpi/src/main.c +++ b/BasicSolver/2D-mpi/src/main.c @@ -1,95 +1,121 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ -#include -#include #include #include #include #include "allocate.h" +#include "comm.h" +#include "discretization.h" #include "parameter.h" #include "progress.h" #include "solver.h" #include "timing.h" -#include + +static void writeResults(Discretization* s) +{ +#ifdef _MPI + size_t bytesize = (s->grid.imax + 2) * (s->grid.jmax + 2) * sizeof(double); + + double* ug = allocate(64, bytesize); + double* vg = allocate(64, bytesize); + double* pg = allocate(64, bytesize); + + commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->grid.imax, s->grid.jmax); + if (commIsMaster(&s->comm)) { + writeResult(s, ug, vg, pg); + } + + free(ug); + free(vg); + free(pg); +#else + writeResult(s, s->u, s->v, s->p); +#endif +} int main(int argc, char** argv) { int rank; - double S, E; - Parameter params; - Solver solver; + double timeStart, timeStop; + Parameter p; + Discretization d; + Solver s; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - initParameter(¶ms); + commInit(&d.comm, argc, argv); + initParameter(&p); + + FILE* fp; + if (commIsMaster(&d.comm)) fp = initResidualWriter(); if (argc != 2) { printf("Usage: %s \n", argv[0]); exit(EXIT_SUCCESS); } - readParameter(¶ms, argv[1]); - if (rank == 0) { - printParameter(¶ms); + readParameter(&p, argv[1]); + commPartition(&d.comm, p.jmax, p.imax); + if (commIsMaster(&d.comm)) { + printParameter(&p); } - initSolver(&solver, ¶ms); - /* debugExchange(&solver); */ - /* exit(EXIT_SUCCESS); */ - initProgress(solver.te); - double tau = solver.tau; - double te = solver.te; + initDiscretiztion(&d, &p); + initSolver(&s, &d, &p); +#ifdef TEST + commPrintConfig(&d.comm); + commTestInit(&d.comm, d.p, d.f, d.g); + commExchange(&d.comm, d.p); + commShift(&d.comm, d.f, d.g); + commTestWrite(&d.comm, d.p, d.f, d.g); + writeResults(&d); + commFinalize(&d.comm); + exit(EXIT_SUCCESS); +#endif +#ifndef VERBOSE + initProgress(d.te); +#endif + + double tau = d.tau; + double te = d.te; double t = 0.0; + double res = 0.0; - S = getTimeStamp(); + timeStart = getTimeStamp(); while (t <= te) { - if (tau > 0.0) { - computeTimestep(&solver); - } + + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + computeFG(&d); + computeRHS(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); - setBoundaryConditions(&solver); - setSpecialBoundaryCondition(&solver); - computeFG(&solver); - computeRHS(&solver); - solve(&solver); - adaptUV(&solver); - t += solver.dt; + if (commIsMaster(&d.comm)) writeResidual(fp, t, res); + + t += d.dt; #ifdef VERBOSE - if (rank == 0) { - printf("TIME %f , TIMESTEP %f\n", t, solver.dt); + if (commIsMaster(s.comm)) { + printf("TIME %f , TIMESTEP %f\n", t, d.dt); } #else printProgress(t); #endif } - E = getTimeStamp(); + timeStop = getTimeStamp(); +#ifndef VERBOSE stopProgress(); - if (rank == 0) { - printf("Solution took %.2fs\n", E - S); +#endif + if (commIsMaster(s.comm)) { + printf("Solution took %.2fs\n", timeStop - timeStart); } - size_t bytesize = solver.imax * solver.jmax * sizeof(double); - - double* ug = allocate(64, bytesize); - double* vg = allocate(64, bytesize); - double* pg = allocate(64, bytesize); - - commCollectResult(&solver.comm, - ug, - vg, - pg, - solver.u, - solver.v, - solver.p, - solver.jmax, - solver.imax); - writeResult(&solver, ug, vg, pg); - - MPI_Finalize(); + if (commIsMaster(&d.comm)) fclose(fp); + writeResults(&d); + commFinalize(s.comm); return EXIT_SUCCESS; } diff --git a/BasicSolver/2D-mpi/src/parameter.c b/BasicSolver/2D-mpi/src/parameter.c index 84304aa..e5e7029 100644 --- a/BasicSolver/2D-mpi/src/parameter.c +++ b/BasicSolver/2D-mpi/src/parameter.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -14,13 +14,16 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->imax = 100; - param->jmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.8; + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.8; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) @@ -72,6 +75,9 @@ void readParameter(Parameter* param, const char* filename) PARSE_INT(bcRight); PARSE_INT(bcBottom); PARSE_INT(bcTop); + PARSE_INT(levels); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); PARSE_REAL(u_init); PARSE_REAL(v_init); PARSE_REAL(p_init); diff --git a/BasicSolver/2D-mpi/src/parameter.h b/BasicSolver/2D-mpi/src/parameter.h index f4c331a..ed765eb 100644 --- a/BasicSolver/2D-mpi/src/parameter.h +++ b/BasicSolver/2D-mpi/src/parameter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -18,6 +18,7 @@ typedef struct { char* name; int bcLeft, bcRight, bcBottom, bcTop; double u_init, v_init, p_init; + int levels, presmooth, postsmooth; } Parameter; void initParameter(Parameter*); diff --git a/BasicSolver/2D-mpi/src/progress.c b/BasicSolver/2D-mpi/src/progress.c index 31a8a90..4f3b22e 100644 --- a/BasicSolver/2D-mpi/src/progress.c +++ b/BasicSolver/2D-mpi/src/progress.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -7,54 +7,64 @@ #include #include #include -#include #include - +#include #include "progress.h" static double _end; static int _current; -static int _rank = -1; void initProgress(double end) { - MPI_Comm_rank(MPI_COMM_WORLD, &_rank); _end = end; _current = 0; - if (_rank == 0) { - printf("[ ]"); - fflush(stdout); - } + printf("[ ]"); + fflush(stdout); } void printProgress(double current) { - if (_rank == 0) { - int new = (int)rint((current / _end) * 10.0); + int new = (int)rint((current / _end) * 10.0); - if (new > _current) { - char progress[11]; - _current = new; - progress[0] = 0; + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; - for (int i = 0; i < 10; i++) { - if (i < _current) { - sprintf(progress + strlen(progress), "#"); - } else { - sprintf(progress + strlen(progress), " "); - } + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); } - printf("\r[%s]", progress); } - fflush(stdout); + printf("\r[%s]", progress); } + fflush(stdout); } void stopProgress() { - if (_rank == 0) { - printf("\n"); - fflush(stdout); - } + printf("\n"); + fflush(stdout); +} + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); } diff --git a/BasicSolver/2D-mpi/src/progress.h b/BasicSolver/2D-mpi/src/progress.h index 9ef2d96..6803315 100644 --- a/BasicSolver/2D-mpi/src/progress.h +++ b/BasicSolver/2D-mpi/src/progress.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -10,5 +10,6 @@ extern void initProgress(double); extern void printProgress(double); extern void stopProgress(); - +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/BasicSolver/2D-mpi/src/solver-mg.c b/BasicSolver/2D-mpi/src/solver-mg.c new file mode 100644 index 0000000..23d64f3 --- /dev/null +++ b/BasicSolver/2D-mpi/src/solver-mg.c @@ -0,0 +1,302 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +// #define S(i, j) s[(j) * (imaxLocal + 2) + (i)] +#define E(i, j) e[(j) * (imaxLocal + 2) + (i)] +#define R(i, j) r[(j) * (imaxLocal + 2) + (i)] +#define OLD(i, j) old[(j) * (imaxLocal + 2) + (i)] + +static void restrictMG(Solver* s, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double* r = s->r[level + 1]; + double* old = s->r[level]; + +#ifdef _MPI + commExchange(comm, old); +#endif + + for (int j = 1; j < (jmaxLocal / 2) + 1; j++) { + for (int i = 1; i < (imaxLocal / 2) + 1; i++) { + R(i, j) = (OLD(2 * i - 1, 2 * j - 1) + OLD(2 * i, 2 * j - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1) + OLD(2 * i - 1, 2 * j) * 2 + + OLD(2 * i, 2 * j) * 4 + OLD(2 * i + 1, 2 * j) * 2 + + OLD(2 * i - 1, 2 * j + 1) + OLD(2 * i, 2 * j + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1)) / + 16.0; + } + } +} + +static void prolongate(Solver* s, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double* old = s->r[level + 1]; + double* e = s->r[level]; + + for (int j = 2; j < jmaxLocal + 1; j += 2) { + for (int i = 2; i < imaxLocal + 1; i += 2) { + E(i, j) = OLD(i / 2, j / 2); + } + } +} + +static void correct(Solver* s, double* p, int level, Comm* comm) +{ + double* e = s->e[level]; + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + for (int j = 1; j < jmaxLocal + 1; ++j) { + for (int i = 1; i < imaxLocal + 1; ++i) { + P(i, j) += E(i, j); + } + } +} + +static void setBoundaryCondition(Solver* s, double* p, int imaxLocal, int jmaxLocal) +{ +#ifdef _MPI + if (commIsBoundary(s->comm, BOTTOM)) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (commIsBoundary(s->comm, TOP)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (commIsBoundary(s->comm, LEFT)) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (commIsBoundary(s->comm, RIGHT)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } +#else + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + P(imaxLocal + 1, j) = P(imaxLocal, j); + } +#endif +} + +static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + +#ifdef _MPI + commExchange(comm, p); +#endif + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + P(i, j) -= factor * + (RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + +#ifdef _MPI + commExchange(comm, p); +#endif + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + R(i, j) = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + res += (R(i, j) * R(i, j)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + +#ifdef _MPI + commReduction(&res, SUM); +#endif + + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + return res; +} + +static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, comm); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + } + + // calculate residuals + res = calculateResidual(s, p, rhs, level, comm); + + // restrict + restrictMG(s, level, comm); + + Comm newcomm; + commUpdateDatatypes(s->comm, &newcomm, comm->imaxLocal, comm->jmaxLocal); + + + // MGSolver on residual and error. + multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm); + + commFreeCommunicator(&newcomm); + + // prolongate + prolongate(s, level, comm); + + // correct p on finer level using residual + correct(s, p, level, comm); + + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->comm = &d->comm; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int levels = s->levels; + printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * sizeof(double); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size); + s->e[j] = allocate(64, size); + + for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->comm); + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Residuum: %.6f\n", res); + } +#endif + + return res; +} diff --git a/BasicSolver/2D-mpi/src/solver-rb.c b/BasicSolver/2D-mpi/src/solver-rb.c new file mode 100644 index 0000000..622a11e --- /dev/null +++ b/BasicSolver/2D-mpi/src/solver-rb.c @@ -0,0 +1,106 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "discretization.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->comm = &d->comm; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double epssq = eps * eps; + int pass, jsw, isw; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + jsw = 1; + for (pass = 0; pass < 2; pass++) { + isw = jsw; + commExchange(s->comm, p); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + + if (commIsBoundary(s->comm, BOTTOM)) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (commIsBoundary(s->comm, TOP)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (commIsBoundary(s->comm, LEFT)) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (commIsBoundary(s->comm, RIGHT)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } + + commReduction(&res, SUM); + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + it++; + } + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + + return res; +} diff --git a/BasicSolver/2D-mpi/src/solver.h b/BasicSolver/2D-mpi/src/solver.h index b13dbe2..dcca9bd 100644 --- a/BasicSolver/2D-mpi/src/solver.h +++ b/BasicSolver/2D-mpi/src/solver.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -7,41 +7,23 @@ #ifndef __SOLVER_H_ #define __SOLVER_H_ #include "comm.h" +#include "discretization.h" +#include "grid.h" +#include "mpi.h" #include "parameter.h" -enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; - typedef struct { /* geometry and grid information */ - double dx, dy; - int imax, jmax; - double xlength, ylength; - /* arrays */ - double *p, *rhs; - double *f, *g; - double *u, *v; + Grid* grid; /* parameters */ double eps, omega; - double re, tau, gamma; - double gx, gy; - /* time stepping */ int itermax; - double dt, te; - double dtBound; - char* problem; - int bcLeft, bcRight, bcBottom, bcTop; + int levels, presmooth, postsmooth; + double **r, **e; /* communication */ - Comm comm; + Comm* comm; } Solver; -void initSolver(Solver*, Parameter*); -void computeRHS(Solver*); -int solve(Solver*); -void normalizePressure(Solver*); -void computeTimestep(Solver*); -void setBoundaryConditions(Solver*); -void setSpecialBoundaryCondition(Solver*); -void computeFG(Solver*); -void adaptUV(Solver*); -void writeResult(Solver* s, double* u, double* v, double* p); +void initSolver(Solver*, Discretization*, Parameter*); +double solve(Solver*, double*, double*); #endif diff --git a/BasicSolver/2D-mpi/src/timing.c b/BasicSolver/2D-mpi/src/timing.c index c4025a4..78b01c4 100644 --- a/BasicSolver/2D-mpi/src/timing.c +++ b/BasicSolver/2D-mpi/src/timing.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,18 +7,16 @@ #include #include -double getTimeStamp() +double getTimeStamp(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } -double getTimeResolution() +double getTimeResolution(void) { struct timespec ts; clock_getres(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } - -double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-mpi/src/timing.h b/BasicSolver/2D-mpi/src/timing.h index db95329..ed05a8c 100644 --- a/BasicSolver/2D-mpi/src/timing.h +++ b/BasicSolver/2D-mpi/src/timing.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,8 +7,7 @@ #ifndef __TIMING_H_ #define __TIMING_H_ -extern double getTimeStamp(); -extern double getTimeResolution(); -extern double getTimeStamp_(); +extern double getTimeStamp(void); +extern double getTimeResolution(void); #endif // __TIMING_H_ diff --git a/BasicSolver/2D-mpi/src/util.h b/BasicSolver/2D-mpi/src/util.h index 657b009..e36948a 100644 --- a/BasicSolver/2D-mpi/src/util.h +++ b/BasicSolver/2D-mpi/src/util.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -19,4 +19,11 @@ #define ABS(a) ((a) >= 0 ? (a) : -(a)) #endif +#define P(i, j) p[(j) * (imaxLocal + 2) + (i)] +#define F(i, j) f[(j) * (imaxLocal + 2) + (i)] +#define G(i, j) g[(j) * (imaxLocal + 2) + (i)] +#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] +#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)] + #endif // __UTIL_H_ diff --git a/BasicSolver/2D-mpi/velocity.png b/BasicSolver/2D-mpi/velocity.png deleted file mode 100644 index a8c9f05..0000000 Binary files a/BasicSolver/2D-mpi/velocity.png and /dev/null differ diff --git a/BasicSolver/2D-seq-pt/src/affinity.c b/BasicSolver/2D-seq-pt/src/affinity.c deleted file mode 100644 index b501665..0000000 --- a/BasicSolver/2D-seq-pt/src/affinity.c +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifdef __linux__ -#ifdef _OPENMP -#include -#include -#include -#include -#include -#include -#include - -#define MAX_NUM_THREADS 128 -#define gettid() syscall(SYS_gettid) - -static int getProcessorID(cpu_set_t* cpu_set) -{ - int processorId; - - for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { - if (CPU_ISSET(processorId, cpu_set)) { - break; - } - } - return processorId; -} - -int affinity_getProcessorId() -{ - cpu_set_t cpu_set; - CPU_ZERO(&cpu_set); - sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); - - return getProcessorID(&cpu_set); -} - -void affinity_pinThread(int processorId) -{ - cpu_set_t cpuset; - pthread_t thread; - - thread = pthread_self(); - CPU_ZERO(&cpuset); - CPU_SET(processorId, &cpuset); - pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); -} - -void affinity_pinProcess(int processorId) -{ - cpu_set_t cpuset; - - CPU_ZERO(&cpuset); - CPU_SET(processorId, &cpuset); - sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); -} -#endif /*_OPENMP*/ -#endif /*__linux__*/ diff --git a/BasicSolver/2D-seq-pt/src/affinity.h b/BasicSolver/2D-seq-pt/src/affinity.h deleted file mode 100644 index d844fe5..0000000 --- a/BasicSolver/2D-seq-pt/src/affinity.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifndef AFFINITY_H -#define AFFINITY_H - -extern int affinity_getProcessorId(); -extern void affinity_pinProcess(int); -extern void affinity_pinThread(int); - -#endif /*AFFINITY_H*/ diff --git a/BasicSolver/2D-seq-pt/src/likwid-marker.h b/BasicSolver/2D-seq-pt/src/likwid-marker.h deleted file mode 100644 index eb7cc78..0000000 --- a/BasicSolver/2D-seq-pt/src/likwid-marker.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * ======================================================================================= - * - * Author: Jan Eitzinger (je), jan.eitzinger@fau.de - * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - * ======================================================================================= - */ -#ifndef LIKWID_MARKERS_H -#define LIKWID_MARKERS_H - -#ifdef LIKWID_PERFMON -#include -#define LIKWID_MARKER_INIT likwid_markerInit() -#define LIKWID_MARKER_THREADINIT likwid_markerThreadInit() -#define LIKWID_MARKER_SWITCH likwid_markerNextGroup() -#define LIKWID_MARKER_REGISTER(regionTag) likwid_markerRegisterRegion(regionTag) -#define LIKWID_MARKER_START(regionTag) likwid_markerStartRegion(regionTag) -#define LIKWID_MARKER_STOP(regionTag) likwid_markerStopRegion(regionTag) -#define LIKWID_MARKER_CLOSE likwid_markerClose() -#define LIKWID_MARKER_RESET(regionTag) likwid_markerResetRegion(regionTag) -#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) \ - likwid_markerGetRegion(regionTag, nevents, events, time, count) -#else /* LIKWID_PERFMON */ -#define LIKWID_MARKER_INIT -#define LIKWID_MARKER_THREADINIT -#define LIKWID_MARKER_SWITCH -#define LIKWID_MARKER_REGISTER(regionTag) -#define LIKWID_MARKER_START(regionTag) -#define LIKWID_MARKER_STOP(regionTag) -#define LIKWID_MARKER_CLOSE -#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count) -#define LIKWID_MARKER_RESET(regionTag) -#endif /* LIKWID_PERFMON */ - -#endif /*LIKWID_MARKERS_H*/ diff --git a/BasicSolver/2D-seq-pt/src/main.c b/BasicSolver/2D-seq-pt/src/main.c deleted file mode 100644 index a67fe84..0000000 --- a/BasicSolver/2D-seq-pt/src/main.c +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include - -#include "parameter.h" -#include "progress.h" -#include "solver.h" -#include "timing.h" -#include "trace.h" - -int main(int argc, char** argv) -{ - double timeStart, timeEnd; - Parameter p; - Solver s; - Tracing t; - initParameter(&p); - - if (argc != 2) { - printf("Usage: %s \n", argv[0]); - exit(EXIT_SUCCESS); - } - - readParameter(&p, argv[1]); - printParameter(&p); - initSolver(&s, &p); - initTrace(&t, &p); -#ifndef VERBOSE - initProgress(s.te); -#endif - - double tau = s.tau; - double te = s.te; - double time = 0.0; - int nt = 0; - - timeStart = getTimeStamp(); - while (time <= te) { - if (tau > 0.0) computeTimestep(&s); - setBoundaryConditions(&s); - setSpecialBoundaryCondition(&s); - computeFG(&s); - computeRHS(&s); - if (nt % 100 == 0) normalizePressure(&s); - solve(&s); - adaptUV(&s); - time += s.dt; - nt++; - - trace(&t, s.u, s.v, time); - -#ifdef VERBOSE - printf("TIME %f , TIMESTEP %f\n", time, s.dt); -#else - printProgress(time); -#endif - } - timeEnd = getTimeStamp(); - stopProgress(); - printf("Solution took %.2fs\n", timeEnd - timeStart); - writeResult(&s); - return EXIT_SUCCESS; -} diff --git a/BasicSolver/2D-seq-pt/src/solver.h b/BasicSolver/2D-seq-pt/src/solver.h deleted file mode 100644 index 42d5fee..0000000 --- a/BasicSolver/2D-seq-pt/src/solver.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#ifndef __SOLVER_H_ -#define __SOLVER_H_ -#include "parameter.h" - -#define U(i, j) u[(j) * (imax + 2) + (i)] -#define V(i, j) v[(j) * (imax + 2) + (i)] - -enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; - -typedef struct { - /* geometry and grid information */ - double dx, dy; - int imax, jmax; - double xlength, ylength; - /* arrays */ - double *p, *rhs; - double *f, *g; - double *u, *v; - /* parameters */ - double eps, omega; - double re, tau, gamma; - double gx, gy; - /* time stepping */ - int itermax; - double dt, te; - double dtBound; - char* problem; - int bcLeft, bcRight, bcBottom, bcTop; -} Solver; - -void initSolver(Solver*, Parameter*); -void computeRHS(Solver*); -void solve(Solver*); -void normalizePressure(Solver*); -void computeTimestep(Solver*); -void setBoundaryConditions(Solver*); -void setSpecialBoundaryCondition(Solver*); -void computeFG(Solver*); -void adaptUV(Solver*); -void writeResult(Solver*); -#endif diff --git a/BasicSolver/2D-seq-pt/src/trace.c b/BasicSolver/2D-seq-pt/src/trace.c deleted file mode 100644 index 6221eb9..0000000 --- a/BasicSolver/2D-seq-pt/src/trace.c +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include - -#include "trace.h" -#define U(i, j) u[(j) * (imax + 2) + (i)] -#define V(i, j) v[(j) * (imax + 2) + (i)] - -static int ts = 0; - -static void printState(Tracing* t) -{ - printf("Cursor: %d Total particles: %d\n", t->cursor, t->totalParticles); -} - -static void advanceParticles( - Tracing* t, double delt, double* restrict u, double* restrict v) -{ - double delx = t->grid.dx; - double dely = t->grid.dy; - - double* m = t->memorypool; - int* p = t->particles; - int imax = t->grid.imax; - int jmax = t->grid.jmax; - - for (int i = 0; i < t->totalParticles; i++) { - int particleId = p[i]; - - double x = m[particleId * NCOORD + X]; - double y = m[particleId * NCOORD + Y]; - // printf("P%d - X %f Y %f\n", i, x, y); - - // Interpolate U - int iCoord = (int)(x / delx) + 1; - int jCoord = (int)((y + 0.5 * dely) / dely) + 1; - - double x1 = (double)(iCoord - 1) * delx; - double y1 = ((double)(jCoord - 1) - 0.5) * dely; - double x2 = (double)iCoord * delx; - double y2 = ((double)jCoord - 0.5) * dely; - - // printf("U - iCoord %d jCoord %d\n", iCoord, jCoord); - - double un = (1.0 / (delx * dely)) * - ((x2 - x) * (y2 - y) * U(iCoord - 1, jCoord - 1) + - (x - x1) * (y2 - y) * U(iCoord, jCoord - 1) + - (x2 - x) * (y - y1) * U(iCoord - 1, jCoord) + - (x - x1) * (y - y1) * U(iCoord, jCoord)); - - double xn = x + delt * un; - m[particleId * NCOORD + X] = xn; - - // Interpolate V - iCoord = (int)((x + 0.5 * delx) / delx) + 1; - jCoord = (int)(y / dely) + 1; - - x1 = ((double)(iCoord - 1) - 0.5) * delx; - y1 = (double)(jCoord - 1) * dely; - x2 = ((double)iCoord - 0.5) * delx; - y2 = (double)jCoord * dely; - - // printf("V - iCoord %d jCoord %d\n", iCoord, jCoord); - - double vn = (1.0 / (delx * dely)) * - ((x2 - x) * (y2 - y) * V(iCoord - 1, jCoord - 1) + - (x - x1) * (y2 - y) * V(iCoord, jCoord - 1) + - (x2 - x) * (y - y1) * V(iCoord - 1, jCoord) + - (x - x1) * (y - y1) * V(iCoord, jCoord)); - - double yn = y + delt * vn; - m[particleId * NCOORD + Y] = yn; - printf("P%i VEL %f %f dt %f OP %f %f NP %f %f\n", i, un, vn, delt, x, y, xn, yn); - } - - double xlength = t->grid.xlength; - double ylength = t->grid.ylength; - int cntNew = 0; - int tmp[t->totalParticles]; - - // Check for particles to remove - for (int i = 0; i < t->totalParticles; i++) { - int particleId = p[i]; - - double x = m[particleId * NCOORD + X]; - double y = m[particleId * NCOORD + Y]; - - if (!((x < 0.0) || (x > xlength) || (y < 0.0) || (y > ylength))) { - tmp[cntNew++] = i; - } - } - - t->totalParticles = cntNew; - memcpy(t->particles, tmp, cntNew * sizeof(int)); -} - -static void injectParticles(Tracing* t) -{ - double* line = t->line; - double* m = t->memorypool; - - for (int i = 0; i < t->numParticles; i++) { - printf("Inject %d as %d mem %d\n", i, t->totalParticles, t->cursor); - t->particles[t->totalParticles] = t->cursor; - m[(t->cursor) * NCOORD + X] = line[i * NCOORD + X]; - m[(t->cursor) * NCOORD + Y] = line[i * NCOORD + Y]; - t->cursor++; - t->totalParticles++; - } -} - -static void writeParticles(Tracing* t) -{ - FILE* fp; - double* m = t->memorypool; - int* p = t->particles; - - char filename[50]; - snprintf(filename, 50, "particles_%d.dat", ts++); - fp = fopen(filename, "w"); - - if (fp == NULL) { - printf("Error!\n"); - exit(EXIT_FAILURE); - } - - for (int i = 0; i < t->totalParticles; i++) { - int particleId = p[i]; - - double x = m[particleId * NCOORD + X]; - double y = m[particleId * NCOORD + Y]; - fprintf(fp, "%f %f\n", x, y); - } - fclose(fp); -} - -void trace(Tracing* t, double* restrict u, double* restrict v, double time) -{ - if (time >= t->traceStart) { - if ((time - t->lastUpdate[INJECT]) > t->traceInject) { - printf("Inject at %f\n", time); - printState(t); - injectParticles(t); - t->lastUpdate[INJECT] = time; - } - - if ((time - t->lastUpdate[WRITE]) > t->traceWrite) { - printf("Write at %f\n", time); - writeParticles(t); - t->lastUpdate[WRITE] = time; - } - - advanceParticles(t, time - t->lastUpdate[ADVANCE], u, v); - t->lastUpdate[ADVANCE] = time; - } -} - -void initTrace(Tracing* t, Parameter* p) -{ - size_t numParticles = p->nparticles; - size_t totalParticles = (size_t)(p->te - p->traceStart) / (size_t)p->traceInject; - totalParticles += 2; - totalParticles *= numParticles; - - double x1 = p->lineX1; - double y1 = p->lineY1; - double x2 = p->lineX2; - double y2 = p->lineY2; - - for (int i = 0; i < NUMTIMERS; i++) { - t->lastUpdate[i] = p->traceStart; - } - t->grid.imax = p->imax; - t->grid.jmax = p->jmax; - t->grid.xlength = p->xlength; - t->grid.ylength = p->ylength; - t->grid.dx = p->xlength / p->imax; - t->grid.dy = p->ylength / p->jmax; - t->numParticles = numParticles; - t->totalParticles = 0; - t->cursor = 0; - t->traceStart = p->traceStart; - t->traceWrite = p->traceWrite; - t->traceInject = p->traceInject; - t->particles = (int*)malloc(totalParticles * sizeof(int)); - t->memorypool = (double*)malloc(totalParticles * NCOORD * sizeof(double)); - t->line = (double*)malloc(numParticles * NCOORD * sizeof(double)); - double* line = t->line; - - for (int i = 0; i < numParticles; i++) { - double spacing = (double)i / (double)(numParticles - 1); - double x = spacing * x1 + (1.0 - spacing) * x2; - double y = spacing * y1 + (1.0 - spacing) * y2; - - printf("S: %f x: %f y: %f\n", spacing, x, y); - line[i * NCOORD + X] = x; - line[i * NCOORD + Y] = y; - } -} - -void freeTrace(Tracing* t) { free(t->line); } diff --git a/BasicSolver/2D-seq-pt/src/trace.h b/BasicSolver/2D-seq-pt/src/trace.h deleted file mode 100644 index 4ab56a3..0000000 --- a/BasicSolver/2D-seq-pt/src/trace.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#ifndef __TRACE_H_ -#define __TRACE_H_ -#include "grid.h" -#include "parameter.h" - -typedef enum COORD { X = 0, Y, NCOORD } COORD; -typedef enum { ADVANCE = 0, INJECT, WRITE, NUMTIMERS } TIMER; - -typedef struct Tracing { - double traceStart; - double traceWrite; - double traceInject; - double dt; - double lastUpdate[NUMTIMERS]; - double* memorypool; - double* line; - int cursor; - int* particles; - int numParticles; - int totalParticles; - Grid grid; -} Tracing; - -extern void initTrace(Tracing* t, Parameter* p); -extern void trace(Tracing* t, double* u, double* v, double time); -#endif diff --git a/BasicSolver/2D-seq-pt/src/util.h b/BasicSolver/2D-seq-pt/src/util.h deleted file mode 100644 index 61a1dff..0000000 --- a/BasicSolver/2D-seq-pt/src/util.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifndef __UTIL_H_ -#define __UTIL_H_ -#define HLINE \ - "------------------------------------------------------------------------" \ - "----\n" - -#ifndef MIN -#define MIN(x, y) ((x) < (y) ? (x) : (y)) -#endif -#ifndef MAX -#define MAX(x, y) ((x) > (y) ? (x) : (y)) -#endif -#ifndef ABS -#define ABS(a) ((a) >= 0 ? (a) : -(a)) -#endif - -#endif // __UTIL_H_ diff --git a/BasicSolver/2D-seq-pt/surface.plot b/BasicSolver/2D-seq-pt/surface.plot deleted file mode 100644 index 4f7ccd9..0000000 --- a/BasicSolver/2D-seq-pt/surface.plot +++ /dev/null @@ -1,7 +0,0 @@ -set terminal png size 1024,768 enhanced font ,12 -set output 'p.png' -set datafile separator whitespace - -set grid -set hidden3d -splot 'pressure.dat' using 1:2:3 with lines diff --git a/BasicSolver/2D-seq/Makefile b/BasicSolver/2D-seq/Makefile index 57f99f4..d33dd17 100644 --- a/BasicSolver/2D-seq/Makefile +++ b/BasicSolver/2D-seq/Makefile @@ -1,5 +1,5 @@ #======================================================================================= -# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. # All rights reserved. # Use of this source code is governed by a MIT-style # license that can be found in the LICENSE file. @@ -18,9 +18,10 @@ include $(MAKE_DIR)/include_$(TAG).mk INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) VPATH = $(SRC_DIR) -SRC = $(wildcard $(SRC_DIR)/*.c) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) @@ -37,9 +38,22 @@ $(BUILD_DIR)/%.s: %.c $(info ===> GENERATE ASM $@) $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ -.PHONY: clean distclean tags info asm format +.PHONY: clean distclean vis vis_clean tags info asm format -clean: +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='pressure.dat'" ./surface.plot + @gnuplot -e "filename='velocity.dat'" ./vector.plot + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.png + @rm -f ./vis_files/*.dat + @rm -f ./vis_files/*.gif + +clean: vis_clean $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @rm -f tags @@ -47,6 +61,8 @@ clean: distclean: clean $(info ===> DIST CLEAN) @rm -f $(TARGET) + @rm -f *.dat + @rm -f *.png info: $(info $(CFLAGS)) diff --git a/BasicSolver/2D-seq/canal.par b/BasicSolver/2D-seq/canal.par index 8b9fd9c..6ed28e6 100644 --- a/BasicSolver/2D-seq/canal.par +++ b/BasicSolver/2D-seq/canal.par @@ -36,6 +36,13 @@ te 100.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- diff --git a/BasicSolver/2D-seq/config.mk b/BasicSolver/2D-seq/config.mk index af5f1a0..0b2c509 100644 --- a/BasicSolver/2D-seq/config.mk +++ b/BasicSolver/2D-seq/config.mk @@ -1,12 +1,12 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC ENABLE_OPENMP ?= false +# Supported: sor, rb, mg +SOLVER ?= mg +# Run in debug settings +DEBUG ?= false #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 OPTIONS += -DVERBOSE #OPTIONS += -DDEBUG -#OPTIONS += -DBOUNDCHECK -#OPTIONS += -DVERBOSE_AFFINITY -#OPTIONS += -DVERBOSE_DATASIZE -#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/2D-seq/dcavity.par b/BasicSolver/2D-seq/dcavity.par index 4241393..c4366a1 100644 --- a/BasicSolver/2D-seq/dcavity.par +++ b/BasicSolver/2D-seq/dcavity.par @@ -15,7 +15,7 @@ bcRight 1 # gx 0.0 # Body forces (e.g. gravity) gy 0.0 # -re 10.0 # Reynolds number +re 100.0 # Reynolds number u_init 0.0 # initial value for velocity in x-direction v_init 0.0 # initial value for velocity in y-direction @@ -26,8 +26,8 @@ p_init 0.0 # initial value for pressure xlength 1.0 # domain size in x-direction ylength 1.0 # domain size in y-direction -imax 40 # number of interior cells in x-direction -jmax 40 # number of interior cells in y-direction +imax 128 # number of interior cells in x-direction +jmax 128 # number of interior cells in y-direction # Time Data: # --------- @@ -36,11 +36,19 @@ te 10.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) -# Pressure Iteration Data: +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Solver Data: # ----------------------- itermax 1000 # maximal number of pressure iteration in one time step eps 0.001 # stopping tolerance for pressure iteration +rho 0.5 omg 1.7 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma #=============================================================================== diff --git a/BasicSolver/2D-seq/include_CLANG.mk b/BasicSolver/2D-seq/include_CLANG.mk index 466f441..a124053 100644 --- a/BasicSolver/2D-seq/include_CLANG.mk +++ b/BasicSolver/2D-seq/include_CLANG.mk @@ -2,16 +2,18 @@ CC = clang GCC = cc LINKER = $(CC) -ifeq ($(ENABLE_OPENMP),true) +ifeq ($(strip $(ENABLE_OPENMP)),true) OPENMP = -fopenmp #OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp LIBS = # -lomp endif +ifeq ($(strip $(DEBUG)),true) +CFLAGS = -O0 -g -std=c17 +else +CFLAGS = -O3 -std=c17 $(OPENMP) +endif VERSION = --version -# CFLAGS = -O3 -std=c17 $(OPENMP) -CFLAGS = -Ofast -std=c17 -#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG LFLAGS = $(OPENMP) -lm -DEFINES = -D_GNU_SOURCE# -DDEBUG +DEFINES = -D_GNU_SOURCE INCLUDES = diff --git a/BasicSolver/2D-seq/residual.plot b/BasicSolver/2D-seq/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/BasicSolver/2D-seq/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/2D-seq/src/affinity.c b/BasicSolver/2D-seq/src/affinity.c deleted file mode 100644 index b501665..0000000 --- a/BasicSolver/2D-seq/src/affinity.c +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifdef __linux__ -#ifdef _OPENMP -#include -#include -#include -#include -#include -#include -#include - -#define MAX_NUM_THREADS 128 -#define gettid() syscall(SYS_gettid) - -static int getProcessorID(cpu_set_t* cpu_set) -{ - int processorId; - - for (processorId = 0; processorId < MAX_NUM_THREADS; processorId++) { - if (CPU_ISSET(processorId, cpu_set)) { - break; - } - } - return processorId; -} - -int affinity_getProcessorId() -{ - cpu_set_t cpu_set; - CPU_ZERO(&cpu_set); - sched_getaffinity(gettid(), sizeof(cpu_set_t), &cpu_set); - - return getProcessorID(&cpu_set); -} - -void affinity_pinThread(int processorId) -{ - cpu_set_t cpuset; - pthread_t thread; - - thread = pthread_self(); - CPU_ZERO(&cpuset); - CPU_SET(processorId, &cpuset); - pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); -} - -void affinity_pinProcess(int processorId) -{ - cpu_set_t cpuset; - - CPU_ZERO(&cpuset); - CPU_SET(processorId, &cpuset); - sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); -} -#endif /*_OPENMP*/ -#endif /*__linux__*/ diff --git a/BasicSolver/2D-seq/src/affinity.h b/BasicSolver/2D-seq/src/affinity.h deleted file mode 100644 index d844fe5..0000000 --- a/BasicSolver/2D-seq/src/affinity.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifndef AFFINITY_H -#define AFFINITY_H - -extern int affinity_getProcessorId(); -extern void affinity_pinProcess(int); -extern void affinity_pinThread(int); - -#endif /*AFFINITY_H*/ diff --git a/BasicSolver/2D-seq/src/allocate.c b/BasicSolver/2D-seq/src/allocate.c index 81e1e9d..cf2efd6 100644 --- a/BasicSolver/2D-seq/src/allocate.c +++ b/BasicSolver/2D-seq/src/allocate.c @@ -1,14 +1,17 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ #include +#include #include #include -void* allocate(int alignment, size_t bytesize) +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) { int errorCode; void* ptr; diff --git a/BasicSolver/2D-seq/src/allocate.h b/BasicSolver/2D-seq/src/allocate.h index 54cfe06..77f4ba0 100644 --- a/BasicSolver/2D-seq/src/allocate.h +++ b/BasicSolver/2D-seq/src/allocate.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -8,6 +8,6 @@ #define __ALLOCATE_H_ #include -extern void* allocate(int alignment, size_t bytesize); +extern void* allocate(size_t alignment, size_t bytesize); #endif diff --git a/BasicSolver/2D-seq-pt/src/solver.c b/BasicSolver/2D-seq/src/discretization.c similarity index 52% rename from BasicSolver/2D-seq-pt/src/solver.c rename to BasicSolver/2D-seq/src/discretization.c index 73333bc..184d9ac 100644 --- a/BasicSolver/2D-seq-pt/src/solver.c +++ b/BasicSolver/2D-seq/src/discretization.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -11,22 +11,17 @@ #include #include "allocate.h" +#include "discretization.h" #include "parameter.h" -#include "solver.h" #include "util.h" -#define P(i, j) p[(j) * (imax + 2) + (i)] -#define F(i, j) f[(j) * (imax + 2) + (i)] -#define G(i, j) g[(j) * (imax + 2) + (i)] -#define RHS(i, j) rhs[(j) * (imax + 2) + (i)] - -static void print(Solver* solver, double* grid) +static void print(Discretization* d, double* grid) { - int imax = solver->imax; + int imax = d->grid.imax; - for (int j = 0; j < solver->jmax + 2; j++) { + for (int j = 0; j < d->grid.jmax + 2; j++) { printf("%02d: ", j); - for (int i = 0; i < solver->imax + 2; i++) { + for (int i = 0; i < d->grid.imax + 2; i++) { printf("%12.8f ", grid[j * (imax + 2) + i]); } printf("\n"); @@ -34,92 +29,86 @@ static void print(Solver* solver, double* grid) fflush(stdout); } -static void printConfig(Solver* solver) +static void printConfig(Discretization* d) { - printf("Parameters for #%s#\n", solver->problem); + printf("Parameters for #%s#\n", d->problem); printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", - solver->bcLeft, - solver->bcRight, - solver->bcBottom, - solver->bcTop); - printf("\tReynolds number: %.2f\n", solver->re); - printf("\tGx Gy: %.2f %.2f\n", solver->gx, solver->gy); + d->bcLeft, + d->bcRight, + d->bcBottom, + d->bcTop); + printf("\tReynolds number: %.2f\n", d->re); + printf("\tGx Gy: %.2f %.2f\n", d->gx, d->gy); printf("Geometry data:\n"); - printf("\tDomain box size (x, y): %.2f, %.2f\n", solver->xlength, solver->ylength); - printf("\tCells (x, y): %d, %d\n", solver->imax, solver->jmax); + printf("\tDomain box size (x, y): %.2f, %.2f\n", d->grid.xlength, d->grid.ylength); + printf("\tCells (x, y): %d, %d\n", d->grid.imax, d->grid.jmax); printf("Timestep parameters:\n"); - printf("\tDefault stepsize: %.2f, Final time %.2f\n", solver->dt, solver->te); - printf("\tdt bound: %.6f\n", solver->dtBound); - printf("\tTau factor: %.2f\n", solver->tau); - printf("Iterative solver parameters:\n"); - printf("\tMax iterations: %d\n", solver->itermax); - printf("\tepsilon (stopping tolerance) : %f\n", solver->eps); - printf("\tgamma factor: %f\n", solver->gamma); - printf("\tomega (SOR relaxation): %f\n", solver->omega); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te); + printf("\tdt bound: %.6f\n", d->dtBound); + printf("\tTau factor: %.2f\n", d->tau); + printf("Iterative d parameters:\n"); + printf("\tgamma factor: %f\n", d->gamma); } -void initSolver(Solver* solver, Parameter* params) +void initDiscretization(Discretization* d, Parameter* p) { - solver->problem = params->name; - solver->bcLeft = params->bcLeft; - solver->bcRight = params->bcRight; - solver->bcBottom = params->bcBottom; - solver->bcTop = params->bcTop; - solver->imax = params->imax; - solver->jmax = params->jmax; - solver->xlength = params->xlength; - solver->ylength = params->ylength; - solver->dx = params->xlength / params->imax; - solver->dy = params->ylength / params->jmax; - solver->eps = params->eps; - solver->omega = params->omg; - solver->itermax = params->itermax; - solver->re = params->re; - solver->gx = params->gx; - solver->gy = params->gy; - solver->dt = params->dt; - solver->te = params->te; - solver->tau = params->tau; - solver->gamma = params->gamma; + d->problem = p->name; + d->bcLeft = p->bcLeft; + d->bcRight = p->bcRight; + d->bcBottom = p->bcBottom; + d->bcTop = p->bcTop; + d->grid.imax = p->imax; + d->grid.jmax = p->jmax; + d->grid.xlength = p->xlength; + d->grid.ylength = p->ylength; + d->grid.dx = p->xlength / p->imax; + d->grid.dy = p->ylength / p->jmax; + d->re = p->re; + d->gx = p->gx; + d->gy = p->gy; + d->dt = p->dt; + d->te = p->te; + d->tau = p->tau; + d->gamma = p->gamma; - int imax = solver->imax; - int jmax = solver->jmax; + int imax = d->grid.imax; + int jmax = d->grid.jmax; size_t size = (imax + 2) * (jmax + 2) * sizeof(double); - solver->u = allocate(64, size); - solver->v = allocate(64, size); - solver->p = allocate(64, size); - solver->rhs = allocate(64, size); - solver->f = allocate(64, size); - solver->g = allocate(64, size); + d->u = allocate(64, size); + d->v = allocate(64, size); + d->p = allocate(64, size); + d->rhs = allocate(64, size); + d->f = allocate(64, size); + d->g = allocate(64, size); for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { - solver->u[i] = params->u_init; - solver->v[i] = params->v_init; - solver->p[i] = params->p_init; - solver->rhs[i] = 0.0; - solver->f[i] = 0.0; - solver->g[i] = 0.0; + d->u[i] = p->u_init; + d->v[i] = p->v_init; + d->p[i] = p->p_init; + d->rhs[i] = 0.0; + d->f[i] = 0.0; + d->g[i] = 0.0; } - double dx = solver->dx; - double dy = solver->dy; + double dx = d->grid.dx; + double dy = d->grid.dy; double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy); - solver->dtBound = 0.5 * solver->re * 1.0 / invSqrSum; + d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; #ifdef VERBOSE - printConfig(solver); + printConfig(d); #endif } -void computeRHS(Solver* solver) +void computeRHS(Discretization* d) { - int imax = solver->imax; - int jmax = solver->jmax; - double idx = 1.0 / solver->dx; - double idy = 1.0 / solver->dy; - double idt = 1.0 / solver->dt; - double* rhs = solver->rhs; - double* f = solver->f; - double* g = solver->g; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double idx = 1.0 / d->grid.dx; + double idy = 1.0 / d->grid.dy; + double idt = 1.0 / d->dt; + double* rhs = d->rhs; + double* f = d->f; + double* g = d->g; for (int j = 1; j < jmax + 1; j++) { for (int i = 1; i < imax + 1; i++) { @@ -129,63 +118,9 @@ void computeRHS(Solver* solver) } } -void solve(Solver* solver) +static double maxElement(Discretization* d, double* m) { - int imax = solver->imax; - int jmax = solver->jmax; - double eps = solver->eps; - int itermax = solver->itermax; - double dx2 = solver->dx * solver->dx; - double dy2 = solver->dy * solver->dy; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); - double* p = solver->p; - double* rhs = solver->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - - for (int j = 1; j < jmax + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - - double r = RHS(i, j) - - ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + - (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - - P(i, j) -= (factor * r); - res += (r * r); - } - } - - for (int i = 1; i < imax + 1; i++) { - P(i, 0) = P(i, 1); - P(i, jmax + 1) = P(i, jmax); - } - - for (int j = 1; j < jmax + 1; j++) { - P(0, j) = P(1, j); - P(imax + 1, j) = P(imax, j); - } - - res = res / (double)(imax * jmax); -#ifdef DEBUG - printf("%d Residuum: %e\n", it, res); -#endif - it++; - } - -#ifdef VERBOSE - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); -#endif -} - -static double maxElement(Solver* solver, double* m) -{ - int size = (solver->imax + 2) * (solver->jmax + 2); + int size = (d->grid.imax + 2) * (d->grid.jmax + 2); double maxval = DBL_MIN; for (int i = 0; i < size; i++) { @@ -195,10 +130,10 @@ static double maxElement(Solver* solver, double* m) return maxval; } -void normalizePressure(Solver* solver) +void normalizePressure(Discretization* d) { - int size = (solver->imax + 2) * (solver->jmax + 2); - double* p = solver->p; + int size = (d->grid.imax + 2) * (d->grid.jmax + 2); + double* p = d->p; double avgP = 0.0; for (int i = 0; i < size; i++) { @@ -211,13 +146,13 @@ void normalizePressure(Solver* solver) } } -void computeTimestep(Solver* solver) +void computeTimestep(Discretization* d) { - double dt = solver->dtBound; - double dx = solver->dx; - double dy = solver->dy; - double umax = maxElement(solver, solver->u); - double vmax = maxElement(solver, solver->v); + double dt = d->dtBound; + double dx = d->grid.dx; + double dy = d->grid.dy; + double umax = maxElement(d, d->u); + double vmax = maxElement(d, d->v); if (umax > 0) { dt = (dt > dx / umax) ? dx / umax : dt; @@ -226,18 +161,18 @@ void computeTimestep(Solver* solver) dt = (dt > dy / vmax) ? dy / vmax : dt; } - solver->dt = dt * solver->tau; + d->dt = dt * d->tau; } -void setBoundaryConditions(Solver* solver) +void setBoundaryConditions(Discretization* d) { - int imax = solver->imax; - int jmax = solver->jmax; - double* u = solver->u; - double* v = solver->v; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double* u = d->u; + double* v = d->v; // Left boundary - switch (solver->bcLeft) { + switch (d->bcLeft) { case NOSLIP: for (int j = 1; j < jmax + 1; j++) { U(0, j) = 0.0; @@ -261,7 +196,7 @@ void setBoundaryConditions(Solver* solver) } // Right boundary - switch (solver->bcRight) { + switch (d->bcRight) { case NOSLIP: for (int j = 1; j < jmax + 1; j++) { U(imax, j) = 0.0; @@ -285,7 +220,7 @@ void setBoundaryConditions(Solver* solver) } // Bottom boundary - switch (solver->bcBottom) { + switch (d->bcBottom) { case NOSLIP: for (int i = 1; i < imax + 1; i++) { V(i, 0) = 0.0; @@ -309,7 +244,7 @@ void setBoundaryConditions(Solver* solver) } // Top boundary - switch (solver->bcTop) { + switch (d->bcTop) { case NOSLIP: for (int i = 1; i < imax + 1; i++) { V(i, jmax) = 0.0; @@ -333,19 +268,19 @@ void setBoundaryConditions(Solver* solver) } } -void setSpecialBoundaryCondition(Solver* solver) +void setSpecialBoundaryCondition(Discretization* d) { - int imax = solver->imax; - int jmax = solver->jmax; - double mDy = solver->dy; - double* u = solver->u; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double mDy = d->grid.dy; + double* u = d->u; - if (strcmp(solver->problem, "dcavity") == 0) { + if (strcmp(d->problem, "dcavity") == 0) { for (int i = 1; i < imax; i++) { U(i, jmax + 1) = 2.0 - U(i, jmax); } - } else if (strcmp(solver->problem, "canal") == 0) { - double ylength = solver->ylength; + } else if (strcmp(d->problem, "canal") == 0) { + double ylength = d->grid.ylength; double y; for (int j = 1; j < jmax + 1; j++) { @@ -355,21 +290,21 @@ void setSpecialBoundaryCondition(Solver* solver) } } -void computeFG(Solver* solver) +void computeFG(Discretization* d) { - double* u = solver->u; - double* v = solver->v; - double* f = solver->f; - double* g = solver->g; - int imax = solver->imax; - int jmax = solver->jmax; - double gx = solver->gx; - double gy = solver->gy; - double gamma = solver->gamma; - double dt = solver->dt; - double inverseRe = 1.0 / solver->re; - double inverseDx = 1.0 / solver->dx; - double inverseDy = 1.0 / solver->dy; + double* u = d->u; + double* v = d->v; + double* f = d->f; + double* g = d->g; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double gx = d->gx; + double gy = d->gy; + double gamma = d->gamma; + double dt = d->dt; + double inverseRe = 1.0 / d->re; + double inverseDx = 1.0 / d->grid.dx; + double inverseDy = 1.0 / d->grid.dy; double du2dx, dv2dy, duvdx, duvdy; double du2dx2, du2dy2, dv2dx2, dv2dy2; @@ -428,17 +363,17 @@ void computeFG(Solver* solver) } } -void adaptUV(Solver* solver) +void adaptUV(Discretization* d) { - int imax = solver->imax; - int jmax = solver->jmax; - double* p = solver->p; - double* u = solver->u; - double* v = solver->v; - double* f = solver->f; - double* g = solver->g; - double factorX = solver->dt / solver->dx; - double factorY = solver->dt / solver->dy; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double* p = d->p; + double* u = d->u; + double* v = d->v; + double* f = d->f; + double* g = d->g; + double factorX = d->dt / d->grid.dx; + double factorY = d->dt / d->grid.dy; for (int j = 1; j < jmax + 1; j++) { for (int i = 1; i < imax + 1; i++) { @@ -448,15 +383,15 @@ void adaptUV(Solver* solver) } } -void writeResult(Solver* solver) +void writeResult(Discretization* d) { - int imax = solver->imax; - int jmax = solver->jmax; - double dx = solver->dx; - double dy = solver->dy; - double* p = solver->p; - double* u = solver->u; - double* v = solver->v; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double dx = d->grid.dx; + double dy = d->grid.dy; + double* p = d->p; + double* u = d->u; + double* v = d->v; double x = 0.0, y = 0.0; FILE* fp; @@ -488,11 +423,11 @@ void writeResult(Solver* solver) for (int j = 1; j < jmax + 1; j++) { y = dy * (j - 0.5); for (int i = 1; i < imax + 1; i++) { - x = dx * (i - 0.5); - double vel_u = (U(i, j) + U(i - 1, j)) / 2.0; - double vel_v = (V(i, j) + V(i, j - 1)) / 2.0; - double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); - fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); + x = dx * (i - 0.5); + double velU = (U(i, j) + U(i - 1, j)) / 2.0; + double velV = (V(i, j) + V(i, j - 1)) / 2.0; + double len = sqrt((velU * velU) + (velV * velV)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, velU, velV, len); } } diff --git a/BasicSolver/2D-seq/src/discretization.h b/BasicSolver/2D-seq/src/discretization.h new file mode 100644 index 0000000..3007093 --- /dev/null +++ b/BasicSolver/2D-seq/src/discretization.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ +#include "grid.h" +#include "parameter.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double re, tau, gamma; + double gx, gy; + /* time stepping */ + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop; +} Discretization; + +extern void initDiscretization(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +extern void writeResult(Discretization*); +#endif diff --git a/BasicSolver/2D-seq/src/grid.h b/BasicSolver/2D-seq/src/grid.h new file mode 100644 index 0000000..7e58d1a --- /dev/null +++ b/BasicSolver/2D-seq/src/grid.h @@ -0,0 +1,16 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __GRID_H_ +#define __GRID_H_ + +typedef struct { + double dx, dy; + int imax, jmax; + double xlength, ylength; +} Grid; + +#endif // __GRID_H_ diff --git a/BasicSolver/2D-seq/src/likwid-marker.h b/BasicSolver/2D-seq/src/likwid-marker.h index eb7cc78..c3770c0 100644 --- a/BasicSolver/2D-seq/src/likwid-marker.h +++ b/BasicSolver/2D-seq/src/likwid-marker.h @@ -4,23 +4,23 @@ * Author: Jan Eitzinger (je), jan.eitzinger@fau.de * Copyright (c) 2020 RRZE, University Erlangen-Nuremberg * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. * * ======================================================================================= */ diff --git a/BasicSolver/2D-seq/src/main.c b/BasicSolver/2D-seq/src/main.c index 65b42ab..5f617fa 100644 --- a/BasicSolver/2D-seq/src/main.c +++ b/BasicSolver/2D-seq/src/main.c @@ -1,15 +1,14 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ -#include -#include #include #include #include +#include "discretization.h" #include "parameter.h" #include "progress.h" #include "solver.h" @@ -17,50 +16,61 @@ int main(int argc, char** argv) { - double S, E; - Parameter params; - Solver solver; - initParameter(¶ms); + double timeStart, timeStop; + Parameter p; + Discretization d; + Solver s; + + initParameter(&p); + FILE* fp; + fp = initResidualWriter(); if (argc != 2) { printf("Usage: %s \n", argv[0]); exit(EXIT_SUCCESS); } - readParameter(¶ms, argv[1]); - printParameter(¶ms); - initSolver(&solver, ¶ms); + readParameter(&p, argv[1]); + printParameter(&p); + initDiscretization(&d, &p); + initSolver(&s, &d, &p); + #ifndef VERBOSE - initProgress(solver.te); + initProgress(d.te); #endif - double tau = solver.tau; - double te = solver.te; + double tau = d.tau; + double te = d.te; double t = 0.0; int nt = 0; + double res = 0.0; - S = getTimeStamp(); + timeStart = getTimeStamp(); while (t <= te) { - if (tau > 0.0) computeTimestep(&solver); - setBoundaryConditions(&solver); - setSpecialBoundaryCondition(&solver); - computeFG(&solver); - computeRHS(&solver); - if (nt % 100 == 0) normalizePressure(&solver); - solveRB(&solver); - adaptUV(&solver); - t += solver.dt; + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + computeFG(&d); + computeRHS(&d); + if (nt % 100 == 0) normalizePressure(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + + writeResidual(fp, t, res); + + t += d.dt; nt++; #ifdef VERBOSE - printf("TIME %f , TIMESTEP %f\n", t, solver.dt); + printf("TIME %f , TIMESTEP %f\n", t, d.dt); #else printProgress(t); #endif } - E = getTimeStamp(); + fclose(fp); + timeStop = getTimeStamp(); stopProgress(); - printf("Solution took %.2fs\n", E - S); - writeResult(&solver); + printf("Solution took %.2fs\n", timeStop - timeStart); + writeResult(&d); return EXIT_SUCCESS; } diff --git a/BasicSolver/2D-seq/src/parameter.c b/BasicSolver/2D-seq/src/parameter.c index d691627..b7c32ca 100644 --- a/BasicSolver/2D-seq/src/parameter.c +++ b/BasicSolver/2D-seq/src/parameter.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -9,21 +9,23 @@ #include #include "parameter.h" -#include "util.h" #define MAXLINE 4096 void initParameter(Parameter* param) { param->xlength = 1.0; param->ylength = 1.0; - param->imax = 100; - param->jmax = 100; + param->imax = 128; + param->jmax = 128; param->itermax = 1000; param->eps = 0.0001; param->omg = 1.7; param->re = 100.0; param->gamma = 0.9; param->tau = 0.5; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) @@ -61,6 +63,7 @@ void readParameter(Parameter* param, const char* filename) PARSE_INT(imax); PARSE_INT(jmax); PARSE_INT(itermax); + PARSE_INT(levels); PARSE_REAL(eps); PARSE_REAL(omg); PARSE_REAL(re); @@ -78,6 +81,8 @@ void readParameter(Parameter* param, const char* filename) PARSE_REAL(u_init); PARSE_REAL(v_init); PARSE_REAL(p_init); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); } } @@ -108,4 +113,5 @@ void printParameter(Parameter* param) printf("\tepsilon (stopping tolerance) : %f\n", param->eps); printf("\tgamma (stopping tolerance) : %f\n", param->gamma); printf("\tomega (SOR relaxation): %f\n", param->omg); + printf("\tMultiGrid levels : %d\n", param->levels); } diff --git a/BasicSolver/2D-seq/src/parameter.h b/BasicSolver/2D-seq/src/parameter.h index f4c331a..3714f96 100644 --- a/BasicSolver/2D-seq/src/parameter.h +++ b/BasicSolver/2D-seq/src/parameter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -10,14 +10,15 @@ typedef struct { double xlength, ylength; int imax, jmax; - int itermax; - double eps, omg; + int itermax, levels; + double eps, omg, rho; double re, tau, gamma; double te, dt; double gx, gy; char* name; int bcLeft, bcRight, bcBottom, bcTop; double u_init, v_init, p_init; + int presmooth, postsmooth; } Parameter; void initParameter(Parameter*); diff --git a/BasicSolver/2D-seq/src/progress.c b/BasicSolver/2D-seq/src/progress.c index a9b82bd..bcf6e5f 100644 --- a/BasicSolver/2D-seq/src/progress.c +++ b/BasicSolver/2D-seq/src/progress.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -49,3 +49,22 @@ void stopProgress() printf("\n"); fflush(stdout); } + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} diff --git a/BasicSolver/2D-seq/src/progress.h b/BasicSolver/2D-seq/src/progress.h index 9ef2d96..a79ecfe 100644 --- a/BasicSolver/2D-seq/src/progress.h +++ b/BasicSolver/2D-seq/src/progress.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -9,6 +9,8 @@ extern void initProgress(double); extern void printProgress(double); -extern void stopProgress(); +extern void stopProgress(void); +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/BasicSolver/2D-seq/src/solver-mg.c b/BasicSolver/2D-seq/src/solver-mg.c new file mode 100644 index 0000000..acd3d3b --- /dev/null +++ b/BasicSolver/2D-seq/src/solver-mg.c @@ -0,0 +1,221 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +#define S(i, j) s[(j) * (imax + 2) + (i)] +#define E(i, j) e[(j) * (imax + 2) + (i)] +#define R(i, j) r[(j) * (imax + 2) + (i)] +#define OLD(i, j) old[(j) * (imax + 2) + (i)] + +static void restrictMG(Solver* s, int level, int imax, int jmax) +{ + double* r = s->r[level + 1]; + double* old = s->r[level]; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + R(i, j) = (OLD(2 * i - 1, 2 * j - 1) + OLD(2 * i, 2 * j - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1) + OLD(2 * i - 1, 2 * j) * 2 + + OLD(2 * i, 2 * j) * 4 + OLD(2 * i + 1, 2 * j) * 2 + + OLD(2 * i - 1, 2 * j + 1) + OLD(2 * i, 2 * j + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1)) / + 16.0; + } + } +} + +static void prolongate(Solver* s, int level, int imax, int jmax) +{ + double* old = s->r[level + 1]; + double* e = s->r[level]; + + for (int j = 2; j < jmax + 1; j += 2) { + for (int i = 2; i < imax + 1; i += 2) { + E(i, j) = OLD(i / 2, j / 2); + } + } +} + +static void correct(Solver* s, double* p, int level, int imax, int jmax) +{ + double* e = s->e[level]; + + for (int j = 1; j < jmax + 1; ++j) { + for (int i = 1; i < imax + 1; ++i) { + P(i, j) += E(i, j); + } + } +} + +static void setBoundaryCondition(double* p, int imax, int jmax) +{ + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmax + 1) = P(i, jmax); + } + + for (int j = 1; j < jmax + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } +} + +static double smooth(Solver* s, double* p, double* rhs, int level, int imax, int jmax) +{ + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + P(i, j) -= factor * (RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2)); + + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, int imax, int jmax) +{ + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + R(i, j) = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + res += (R(i, j) * R(i, j)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + + res = res / (double)(imax * jmax); + return res; +} + +static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax, int jmax) +{ + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, imax, jmax); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, imax, jmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); + } + + res = calculateResidual(s, p, rhs, level, imax, jmax); + + // restrict + restrictMG(s, level, imax, jmax); + + // MGSolver on residual and error. + multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, imax / 2, jmax / 2); + + // prolongate + prolongate(s, level, imax, jmax); + + // correct p on finer level using residual + correct(s, p, level, imax, jmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, imax, jmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int levels = s->levels; + printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * sizeof(double); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size); + s->e[j] = allocate(64, size); + + for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax); + +#ifdef VERBOSE + printf("Residuum: %.6f\n", res); +#endif + +return res; +} diff --git a/BasicSolver/2D-seq/src/solver-rb.c b/BasicSolver/2D-seq/src/solver-rb.c new file mode 100644 index 0000000..a9facf4 --- /dev/null +++ b/BasicSolver/2D-seq/src/solver-rb.c @@ -0,0 +1,78 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->itermax = p->itermax; + s->eps = p->eps; + s->omega = p->omg; +} + +double solve(Solver* solver, double* p, double* rhs) +{ + int imax = solver->grid->imax; + int jmax = solver->grid->jmax; + double eps = solver->eps; + int itermax = solver->itermax; + double dx2 = solver->grid->dx * solver->grid->dx; + double dy2 = solver->grid->dy * solver->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double epssq = eps * eps; + int it = 0; + double res = 1.0; + int pass, jsw, isw; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmax + 1) = P(i, jmax); + } + + for (int j = 1; j < jmax + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } + + res = res / (double)(imax * jmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif + + return res; +} diff --git a/BasicSolver/2D-seq/src/solver-sor.c b/BasicSolver/2D-seq/src/solver-sor.c new file mode 100644 index 0000000..21af4eb --- /dev/null +++ b/BasicSolver/2D-seq/src/solver-sor.c @@ -0,0 +1,70 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->itermax = p->itermax; + s->eps = p->eps; + s->omega = p->omg; +} + +double solve(Solver* solver, double* p, double* rhs) +{ + int imax = solver->grid->imax; + int jmax = solver->grid->jmax; + double eps = solver->eps; + int itermax = solver->itermax; + double dx2 = solver->grid->dx * solver->grid->dx; + double dy2 = solver->grid->dy * solver->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double epssq = eps * eps; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + } + + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmax + 1) = P(i, jmax); + } + + for (int j = 1; j < jmax + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } + + res = res / (double)(imax * jmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif + + return res; +} diff --git a/BasicSolver/2D-seq/src/solver.c b/BasicSolver/2D-seq/src/solver.c deleted file mode 100644 index 7720be1..0000000 --- a/BasicSolver/2D-seq/src/solver.c +++ /dev/null @@ -1,564 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include - -#include "allocate.h" -#include "parameter.h" -#include "solver.h" -#include "util.h" - -#define P(i, j) p[(j) * (imax + 2) + (i)] -#define F(i, j) f[(j) * (imax + 2) + (i)] -#define G(i, j) g[(j) * (imax + 2) + (i)] -#define U(i, j) u[(j) * (imax + 2) + (i)] -#define V(i, j) v[(j) * (imax + 2) + (i)] -#define RHS(i, j) rhs[(j) * (imax + 2) + (i)] - -static void print(Solver* solver, double* grid) -{ - int imax = solver->imax; - - for (int j = 0; j < solver->jmax + 2; j++) { - printf("%02d: ", j); - for (int i = 0; i < solver->imax + 2; i++) { - printf("%12.8f ", grid[j * (imax + 2) + i]); - } - printf("\n"); - } - fflush(stdout); -} - -static void printConfig(Solver* solver) -{ - printf("Parameters for #%s#\n", solver->problem); - printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", - solver->bcLeft, - solver->bcRight, - solver->bcBottom, - solver->bcTop); - printf("\tReynolds number: %.2f\n", solver->re); - printf("\tGx Gy: %.2f %.2f\n", solver->gx, solver->gy); - printf("Geometry data:\n"); - printf("\tDomain box size (x, y): %.2f, %.2f\n", solver->xlength, solver->ylength); - printf("\tCells (x, y): %d, %d\n", solver->imax, solver->jmax); - printf("Timestep parameters:\n"); - printf("\tDefault stepsize: %.2f, Final time %.2f\n", solver->dt, solver->te); - printf("\tdt bound: %.6f\n", solver->dtBound); - printf("\tTau factor: %.2f\n", solver->tau); - printf("Iterative solver parameters:\n"); - printf("\tMax iterations: %d\n", solver->itermax); - printf("\tepsilon (stopping tolerance) : %f\n", solver->eps); - printf("\tgamma factor: %f\n", solver->gamma); - printf("\tomega (SOR relaxation): %f\n", solver->omega); -} - -void initSolver(Solver* solver, Parameter* params) -{ - solver->problem = params->name; - solver->bcLeft = params->bcLeft; - solver->bcRight = params->bcRight; - solver->bcBottom = params->bcBottom; - solver->bcTop = params->bcTop; - solver->imax = params->imax; - solver->jmax = params->jmax; - solver->xlength = params->xlength; - solver->ylength = params->ylength; - solver->dx = params->xlength / params->imax; - solver->dy = params->ylength / params->jmax; - solver->eps = params->eps; - solver->omega = params->omg; - solver->itermax = params->itermax; - solver->re = params->re; - solver->gx = params->gx; - solver->gy = params->gy; - solver->dt = params->dt; - solver->te = params->te; - solver->tau = params->tau; - solver->gamma = params->gamma; - - int imax = solver->imax; - int jmax = solver->jmax; - size_t size = (imax + 2) * (jmax + 2) * sizeof(double); - solver->u = allocate(64, size); - solver->v = allocate(64, size); - solver->p = allocate(64, size); - solver->rhs = allocate(64, size); - solver->f = allocate(64, size); - solver->g = allocate(64, size); - - for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { - solver->u[i] = params->u_init; - solver->v[i] = params->v_init; - solver->p[i] = params->p_init; - solver->rhs[i] = 0.0; - solver->f[i] = 0.0; - solver->g[i] = 0.0; - } - - double dx = solver->dx; - double dy = solver->dy; - double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy); - solver->dtBound = 0.5 * solver->re * 1.0 / invSqrSum; -#ifdef VERBOSE - printConfig(solver); -#endif -} - -void computeRHS(Solver* solver) -{ - int imax = solver->imax; - int jmax = solver->jmax; - double idx = 1.0 / solver->dx; - double idy = 1.0 / solver->dy; - double idt = 1.0 / solver->dt; - double* rhs = solver->rhs; - double* f = solver->f; - double* g = solver->g; - - for (int j = 1; j < jmax + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - RHS(i, j) = idt * - ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy); - } - } -} - -void solve(Solver* solver) -{ - int imax = solver->imax; - int jmax = solver->jmax; - double eps = solver->eps; - int itermax = solver->itermax; - double dx2 = solver->dx * solver->dx; - double dy2 = solver->dy * solver->dy; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); - double* p = solver->p; - double* rhs = solver->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - - for (int j = 1; j < jmax + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - - double r = RHS(i, j) - - ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + - (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - - P(i, j) -= (factor * r); - res += (r * r); - } - } - - for (int i = 1; i < imax + 1; i++) { - P(i, 0) = P(i, 1); - P(i, jmax + 1) = P(i, jmax); - } - - for (int j = 1; j < jmax + 1; j++) { - P(0, j) = P(1, j); - P(imax + 1, j) = P(imax, j); - } - - res = res / (double)(imax * jmax); -#ifdef DEBUG - printf("%d Residuum: %e\n", it, res); -#endif - it++; - } - -#ifdef VERBOSE - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); -#endif -} - -void solveRB(Solver* solver) -{ - int imax = solver->imax; - int jmax = solver->jmax; - double eps = solver->eps; - int itermax = solver->itermax; - double dx2 = solver->dx * solver->dx; - double dy2 = solver->dy * solver->dy; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double factor = solver->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); - double* p = solver->p; - double* rhs = solver->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - int pass, jsw, isw; - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - jsw = 1; - - for (pass = 0; pass < 2; pass++) { - isw = jsw; - - for (int j = 1; j < jmax + 1; j++) { - for (int i = isw; i < imax + 1; i += 2) { - - double r = RHS(i, j) - - ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + - (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); - - P(i, j) -= (factor * r); - res += (r * r); - } - isw = 3 - isw; - } - jsw = 3 - jsw; - } - - for (int i = 1; i < imax + 1; i++) { - P(i, 0) = P(i, 1); - P(i, jmax + 1) = P(i, jmax); - } - - for (int j = 1; j < jmax + 1; j++) { - P(0, j) = P(1, j); - P(imax + 1, j) = P(imax, j); - } - - res = res / (double)(imax * jmax); -#ifdef DEBUG - printf("%d Residuum: %e\n", it, res); -#endif - it++; - } - -#ifdef VERBOSE - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); -#endif -} - -static double maxElement(Solver* solver, double* m) -{ - int size = (solver->imax + 2) * (solver->jmax + 2); - double maxval = DBL_MIN; - - for (int i = 0; i < size; i++) { - maxval = MAX(maxval, fabs(m[i])); - } - - return maxval; -} - -void normalizePressure(Solver* solver) -{ - int size = (solver->imax + 2) * (solver->jmax + 2); - double* p = solver->p; - double avgP = 0.0; - - for (int i = 0; i < size; i++) { - avgP += p[i]; - } - avgP /= size; - - for (int i = 0; i < size; i++) { - p[i] = p[i] - avgP; - } -} - -void computeTimestep(Solver* solver) -{ - double dt = solver->dtBound; - double dx = solver->dx; - double dy = solver->dy; - double umax = maxElement(solver, solver->u); - double vmax = maxElement(solver, solver->v); - - if (umax > 0) { - dt = (dt > dx / umax) ? dx / umax : dt; - } - if (vmax > 0) { - dt = (dt > dy / vmax) ? dy / vmax : dt; - } - - solver->dt = dt * solver->tau; -} - -void setBoundaryConditions(Solver* solver) -{ - int imax = solver->imax; - int jmax = solver->jmax; - double* u = solver->u; - double* v = solver->v; - - // Left boundary - switch (solver->bcLeft) { - case NOSLIP: - for (int j = 1; j < jmax + 1; j++) { - U(0, j) = 0.0; - V(0, j) = -V(1, j); - } - break; - case SLIP: - for (int j = 1; j < jmax + 1; j++) { - U(0, j) = 0.0; - V(0, j) = V(1, j); - } - break; - case OUTFLOW: - for (int j = 1; j < jmax + 1; j++) { - U(0, j) = U(1, j); - V(0, j) = V(1, j); - } - break; - case PERIODIC: - break; - } - - // Right boundary - switch (solver->bcRight) { - case NOSLIP: - for (int j = 1; j < jmax + 1; j++) { - U(imax, j) = 0.0; - V(imax + 1, j) = -V(imax, j); - } - break; - case SLIP: - for (int j = 1; j < jmax + 1; j++) { - U(imax, j) = 0.0; - V(imax + 1, j) = V(imax, j); - } - break; - case OUTFLOW: - for (int j = 1; j < jmax + 1; j++) { - U(imax, j) = U(imax - 1, j); - V(imax + 1, j) = V(imax, j); - } - break; - case PERIODIC: - break; - } - - // Bottom boundary - switch (solver->bcBottom) { - case NOSLIP: - for (int i = 1; i < imax + 1; i++) { - V(i, 0) = 0.0; - U(i, 0) = -U(i, 1); - } - break; - case SLIP: - for (int i = 1; i < imax + 1; i++) { - V(i, 0) = 0.0; - U(i, 0) = U(i, 1); - } - break; - case OUTFLOW: - for (int i = 1; i < imax + 1; i++) { - U(i, 0) = U(i, 1); - V(i, 0) = V(i, 1); - } - break; - case PERIODIC: - break; - } - - // Top boundary - switch (solver->bcTop) { - case NOSLIP: - for (int i = 1; i < imax + 1; i++) { - V(i, jmax) = 0.0; - U(i, jmax + 1) = -U(i, jmax); - } - break; - case SLIP: - for (int i = 1; i < imax + 1; i++) { - V(i, jmax) = 0.0; - U(i, jmax + 1) = U(i, jmax); - } - break; - case OUTFLOW: - for (int i = 1; i < imax + 1; i++) { - U(i, jmax + 1) = U(i, jmax); - V(i, jmax) = V(i, jmax - 1); - } - break; - case PERIODIC: - break; - } -} - -void setSpecialBoundaryCondition(Solver* solver) -{ - int imax = solver->imax; - int jmax = solver->jmax; - double mDy = solver->dy; - double* u = solver->u; - - if (strcmp(solver->problem, "dcavity") == 0) { - for (int i = 1; i < imax; i++) { - U(i, jmax + 1) = 2.0 - U(i, jmax); - } - } else if (strcmp(solver->problem, "canal") == 0) { - double ylength = solver->ylength; - double y; - - for (int j = 1; j < jmax + 1; j++) { - y = mDy * (j - 0.5); - U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); - } - } -} - -void computeFG(Solver* solver) -{ - double* u = solver->u; - double* v = solver->v; - double* f = solver->f; - double* g = solver->g; - int imax = solver->imax; - int jmax = solver->jmax; - double gx = solver->gx; - double gy = solver->gy; - double gamma = solver->gamma; - double dt = solver->dt; - double inverseRe = 1.0 / solver->re; - double inverseDx = 1.0 / solver->dx; - double inverseDy = 1.0 / solver->dy; - double du2dx, dv2dy, duvdx, duvdy; - double du2dx2, du2dy2, dv2dx2, dv2dy2; - - for (int j = 1; j < jmax + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - du2dx = inverseDx * 0.25 * - ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - - (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + - fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); - - duvdy = inverseDy * 0.25 * - ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - - (V(i, j - 1) + V(i + 1, j - 1)) * (U(i, j) + U(i, j - 1))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + - fabs(V(i, j - 1) + V(i + 1, j - 1)) * - (U(i, j) - U(i, j - 1))); - - du2dx2 = inverseDx * inverseDx * (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); - du2dy2 = inverseDy * inverseDy * (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); - F(i, j) = U(i, j) + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); - - duvdx = inverseDx * 0.25 * - ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - - (U(i - 1, j) + U(i - 1, j + 1)) * (V(i, j) + V(i - 1, j))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + - fabs(U(i - 1, j) + U(i - 1, j + 1)) * - (V(i, j) - V(i - 1, j))); - - dv2dy = inverseDy * 0.25 * - ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - - (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + - fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); - - dv2dx2 = inverseDx * inverseDx * (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); - dv2dy2 = inverseDy * inverseDy * (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); - G(i, j) = V(i, j) + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); - } - } - - /* ---------------------- boundary of F --------------------------- */ - for (int j = 1; j < jmax + 1; j++) { - F(0, j) = U(0, j); - F(imax, j) = U(imax, j); - } - - /* ---------------------- boundary of G --------------------------- */ - for (int i = 1; i < imax + 1; i++) { - G(i, 0) = V(i, 0); - G(i, jmax) = V(i, jmax); - } -} - -void adaptUV(Solver* solver) -{ - int imax = solver->imax; - int jmax = solver->jmax; - double* p = solver->p; - double* u = solver->u; - double* v = solver->v; - double* f = solver->f; - double* g = solver->g; - double factorX = solver->dt / solver->dx; - double factorY = solver->dt / solver->dy; - - for (int j = 1; j < jmax + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; - V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; - } - } -} - -void writeResult(Solver* solver) -{ - int imax = solver->imax; - int jmax = solver->jmax; - double dx = solver->dx; - double dy = solver->dy; - double* p = solver->p; - double* u = solver->u; - double* v = solver->v; - double x = 0.0, y = 0.0; - - FILE* fp; - fp = fopen("pressure.dat", "w"); - - if (fp == NULL) { - printf("Error!\n"); - exit(EXIT_FAILURE); - } - - for (int j = 1; j < jmax + 1; j++) { - y = (double)(j - 0.5) * dy; - for (int i = 1; i < imax + 1; i++) { - x = (double)(i - 0.5) * dx; - fprintf(fp, "%.2f %.2f %f\n", x, y, P(i, j)); - } - fprintf(fp, "\n"); - } - - fclose(fp); - - fp = fopen("velocity.dat", "w"); - - if (fp == NULL) { - printf("Error!\n"); - exit(EXIT_FAILURE); - } - - for (int j = 1; j < jmax + 1; j++) { - y = dy * (j - 0.5); - for (int i = 1; i < imax + 1; i++) { - x = dx * (i - 0.5); - double vel_u = (U(i, j) + U(i - 1, j)) / 2.0; - double vel_v = (V(i, j) + V(i, j - 1)) / 2.0; - double len = sqrt((vel_u * vel_u) + (vel_v * vel_v)); - fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, vel_u, vel_v, len); - } - } - - fclose(fp); -} diff --git a/BasicSolver/2D-seq/src/solver.h b/BasicSolver/2D-seq/src/solver.h index 56f0297..0343f6e 100644 --- a/BasicSolver/2D-seq/src/solver.h +++ b/BasicSolver/2D-seq/src/solver.h @@ -1,46 +1,27 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ #ifndef __SOLVER_H_ #define __SOLVER_H_ +#include "discretization.h" +#include "grid.h" #include "parameter.h" -enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; - typedef struct { /* geometry and grid information */ - double dx, dy; - int imax, jmax; - double xlength, ylength; - /* arrays */ - double *p, *rhs; - double *f, *g; - double *u, *v; + Grid* grid; /* parameters */ - double eps, omega; - double re, tau, gamma; - double gx, gy; - /* time stepping */ + double eps, omega, rho; int itermax; - double dt, te; - double dtBound; - char* problem; - int bcLeft, bcRight, bcBottom, bcTop; + int levels; + double **r, **e; + int presmooth, postsmooth; } Solver; -extern void initSolver(Solver*, Parameter*); -extern void computeRHS(Solver*); -extern void solve(Solver*); -extern void solveRB(Solver*); -extern void solveRBA(Solver*); -extern void normalizePressure(Solver*); -extern void computeTimestep(Solver*); -extern void setBoundaryConditions(Solver*); -extern void setSpecialBoundaryCondition(Solver*); -extern void computeFG(Solver*); -extern void adaptUV(Solver*); -extern void writeResult(Solver*); +extern void initSolver(Solver*, Discretization*, Parameter*); +extern double solve(Solver*, double*, double*); + #endif diff --git a/BasicSolver/2D-seq/src/timing.c b/BasicSolver/2D-seq/src/timing.c index c4025a4..78b01c4 100644 --- a/BasicSolver/2D-seq/src/timing.c +++ b/BasicSolver/2D-seq/src/timing.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,18 +7,16 @@ #include #include -double getTimeStamp() +double getTimeStamp(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } -double getTimeResolution() +double getTimeResolution(void) { struct timespec ts; clock_getres(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } - -double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-seq/src/timing.h b/BasicSolver/2D-seq/src/timing.h index db95329..ed05a8c 100644 --- a/BasicSolver/2D-seq/src/timing.h +++ b/BasicSolver/2D-seq/src/timing.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,8 +7,7 @@ #ifndef __TIMING_H_ #define __TIMING_H_ -extern double getTimeStamp(); -extern double getTimeResolution(); -extern double getTimeStamp_(); +extern double getTimeStamp(void); +extern double getTimeResolution(void); #endif // __TIMING_H_ diff --git a/BasicSolver/2D-seq/src/util.h b/BasicSolver/2D-seq/src/util.h index 61a1dff..e4a93c2 100644 --- a/BasicSolver/2D-seq/src/util.h +++ b/BasicSolver/2D-seq/src/util.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,8 +7,7 @@ #ifndef __UTIL_H_ #define __UTIL_H_ #define HLINE \ - "------------------------------------------------------------------------" \ - "----\n" + "----------------------------------------------------------------------------\n" #ifndef MIN #define MIN(x, y) ((x) < (y) ? (x) : (y)) @@ -20,4 +19,11 @@ #define ABS(a) ((a) >= 0 ? (a) : -(a)) #endif +#define P(i, j) p[(j) * (imax + 2) + (i)] +#define F(i, j) f[(j) * (imax + 2) + (i)] +#define G(i, j) g[(j) * (imax + 2) + (i)] +#define U(i, j) u[(j) * (imax + 2) + (i)] +#define V(i, j) v[(j) * (imax + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imax + 2) + (i)] + #endif // __UTIL_H_ diff --git a/BasicSolver/3D-mpi-io/Makefile b/BasicSolver/3D-mpi-io/Makefile deleted file mode 100644 index 57f99f4..0000000 --- a/BasicSolver/3D-mpi-io/Makefile +++ /dev/null @@ -1,71 +0,0 @@ -#======================================================================================= -# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. -# All rights reserved. -# Use of this source code is governed by a MIT-style -# license that can be found in the LICENSE file. -#======================================================================================= - -#CONFIGURE BUILD SYSTEM -TARGET = exe-$(TAG) -BUILD_DIR = ./$(TAG) -SRC_DIR = ./src -MAKE_DIR = ./ -Q ?= @ - -#DO NOT EDIT BELOW -include $(MAKE_DIR)/config.mk -include $(MAKE_DIR)/include_$(TAG).mk -INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) - -VPATH = $(SRC_DIR) -SRC = $(wildcard $(SRC_DIR)/*.c) -ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) -OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) -SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) -CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) - -${TARGET}: $(BUILD_DIR) $(OBJ) - $(info ===> LINKING $(TARGET)) - $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) - -$(BUILD_DIR)/%.o: %.c $(MAKE_DIR)/include_$(TAG).mk $(MAKE_DIR)/config.mk - $(info ===> COMPILE $@) - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ - $(Q)$(GCC) $(CPPFLAGS) -MT $(@:.d=.o) -MM $< > $(BUILD_DIR)/$*.d - -$(BUILD_DIR)/%.s: %.c - $(info ===> GENERATE ASM $@) - $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ - -.PHONY: clean distclean tags info asm format - -clean: - $(info ===> CLEAN) - @rm -rf $(BUILD_DIR) - @rm -f tags - -distclean: clean - $(info ===> DIST CLEAN) - @rm -f $(TARGET) - -info: - $(info $(CFLAGS)) - $(Q)$(CC) $(VERSION) - -asm: $(BUILD_DIR) $(ASM) - -tags: - $(info ===> GENERATE TAGS) - $(Q)ctags -R - -format: - @for src in $(SOURCES) ; do \ - echo "Formatting $$src" ; \ - clang-format -i $$src ; \ - done - @echo "Done" - -$(BUILD_DIR): - @mkdir $(BUILD_DIR) - --include $(OBJ:.o=.d) diff --git a/BasicSolver/3D-mpi-io/include_GCC.mk b/BasicSolver/3D-mpi-io/include_GCC.mk deleted file mode 100644 index 427e798..0000000 --- a/BasicSolver/3D-mpi-io/include_GCC.mk +++ /dev/null @@ -1,14 +0,0 @@ -CC = gcc -GCC = gcc -LINKER = $(CC) - -ifeq ($(ENABLE_OPENMP),true) -OPENMP = -fopenmp -endif - -VERSION = --version -CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) -LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE -INCLUDES = -LIBS = diff --git a/BasicSolver/3D-mpi-io/src/allocate.c b/BasicSolver/3D-mpi-io/src/allocate.c deleted file mode 100644 index 81e1e9d..0000000 --- a/BasicSolver/3D-mpi-io/src/allocate.c +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#include -#include -#include - -void* allocate(int alignment, size_t bytesize) -{ - int errorCode; - void* ptr; - - errorCode = posix_memalign(&ptr, alignment, bytesize); - - if (errorCode) { - if (errorCode == EINVAL) { - fprintf(stderr, "Error: Alignment parameter is not a power of two\n"); - exit(EXIT_FAILURE); - } - if (errorCode == ENOMEM) { - fprintf(stderr, "Error: Insufficient memory to fulfill the request\n"); - exit(EXIT_FAILURE); - } - } - - if (ptr == NULL) { - fprintf(stderr, "Error: posix_memalign failed!\n"); - exit(EXIT_FAILURE); - } - - return ptr; -} diff --git a/BasicSolver/3D-mpi-io/src/allocate.h b/BasicSolver/3D-mpi-io/src/allocate.h deleted file mode 100644 index 54cfe06..0000000 --- a/BasicSolver/3D-mpi-io/src/allocate.h +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifndef __ALLOCATE_H_ -#define __ALLOCATE_H_ -#include - -extern void* allocate(int alignment, size_t bytesize); - -#endif diff --git a/BasicSolver/3D-mpi-io/src/comm.c b/BasicSolver/3D-mpi-io/src/comm.c deleted file mode 100644 index add6b20..0000000 --- a/BasicSolver/3D-mpi-io/src/comm.c +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include - -#include "allocate.h" -#include "comm.h" - -// subroutines local to this module -static int sizeOfRank(int rank, int size, int N) -{ - return N / size + ((N % size > rank) ? 1 : 0); -} - -static void setupCommunication(Comm* c, Direction direction, int layer) -{ - int imaxLocal = c->imaxLocal; - int jmaxLocal = c->jmaxLocal; - int kmaxLocal = c->kmaxLocal; - - size_t dblsize = sizeof(double); - int sizes[NDIMS]; - int subSizes[NDIMS]; - int starts[NDIMS]; - int offset = 0; - - sizes[IDIM] = imaxLocal + 2; - sizes[JDIM] = jmaxLocal + 2; - sizes[KDIM] = kmaxLocal + 2; - - if (layer == HALO) { - offset = 1; - } - - switch (direction) { - case LEFT: - subSizes[IDIM] = 1; - subSizes[JDIM] = jmaxLocal; - subSizes[KDIM] = kmaxLocal; - starts[IDIM] = 1 - offset; - starts[JDIM] = 1; - starts[KDIM] = 1; - break; - case RIGHT: - subSizes[IDIM] = 1; - subSizes[JDIM] = jmaxLocal; - subSizes[KDIM] = kmaxLocal; - starts[IDIM] = imaxLocal + offset; - starts[JDIM] = 1; - starts[KDIM] = 1; - break; - case BOTTOM: - subSizes[IDIM] = imaxLocal; - subSizes[JDIM] = 1; - subSizes[KDIM] = kmaxLocal; - starts[IDIM] = 1; - starts[JDIM] = 1 - offset; - starts[KDIM] = 1; - break; - case TOP: - subSizes[IDIM] = imaxLocal; - subSizes[JDIM] = 1; - subSizes[KDIM] = kmaxLocal; - starts[IDIM] = 1; - starts[JDIM] = jmaxLocal + offset; - starts[KDIM] = 1; - break; - case FRONT: - subSizes[IDIM] = imaxLocal; - subSizes[JDIM] = jmaxLocal; - subSizes[KDIM] = 1; - starts[IDIM] = 1; - starts[JDIM] = 1; - starts[KDIM] = 1 - offset; - break; - case BACK: - subSizes[IDIM] = imaxLocal; - subSizes[JDIM] = jmaxLocal; - subSizes[KDIM] = 1; - starts[IDIM] = 1; - starts[JDIM] = 1; - starts[KDIM] = kmaxLocal + offset; - break; - case NDIRS: - printf("ERROR!\n"); - break; - } - - if (layer == HALO) { - MPI_Type_create_subarray(NDIMS, - sizes, - subSizes, - starts, - MPI_ORDER_C, - MPI_DOUBLE, - &c->rbufferTypes[direction]); - MPI_Type_commit(&c->rbufferTypes[direction]); - } else if (layer == BULK) { - MPI_Type_create_subarray(NDIMS, - sizes, - subSizes, - starts, - MPI_ORDER_C, - MPI_DOUBLE, - &c->sbufferTypes[direction]); - MPI_Type_commit(&c->sbufferTypes[direction]); - } -} - -static int sum(int* sizes, int position) -{ - int sum = 0; - - for (int i = 0; i < position; i++) { - sum += sizes[i]; - } - - return sum; -} - -// exported subroutines -void commReduction(double* v, int op) -{ - if (op == MAX) { - MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - } else if (op == SUM) { - MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - } -} - -int commIsBoundary(Comm* c, Direction direction) -{ - switch (direction) { - case LEFT: - return c->coords[ICORD] == 0; - break; - case RIGHT: - return c->coords[ICORD] == (c->dims[ICORD] - 1); - break; - case BOTTOM: - return c->coords[JCORD] == 0; - break; - case TOP: - return c->coords[JCORD] == (c->dims[JCORD] - 1); - break; - case FRONT: - return c->coords[KCORD] == 0; - break; - case BACK: - return c->coords[KCORD] == (c->dims[KCORD] - 1); - break; - case NDIRS: - printf("ERROR!\n"); - break; - } - - return 0; -} - -void commExchange(Comm* c, double* grid) -{ - int counts[6] = { 1, 1, 1, 1, 1, 1 }; - MPI_Aint displs[6] = { 0, 0, 0, 0, 0, 0 }; - - MPI_Neighbor_alltoallw(grid, - counts, - displs, - c->sbufferTypes, - grid, - counts, - displs, - c->rbufferTypes, - c->comm); -} - -void commShift(Comm* c, double* f, double* g, double* h) -{ - MPI_Request requests[6] = { MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL, - MPI_REQUEST_NULL }; - - /* shift G */ - /* receive ghost cells from bottom neighbor */ - MPI_Irecv(g, - 1, - c->rbufferTypes[BOTTOM], - c->neighbours[BOTTOM], - 0, - c->comm, - &requests[0]); - - /* send ghost cells to top neighbor */ - MPI_Isend(g, 1, c->sbufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]); - - /* shift F */ - /* receive ghost cells from left neighbor */ - MPI_Irecv(f, 1, c->rbufferTypes[LEFT], c->neighbours[LEFT], 1, c->comm, &requests[2]); - - /* send ghost cells to right neighbor */ - MPI_Isend(f, - 1, - c->sbufferTypes[RIGHT], - c->neighbours[RIGHT], - 1, - c->comm, - &requests[3]); - - /* shift H */ - /* receive ghost cells from front neighbor */ - MPI_Irecv(h, - 1, - c->rbufferTypes[FRONT], - c->neighbours[FRONT], - 2, - c->comm, - &requests[4]); - - /* send ghost cells to back neighbor */ - MPI_Isend(h, 1, c->sbufferTypes[BACK], c->neighbours[BACK], 2, c->comm, &requests[5]); - - MPI_Waitall(6, requests, MPI_STATUSES_IGNORE); -} - -void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax) -{ - int sum = 0; - - for (int i = 0; i < c->coords[ICORD]; i++) { - sum += sizeOfRank(i, c->dims[ICORD], imax); - } - offsets[IDIM] = sum; - sum = 0; - - for (int i = 0; i < c->coords[JCORD]; i++) { - sum += sizeOfRank(i, c->dims[JCORD], jmax); - } - offsets[JDIM] = sum; - sum = 0; - - for (int i = 0; i < c->coords[KCORD]; i++) { - sum += sizeOfRank(i, c->dims[KCORD], kmax); - } - offsets[KDIM] = sum; -} - -void commPrintConfig(Comm* c) -{ - fflush(stdout); - MPI_Barrier(MPI_COMM_WORLD); - if (commIsMaster(c)) { - printf("Communication setup:\n"); - } - - for (int i = 0; i < c->size; i++) { - if (i == c->rank) { - printf("\tRank %d of %d\n", c->rank, c->size); - printf("\tNeighbours (front, back, bottom, top, left, right): %d, %d, %d, " - "%d, %d, %d\n", - c->neighbours[FRONT], - c->neighbours[BACK], - c->neighbours[BOTTOM], - c->neighbours[TOP], - c->neighbours[LEFT], - c->neighbours[RIGHT]); - printf("\tCoordinates (k,j,i) %d %d %d\n", - c->coords[KCORD], - c->coords[JCORD], - c->coords[ICORD]); - printf("\tLocal domain size (k,j,i) %dx%dx%d\n", - c->kmaxLocal, - c->jmaxLocal, - c->imaxLocal); - fflush(stdout); - } - } - MPI_Barrier(MPI_COMM_WORLD); -} - -void commInit(Comm* c, int kmax, int jmax, int imax) -{ - /* setup communication */ - MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank)); - MPI_Comm_size(MPI_COMM_WORLD, &(c->size)); - int dims[NDIMS] = { 0, 0, 0 }; - int periods[NDIMS] = { 0, 0, 0 }; - MPI_Dims_create(c->size, NDIMS, dims); - MPI_Cart_create(MPI_COMM_WORLD, NCORDS, dims, periods, 0, &c->comm); - MPI_Cart_shift(c->comm, ICORD, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]); - MPI_Cart_shift(c->comm, JCORD, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]); - MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]); - MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords); - - c->imaxLocal = sizeOfRank(c->rank, dims[ICORD], imax); - c->jmaxLocal = sizeOfRank(c->rank, dims[JCORD], jmax); - c->kmaxLocal = sizeOfRank(c->rank, dims[KCORD], kmax); - - // setup buffer types for communication - setupCommunication(c, LEFT, BULK); - setupCommunication(c, LEFT, HALO); - setupCommunication(c, RIGHT, BULK); - setupCommunication(c, RIGHT, HALO); - setupCommunication(c, BOTTOM, BULK); - setupCommunication(c, BOTTOM, HALO); - setupCommunication(c, TOP, BULK); - setupCommunication(c, TOP, HALO); - setupCommunication(c, FRONT, BULK); - setupCommunication(c, FRONT, HALO); - setupCommunication(c, BACK, BULK); - setupCommunication(c, BACK, HALO); -} - -void commFree(Comm* c) -{ - for (int i = 0; i < NDIRS; i++) { - MPI_Type_free(&c->sbufferTypes[i]); - MPI_Type_free(&c->rbufferTypes[i]); - } -} diff --git a/BasicSolver/3D-mpi-io/src/main.c b/BasicSolver/3D-mpi-io/src/main.c deleted file mode 100644 index 17fc72f..0000000 --- a/BasicSolver/3D-mpi-io/src/main.c +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include -#include - -#include "allocate.h" -#include "parameter.h" -#include "progress.h" -#include "solver.h" -#include "test.h" -#include "timing.h" -#include "vtkWriter.h" - -int main(int argc, char** argv) -{ - int rank; - double timeStart, timeStop; - Parameter params; - Solver solver; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - initParameter(¶ms); - - if (argc != 2) { - printf("Usage: %s \n", argv[0]); - exit(EXIT_SUCCESS); - } - - readParameter(¶ms, argv[1]); - if (commIsMaster(&solver.comm)) { - printParameter(¶ms); - } - initSolver(&solver, ¶ms); -#ifndef VERBOSE - initProgress(solver.te); -#endif - - double tau = solver.tau; - double te = solver.te; - double t = 0.0; - int nt = 0; - - timeStart = getTimeStamp(); - while (t <= te) { - if (tau > 0.0) computeTimestep(&solver); - setBoundaryConditions(&solver); - setSpecialBoundaryCondition(&solver); - computeFG(&solver); - computeRHS(&solver); - // if (nt % 100 == 0) normalizePressure(&solver); - solve(&solver); - adaptUV(&solver); - t += solver.dt; - nt++; - -#ifdef VERBOSE - if (commIsMaster(&solver.comm)) { - printf("TIME %f , TIMESTEP %f\n", t, solver.dt); - } -#else - printProgress(t); -#endif - } - timeStop = getTimeStamp(); - stopProgress(); - if (commIsMaster(&solver.comm)) { - printf("Solution took %.2fs\n", timeStop - timeStart); - } - - // testInit(&solver); - VtkOptions opts = { .grid = solver.grid, .comm = solver.comm }; - vtkOpen(&opts, solver.problem); - vtkScalar(&opts, "pressure", solver.p); - vtkVector(&opts, "velocity", (VtkVector) { solver.u, solver.v, solver.w }); - vtkClose(&opts); - - commFree(&solver.comm); - MPI_Finalize(); - return EXIT_SUCCESS; -} diff --git a/BasicSolver/3D-mpi-io/src/progress.h b/BasicSolver/3D-mpi-io/src/progress.h deleted file mode 100644 index 9ef2d96..0000000 --- a/BasicSolver/3D-mpi-io/src/progress.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifndef __PROGRESS_H_ -#define __PROGRESS_H_ - -extern void initProgress(double); -extern void printProgress(double); -extern void stopProgress(); - -#endif diff --git a/BasicSolver/3D-mpi-io/src/timing.c b/BasicSolver/3D-mpi-io/src/timing.c deleted file mode 100644 index c4025a4..0000000 --- a/BasicSolver/3D-mpi-io/src/timing.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#include -#include - -double getTimeStamp() -{ - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; -} - -double getTimeResolution() -{ - struct timespec ts; - clock_getres(CLOCK_MONOTONIC, &ts); - return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; -} - -double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/3D-mpi-io/src/timing.h b/BasicSolver/3D-mpi-io/src/timing.h deleted file mode 100644 index db95329..0000000 --- a/BasicSolver/3D-mpi-io/src/timing.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifndef __TIMING_H_ -#define __TIMING_H_ - -extern double getTimeStamp(); -extern double getTimeResolution(); -extern double getTimeStamp_(); - -#endif // __TIMING_H_ diff --git a/BasicSolver/3D-mpi-io/src/util.h b/BasicSolver/3D-mpi-io/src/util.h deleted file mode 100644 index 657b009..0000000 --- a/BasicSolver/3D-mpi-io/src/util.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. - * Use of this source code is governed by a MIT-style - * license that can be found in the LICENSE file. - */ -#ifndef __UTIL_H_ -#define __UTIL_H_ -#define HLINE \ - "----------------------------------------------------------------------------\n" - -#ifndef MIN -#define MIN(x, y) ((x) < (y) ? (x) : (y)) -#endif -#ifndef MAX -#define MAX(x, y) ((x) > (y) ? (x) : (y)) -#endif -#ifndef ABS -#define ABS(a) ((a) >= 0 ? (a) : -(a)) -#endif - -#endif // __UTIL_H_ diff --git a/BasicSolver/3D-mpi/Makefile b/BasicSolver/3D-mpi/Makefile index 57f99f4..883bb88 100644 --- a/BasicSolver/3D-mpi/Makefile +++ b/BasicSolver/3D-mpi/Makefile @@ -1,5 +1,5 @@ #======================================================================================= -# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. # All rights reserved. # Use of this source code is governed by a MIT-style # license that can be found in the LICENSE file. @@ -18,13 +18,19 @@ include $(MAKE_DIR)/include_$(TAG).mk INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) VPATH = $(SRC_DIR) -SRC = $(wildcard $(SRC_DIR)/*.c) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/vtkWriter-$(VTK_OUTPUT_FMT).o +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +ifeq ($(VTK_OUTPUT_FMT),mpi) +DEFINES += -D_VTK_WRITER_MPI +endif + CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) -${TARGET}: $(BUILD_DIR) $(OBJ) +${TARGET}: sanity-checks $(BUILD_DIR) $(OBJ) $(info ===> LINKING $(TARGET)) $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) @@ -37,9 +43,19 @@ $(BUILD_DIR)/%.s: %.c $(info ===> GENERATE ASM $@) $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ -.PHONY: clean distclean tags info asm format +.PHONY: clean distclean vis vis_clean tags info asm format -clean: +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.vtk + @rm -f *.png + +clean: vis_clean $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @rm -f tags @@ -65,6 +81,14 @@ format: done @echo "Done" +sanity-checks: +ifeq ($(VTK_OUTPUT_FMT),mpi) +ifeq ($(ENABLE_MPI),false) + $(error VTK_OUTPUT_FMT mpi only supported for ENABLE_MPI true!) +endif +endif + + $(BUILD_DIR): @mkdir $(BUILD_DIR) diff --git a/BasicSolver/3D-mpi/canal.par b/BasicSolver/3D-mpi/canal.par index 3ff4a5f..b8d35c0 100644 --- a/BasicSolver/3D-mpi/canal.par +++ b/BasicSolver/3D-mpi/canal.par @@ -32,21 +32,28 @@ xlength 30.0 # domain size in x-direction ylength 4.0 # domain size in y-direction zlength 4.0 # domain size in z-direction imax 200 # number of interior cells in x-direction -jmax 50 # number of interior cells in y-direction -kmax 50 # number of interior cells in z-direction +jmax 40 # number of interior cells in y-direction +kmax 40 # number of interior cells in z-direction # Time Data: # --------- -te 100.0 # final time +te 60.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- itermax 500 # maximal number of pressure iteration in one time step eps 0.0001 # stopping tolerance for pressure iteration -omg 1.3 # relaxation parameter for SOR iteration +omg 1.7 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma #=============================================================================== diff --git a/BasicSolver/3D-mpi/config.mk b/BasicSolver/3D-mpi/config.mk index af5f1a0..0dbe440 100644 --- a/BasicSolver/3D-mpi/config.mk +++ b/BasicSolver/3D-mpi/config.mk @@ -1,6 +1,12 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC +# Supported: true, false +ENABLE_MPI ?= true ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg +# Supported: seq, mpi +VTK_OUTPUT_FMT ?= seq #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 diff --git a/BasicSolver/3D-mpi/dcavity.par b/BasicSolver/3D-mpi/dcavity.par index c5cb201..77f1991 100644 --- a/BasicSolver/3D-mpi/dcavity.par +++ b/BasicSolver/3D-mpi/dcavity.par @@ -18,7 +18,7 @@ gx 0.0 # Body forces (e.g. gravity) gy 0.0 # gz 0.0 # -re 1000.0 # Reynolds number +re 1000.0 # Reynolds number u_init 0.0 # initial value for velocity in x-direction v_init 0.0 # initial value for velocity in y-direction @@ -42,6 +42,13 @@ te 10.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- diff --git a/BasicSolver/3D-mpi/include_CLANG.mk b/BasicSolver/3D-mpi/include_CLANG.mk index 889fa93..3641cad 100644 --- a/BasicSolver/3D-mpi/include_CLANG.mk +++ b/BasicSolver/3D-mpi/include_CLANG.mk @@ -1,4 +1,10 @@ +ifeq ($(ENABLE_MPI),true) CC = mpicc +DEFINES = -D_MPI +else +CC = cc +endif + GCC = cc LINKER = $(CC) @@ -9,9 +15,7 @@ LIBS = # -lomp endif VERSION = --version -# CFLAGS = -O3 -std=c17 $(OPENMP) CFLAGS = -Ofast -std=c17 -#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG LFLAGS = $(OPENMP) -lm -DEFINES = -D_GNU_SOURCE# -DDEBUG -INCLUDES = +DEFINES += -D_GNU_SOURCE# -DDEBUG +INCLUDES = -I/opt/homebrew/include diff --git a/BasicSolver/3D-mpi/include_GCC.mk b/BasicSolver/3D-mpi/include_GCC.mk index 427e798..cf1e2aa 100644 --- a/BasicSolver/3D-mpi/include_GCC.mk +++ b/BasicSolver/3D-mpi/include_GCC.mk @@ -1,4 +1,10 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpicc +DEFINES = -D_MPI +else CC = gcc +endif + GCC = gcc LINKER = $(CC) @@ -9,6 +15,6 @@ endif VERSION = --version CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE +DEFINES += -D_GNU_SOURCE INCLUDES = LIBS = diff --git a/BasicSolver/3D-mpi/include_ICC.mk b/BasicSolver/3D-mpi/include_ICC.mk index 94b8e20..0575de6 100644 --- a/BasicSolver/3D-mpi/include_ICC.mk +++ b/BasicSolver/3D-mpi/include_ICC.mk @@ -1,5 +1,11 @@ -CC = icc -GCC = gcc +ifeq ($(ENABLE_MPI),true) +CC = mpiicc +DEFINES = -D_MPI +else +CC = icc +endif + +GCC = gcc LINKER = $(CC) ifeq ($(ENABLE_OPENMP),true) @@ -9,6 +15,6 @@ endif VERSION = --version CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE +DEFINES += -D_GNU_SOURCE# -DDEBUG INCLUDES = LIBS = diff --git a/BasicSolver/3D-mpi/residual.plot b/BasicSolver/3D-mpi/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/BasicSolver/3D-mpi/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/3D-mpi/src/allocate.c b/BasicSolver/3D-mpi/src/allocate.c index 81e1e9d..cf2efd6 100644 --- a/BasicSolver/3D-mpi/src/allocate.c +++ b/BasicSolver/3D-mpi/src/allocate.c @@ -1,14 +1,17 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ #include +#include #include #include -void* allocate(int alignment, size_t bytesize) +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) { int errorCode; void* ptr; diff --git a/BasicSolver/3D-mpi/src/allocate.h b/BasicSolver/3D-mpi/src/allocate.h index 54cfe06..77f4ba0 100644 --- a/BasicSolver/3D-mpi/src/allocate.h +++ b/BasicSolver/3D-mpi/src/allocate.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -8,6 +8,6 @@ #define __ALLOCATE_H_ #include -extern void* allocate(int alignment, size_t bytesize); +extern void* allocate(size_t alignment, size_t bytesize); #endif diff --git a/BasicSolver/3D-mpi/src/comm.c b/BasicSolver/3D-mpi/src/comm.c index 568b550..df9f795 100644 --- a/BasicSolver/3D-mpi/src/comm.c +++ b/BasicSolver/3D-mpi/src/comm.c @@ -1,10 +1,9 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ -#include #include #include #include @@ -12,6 +11,7 @@ #include "allocate.h" #include "comm.h" +#if defined(_MPI) // subroutines local to this module static int sizeOfRank(int rank, int size, int N) { @@ -167,29 +167,34 @@ static void assembleResult(Comm* c, MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); } -static int sum(int* sizes, int position) +// subroutines local to this module +static int sum(int* sizes, int init, int offset, int coord) { int sum = 0; - for (int i = 0; i < position; i++) { + for (int i = init - offset; coord > 0; i -= offset, --coord) { sum += sizes[i]; } return sum; } +#endif // defined _MPI // exported subroutines void commReduction(double* v, int op) { +#if defined(_MPI) if (op == MAX) { MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); } else if (op == SUM) { MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); } +#endif } int commIsBoundary(Comm* c, Direction direction) { +#if defined(_MPI) switch (direction) { case LEFT: return c->coords[ICORD] == 0; @@ -213,12 +218,14 @@ int commIsBoundary(Comm* c, Direction direction) printf("ERROR!\n"); break; } +#endif - return 0; + return 1; } void commExchange(Comm* c, double* grid) { +#if defined(_MPI) int counts[6] = { 1, 1, 1, 1, 1, 1 }; MPI_Aint displs[6] = { 0, 0, 0, 0, 0, 0 }; @@ -231,10 +238,12 @@ void commExchange(Comm* c, double* grid) displs, c->rbufferTypes, c->comm); +#endif } void commShift(Comm* c, double* f, double* g, double* h) { +#if defined(_MPI) MPI_Request requests[6] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL, MPI_REQUEST_NULL, @@ -282,6 +291,31 @@ void commShift(Comm* c, double* f, double* g, double* h) MPI_Isend(h, 1, c->sbufferTypes[BACK], c->neighbours[BACK], 2, c->comm, &requests[5]); MPI_Waitall(6, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax) +{ +#if defined(_MPI) + int sum = 0; + + for (int i = 0; i < c->coords[ICORD]; i++) { + sum += sizeOfRank(i, c->dims[ICORD], imax); + } + offsets[IDIM] = sum; + sum = 0; + + for (int i = 0; i < c->coords[JCORD]; i++) { + sum += sizeOfRank(i, c->dims[JCORD], jmax); + } + offsets[JDIM] = sum; + sum = 0; + + for (int i = 0; i < c->coords[KCORD]; i++) { + sum += sizeOfRank(i, c->dims[KCORD], kmax); + } + offsets[KDIM] = sum; +#endif } #define G(v, i, j, k) \ @@ -303,7 +337,7 @@ void commCollectResult(Comm* c, int imaxLocal = c->imaxLocal; int jmaxLocal = c->jmaxLocal; int kmaxLocal = c->kmaxLocal; - +#if defined(_MPI) int offset[c->size * NDIMS]; int imaxLocalAll[c->size]; int jmaxLocalAll[c->size]; @@ -317,9 +351,13 @@ void commCollectResult(Comm* c, for (int i = 0; i < c->size; i++) { int coords[NCORDS]; MPI_Cart_coords(c->comm, i, NDIMS, coords); - offset[i * NDIMS + IDIM] = sum(imaxLocalAll, coords[ICORD]); - offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, coords[JCORD]); - offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, coords[KCORD]); + offset[i * NDIMS + IDIM] = sum(imaxLocalAll, + i, + c->dims[IDIM] * c->dims[JDIM], + coords[ICORD]); + offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, i, c->dims[IDIM], coords[JCORD]); + offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, i, 1, coords[KCORD]); + printf("Rank: %d, Coords(k,j,i): %d %d %d, Size(k,j,i): %d %d %d, " "Offset(k,j,i): %d %d %d\n", i, @@ -426,10 +464,52 @@ void commCollectResult(Comm* c, imax); free(tmp); +#else + int idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + pg[idx++] = G(p, i, j, k); + } + } + } + + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + ug[idx++] = (G(u, i, j, k) + G(u, i - 1, j, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + vg[idx++] = (G(v, i, j, k) + G(v, i, j - 1, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + wg[idx++] = (G(w, i, j, k) + G(w, i, j, k - 1)) / 2.0; + } + } + } +#endif } void commPrintConfig(Comm* c) { +#if defined(_MPI) fflush(stdout); MPI_Barrier(MPI_COMM_WORLD); if (commIsMaster(c)) { @@ -459,13 +539,24 @@ void commPrintConfig(Comm* c) } } MPI_Barrier(MPI_COMM_WORLD); +#endif } -void commInit(Comm* c, int kmax, int jmax, int imax) +void commInit(Comm* c, int argc, char** argv) { - /* setup communication */ +#if defined(_MPI) + MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank)); MPI_Comm_size(MPI_COMM_WORLD, &(c->size)); +#else + c->rank = 0; + c->size = 1; +#endif +} + +void commPartition(Comm* c, int kmax, int jmax, int imax) +{ +#if defined(_MPI) int dims[NDIMS] = { 0, 0, 0 }; int periods[NDIMS] = { 0, 0, 0 }; MPI_Dims_create(c->size, NDIMS, dims); @@ -475,9 +566,9 @@ void commInit(Comm* c, int kmax, int jmax, int imax) MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]); MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords); - c->imaxLocal = sizeOfRank(c->rank, dims[ICORD], imax); - c->jmaxLocal = sizeOfRank(c->rank, dims[JCORD], jmax); - c->kmaxLocal = sizeOfRank(c->rank, dims[KCORD], kmax); + c->imaxLocal = sizeOfRank(c->coords[KDIM], dims[ICORD], imax); + c->jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JCORD], jmax); + c->kmaxLocal = sizeOfRank(c->coords[IDIM], dims[KCORD], kmax); // setup buffer types for communication setupCommunication(c, LEFT, BULK); @@ -492,12 +583,65 @@ void commInit(Comm* c, int kmax, int jmax, int imax) setupCommunication(c, FRONT, HALO); setupCommunication(c, BACK, BULK); setupCommunication(c, BACK, HALO); +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; + c->kmaxLocal = kmax; +#endif } -void commFree(Comm* c) +void commFinalize(Comm* c) { +#if defined(_MPI) for (int i = 0; i < NDIRS; i++) { MPI_Type_free(&c->sbufferTypes[i]); MPI_Type_free(&c->rbufferTypes[i]); } + + MPI_Finalize(); +#endif } + +void commUpdateDatatypes( + Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal) +{ +#if defined _MPI + + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + newcomm->kmaxLocal = kmaxLocal / 2; + + setupCommunication(newcomm, LEFT, BULK); + setupCommunication(newcomm, LEFT, HALO); + setupCommunication(newcomm, RIGHT, BULK); + setupCommunication(newcomm, RIGHT, HALO); + setupCommunication(newcomm, BOTTOM, BULK); + setupCommunication(newcomm, BOTTOM, HALO); + setupCommunication(newcomm, TOP, BULK); + setupCommunication(newcomm, TOP, HALO); + setupCommunication(newcomm, FRONT, BULK); + setupCommunication(newcomm, FRONT, HALO); + setupCommunication(newcomm, BACK, BULK); + setupCommunication(newcomm, BACK, HALO); +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; + newcomm->kmaxLocal = kmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/BasicSolver/3D-mpi/src/comm.h b/BasicSolver/3D-mpi/src/comm.h index 243db14..58ed4d7 100644 --- a/BasicSolver/3D-mpi/src/comm.h +++ b/BasicSolver/3D-mpi/src/comm.h @@ -1,12 +1,14 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ #ifndef __COMM_H_ #define __COMM_H_ +#if defined(_MPI) #include +#endif /* * Spatial directions: * ICORD (0) from 0 (LEFT) to imax (RIGHT) @@ -21,25 +23,32 @@ typedef enum dimension { KDIM = 0, JDIM, IDIM, NDIMS } Dimension; enum layer { HALO = 0, BULK }; enum op { MAX = 0, SUM }; + typedef struct { int rank; int size; +#if defined(_MPI) MPI_Comm comm; MPI_Datatype sbufferTypes[NDIRS]; MPI_Datatype rbufferTypes[NDIRS]; +#endif int neighbours[NDIRS]; int coords[NDIMS], dims[NDIMS]; int imaxLocal, jmaxLocal, kmaxLocal; - MPI_File fh; } Comm; -extern void commInit(Comm* comm, int kmax, int jmax, int imax); -extern void commFree(Comm* comm); +extern void commInit(Comm* c, int argc, char** argv); +extern void commPartition(Comm* c, int kmax, int jmax, int imax); +extern void commFinalize(Comm* comm); extern void commPrintConfig(Comm*); extern void commExchange(Comm*, double*); extern void commShift(Comm* c, double* f, double* g, double* h); extern void commReduction(double* v, int op); extern int commIsBoundary(Comm* c, Direction direction); +extern void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax); +extern void commFreeCommunicator(Comm* comm); +extern void commUpdateDatatypes( + Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal); extern void commCollectResult(Comm* c, double* ug, double* vg, diff --git a/BasicSolver/3D-mpi-io/src/solver.c b/BasicSolver/3D-mpi/src/discretization.c similarity index 86% rename from BasicSolver/3D-mpi-io/src/solver.c rename to BasicSolver/3D-mpi/src/discretization.c index 2a49504..138a037 100644 --- a/BasicSolver/3D-mpi-io/src/solver.c +++ b/BasicSolver/3D-mpi/src/discretization.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -7,13 +7,11 @@ #include #include #include -#include #include #include "allocate.h" -#include "comm.h" +#include "discretization.h" #include "parameter.h" -#include "solver.h" #include "util.h" #define P(i, j, k) \ @@ -33,7 +31,7 @@ #define RHS(i, j, k) \ rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] -static void printConfig(Solver* s) +static void printConfig(Discretization* s) { if (commIsMaster(&s->comm)) { printf("Parameters for #%s#\n", s->problem); @@ -72,7 +70,7 @@ static void printConfig(Solver* s) commPrintConfig(&s->comm); } -void initSolver(Solver* s, Parameter* params) +void initDiscretization(Discretization* s, Parameter* params) { s->problem = params->name; s->bcLeft = params->bcLeft; @@ -104,7 +102,6 @@ void initSolver(Solver* s, Parameter* params) s->tau = params->tau; s->gamma = params->gamma; - commInit(&s->comm, s->grid.kmax, s->grid.jmax, s->grid.imax); /* allocate arrays */ int imaxLocal = s->comm.imaxLocal; int jmaxLocal = s->comm.jmaxLocal; @@ -143,217 +140,7 @@ void initSolver(Solver* s, Parameter* params) #endif /* VERBOSE */ } -void computeRHS(Solver* s) -{ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - - double idx = 1.0 / s->grid.dx; - double idy = 1.0 / s->grid.dy; - double idz = 1.0 / s->grid.dz; - double idt = 1.0 / s->dt; - - double* rhs = s->rhs; - double* f = s->f; - double* g = s->g; - double* h = s->h; - - commShift(&s->comm, f, g, h); - - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx + - (G(i, j, k) - G(i, j - 1, k)) * idy + - (H(i, j, k) - H(i, j, k - 1)) * idz) * - idt; - } - } - } -} - -void solve(Solver* s) -{ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - - int imax = s->grid.imax; - int jmax = s->grid.jmax; - int kmax = s->grid.kmax; - - double eps = s->eps; - int itermax = s->itermax; - double dx2 = s->grid.dx * s->grid.dx; - double dy2 = s->grid.dy * s->grid.dy; - double dz2 = s->grid.dz * s->grid.dz; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double idz2 = 1.0 / dz2; - - double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / - (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); - double* p = s->p; - double* rhs = s->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - commExchange(&s->comm, p); - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - - double r = RHS(i, j, k) - - ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * - idx2 + - (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * - idy2 + - (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * - idz2); - - P(i, j, k) -= (factor * r); - res += (r * r); - } - } - } - - if (commIsBoundary(&s->comm, FRONT)) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, j, 0) = P(i, j, 1); - } - } - } - - if (commIsBoundary(&s->comm, BACK)) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); - } - } - } - - if (commIsBoundary(&s->comm, BOTTOM)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, 0, k) = P(i, 1, k); - } - } - } - - if (commIsBoundary(&s->comm, TOP)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); - } - } - } - - if (commIsBoundary(&s->comm, LEFT)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - P(0, j, k) = P(1, j, k); - } - } - } - - if (commIsBoundary(&s->comm, RIGHT)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); - } - } - } - - commReduction(&res, SUM); - res = res / (double)(imax * jmax * kmax); -#ifdef DEBUG - if (commIsMaster(&s->comm)) { - printf("%d Residuum: %e\n", it, res); - } -#endif - commExchange(&s->comm, p); - it++; - } - -#ifdef VERBOSE - if (commIsMaster(&s->comm)) { - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); - } -#endif -} - -static double maxElement(Solver* s, double* m) -{ - int size = (s->comm.imaxLocal + 2) * (s->comm.jmaxLocal + 2) * - (s->comm.kmaxLocal + 2); - double maxval = DBL_MIN; - - for (int i = 0; i < size; i++) { - maxval = MAX(maxval, fabs(m[i])); - } - commReduction(&maxval, MAX); - return maxval; -} - -void normalizePressure(Solver* s) -{ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - - double* p = s->p; - double avgP = 0.0; - - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - avgP += P(i, j, k); - } - } - } - commReduction(&avgP, SUM); - avgP /= (s->grid.imax * s->grid.jmax * s->grid.kmax); - - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, j, k) = P(i, j, k) - avgP; - } - } - } -} - -void computeTimestep(Solver* s) -{ - double dt = s->dtBound; - double dx = s->grid.dx; - double dy = s->grid.dy; - double dz = s->grid.dz; - - double umax = maxElement(s, s->u); - double vmax = maxElement(s, s->v); - double wmax = maxElement(s, s->w); - - if (umax > 0) { - dt = (dt > dx / umax) ? dx / umax : dt; - } - if (vmax > 0) { - dt = (dt > dy / vmax) ? dy / vmax : dt; - } - if (wmax > 0) { - dt = (dt > dz / wmax) ? dz / wmax : dt; - } - - s->dt = dt * s->tau; -} - -void setBoundaryConditions(Solver* s) +void setBoundaryConditions(Discretization* s) { int imaxLocal = s->comm.imaxLocal; int jmaxLocal = s->comm.jmaxLocal; @@ -568,7 +355,37 @@ void setBoundaryConditions(Solver* s) } } -void setSpecialBoundaryCondition(Solver* s) +void computeRHS(Discretization* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double idx = 1.0 / s->grid.dx; + double idy = 1.0 / s->grid.dy; + double idz = 1.0 / s->grid.dz; + double idt = 1.0 / s->dt; + + double* rhs = s->rhs; + double* f = s->f; + double* g = s->g; + double* h = s->h; + + commShift(&s->comm, f, g, h); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx + + (G(i, j, k) - G(i, j - 1, k)) * idy + + (H(i, j, k) - H(i, j, k - 1)) * idz) * + idt; + } + } + } +} + +void setSpecialBoundaryCondition(Discretization* s) { int imaxLocal = s->comm.imaxLocal; int jmaxLocal = s->comm.jmaxLocal; @@ -595,7 +412,74 @@ void setSpecialBoundaryCondition(Solver* s) } } -void computeFG(Solver* s) +static double maxElement(Discretization* s, double* m) +{ + int size = (s->comm.imaxLocal + 2) * (s->comm.jmaxLocal + 2) * + (s->comm.kmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + commReduction(&maxval, MAX); + return maxval; +} + +void normalizePressure(Discretization* s) +{ + int imaxLocal = s->comm.imaxLocal; + int jmaxLocal = s->comm.jmaxLocal; + int kmaxLocal = s->comm.kmaxLocal; + + double* p = s->p; + double avgP = 0.0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + avgP += P(i, j, k); + } + } + } + commReduction(&avgP, SUM); + avgP /= (s->grid.imax * s->grid.jmax * s->grid.kmax); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, k) = P(i, j, k) - avgP; + } + } + } +} + +void computeTimestep(Discretization* s) +{ + double dt = s->dtBound; + double dx = s->grid.dx; + double dy = s->grid.dy; + double dz = s->grid.dz; + + double umax = maxElement(s, s->u); + double vmax = maxElement(s, s->v); + double wmax = maxElement(s, s->w); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + if (wmax > 0) { + dt = (dt > dz / wmax) ? dz / wmax : dt; + } + + s->dt = dt * s->tau; +} + + + +void computeFG(Discretization* s) { int imaxLocal = s->comm.imaxLocal; int jmaxLocal = s->comm.jmaxLocal; @@ -815,7 +699,7 @@ void computeFG(Solver* s) } } -void adaptUV(Solver* s) +void adaptUV(Discretization* s) { int imaxLocal = s->comm.imaxLocal; int jmaxLocal = s->comm.jmaxLocal; @@ -842,4 +726,4 @@ void adaptUV(Solver* s) } } } -} +} \ No newline at end of file diff --git a/BasicSolver/3D-mpi/src/discretization.h b/BasicSolver/3D-mpi/src/discretization.h new file mode 100644 index 0000000..2c04800 --- /dev/null +++ b/BasicSolver/3D-mpi/src/discretization.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ + +#include "grid.h" +#include "parameter.h" +#include "comm.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g, *h; + double *u, *v, *w; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy, gz; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + Comm comm; +} Discretization; + +extern void initDiscretization(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +#endif diff --git a/BasicSolver/3D-mpi/src/grid.h b/BasicSolver/3D-mpi/src/grid.h index c963429..0689ebc 100644 --- a/BasicSolver/3D-mpi/src/grid.h +++ b/BasicSolver/3D-mpi/src/grid.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. diff --git a/BasicSolver/3D-mpi/src/main.c b/BasicSolver/3D-mpi/src/main.c index f53e2a1..3b68f70 100644 --- a/BasicSolver/3D-mpi/src/main.c +++ b/BasicSolver/3D-mpi/src/main.c @@ -1,90 +1,104 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ -#include -#include -#include #include #include #include #include "allocate.h" +#include "discretization.h" #include "parameter.h" #include "progress.h" #include "solver.h" -#include "test.h" #include "timing.h" #include "vtkWriter.h" int main(int argc, char** argv) { - int rank; double timeStart, timeStop; - Parameter params; - Solver solver; + Parameter p; + Solver s; + Discretization d; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - initParameter(¶ms); + commInit(&d.comm, argc, argv); + initParameter(&p); + FILE* fp; + if (commIsMaster(&d.comm)) fp = initResidualWriter(); if (argc != 2) { printf("Usage: %s \n", argv[0]); exit(EXIT_SUCCESS); } - readParameter(¶ms, argv[1]); - if (commIsMaster(&solver.comm)) { - printParameter(¶ms); + readParameter(&p, argv[1]); + commPartition(&d.comm, p.kmax, p.jmax, p.imax); + + if (commIsMaster(&d.comm)) { + printParameter(&p); } - initSolver(&solver, ¶ms); + + initDiscretization(&d, &p); + initSolver(&s, &d, &p); + #ifndef VERBOSE - initProgress(solver.te); + initProgress(d.te); #endif - double tau = solver.tau; - double te = solver.te; + double tau = d.tau; + double te = d.te; double t = 0.0; int nt = 0; + double res = 0.0; timeStart = getTimeStamp(); while (t <= te) { - if (tau > 0.0) computeTimestep(&solver); - setBoundaryConditions(&solver); - setSpecialBoundaryCondition(&solver); - computeFG(&solver); - computeRHS(&solver); - // if (nt % 100 == 0) normalizePressure(&solver); - solve(&solver); - adaptUV(&solver); - t += solver.dt; + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + computeFG(&d); + computeRHS(&d); + if (nt % 100 == 0) normalizePressure(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + + if (commIsMaster(&d.comm)) writeResidual(fp, t, res); + + t += d.dt; nt++; #ifdef VERBOSE - if (commIsMaster(&solver.comm)) { - printf("TIME %f , TIMESTEP %f\n", t, solver.dt); + if (commIsMaster(s.comm)) { + printf("TIME %f , TIMESTEP %f\n", t, d.dt); } #else printProgress(t); #endif } timeStop = getTimeStamp(); +#ifndef VERBOSE stopProgress(); - if (commIsMaster(&solver.comm)) { +#endif + if (commIsMaster(s.comm)) { printf("Solution took %.2fs\n", timeStop - timeStart); } - // testInit(&solver); - // commExchange(&solver.comm, solver.p); - // testPrintHalo(&solver, solver.p); + timeStart = getTimeStamp(); +#ifdef _VTK_WRITER_MPI + VtkOptions opts = { .grid = s.grid, .comm = s.comm }; + vtkOpen(&opts, s.problem); + vtkScalar(&opts, "pressure", d.p); + vtkVector(&opts, "velocity", (VtkVector) { d.u, d.v, d.w }); + vtkClose(&opts); +#else + if (commIsMaster(&d.comm)) fclose(fp); double *pg, *ug, *vg, *wg; - if (commIsMaster(&solver.comm)) { - size_t bytesize = solver.grid.imax * solver.grid.jmax * solver.grid.kmax * - sizeof(double); + if (commIsMaster(s.comm)) { + size_t bytesize = s.grid->imax * s.grid->jmax * s.grid->kmax * sizeof(double); pg = allocate(64, bytesize); ug = allocate(64, bytesize); @@ -92,26 +106,35 @@ int main(int argc, char** argv) wg = allocate(64, bytesize); } - commCollectResult(&solver.comm, + commCollectResult(s.comm, ug, vg, wg, pg, - solver.u, - solver.v, - solver.w, - solver.p, - solver.grid.kmax, - solver.grid.jmax, - solver.grid.imax); + d.u, + d.v, + d.w, + d.p, + s.grid->kmax, + s.grid->jmax, + s.grid->imax); - VtkOptions opts = { .grid = solver.grid, .comm = solver.comm }; - vtkOpen(&opts, solver.problem); - vtkScalar(&opts, "pressure", pg); - vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); - vtkClose(&opts); + if (commIsMaster(s.comm)) { + VtkOptions opts = { .grid = s.grid }; + vtkOpen(&opts, s.problem); + vtkScalar(&opts, "pressure", pg); + vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); + vtkClose(&opts); + } - commFree(&solver.comm); - MPI_Finalize(); +#endif + + timeStop = getTimeStamp(); + + if (commIsMaster(s.comm)) { + printf("Result output took %.2fs\n", timeStop - timeStart); + } + + commFinalize(s.comm); return EXIT_SUCCESS; } diff --git a/BasicSolver/3D-mpi/src/parameter.c b/BasicSolver/3D-mpi/src/parameter.c index d4ea5b6..3f98561 100644 --- a/BasicSolver/3D-mpi/src/parameter.c +++ b/BasicSolver/3D-mpi/src/parameter.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -14,18 +14,22 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->zlength = 1.0; - param->imax = 100; - param->jmax = 100; - param->kmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.7; - param->re = 100.0; - param->gamma = 0.9; - param->tau = 0.5; + param->xlength = 1.0; + param->ylength = 1.0; + param->zlength = 1.0; + param->imax = 100; + param->jmax = 100; + param->kmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; + } void readParameter(Parameter* param, const char* filename) @@ -65,6 +69,9 @@ void readParameter(Parameter* param, const char* filename) PARSE_INT(jmax); PARSE_INT(kmax); PARSE_INT(itermax); + PARSE_INT(levels); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); PARSE_REAL(eps); PARSE_REAL(omg); PARSE_REAL(re); diff --git a/BasicSolver/3D-mpi/src/parameter.h b/BasicSolver/3D-mpi/src/parameter.h index 6dddf5f..e40b9a4 100644 --- a/BasicSolver/3D-mpi/src/parameter.h +++ b/BasicSolver/3D-mpi/src/parameter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -18,6 +18,7 @@ typedef struct { char* name; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; double u_init, v_init, w_init, p_init; + int levels, presmooth, postsmooth; } Parameter; void initParameter(Parameter*); diff --git a/BasicSolver/3D-mpi/src/progress.c b/BasicSolver/3D-mpi/src/progress.c index a9b82bd..8249ed1 100644 --- a/BasicSolver/3D-mpi/src/progress.c +++ b/BasicSolver/3D-mpi/src/progress.c @@ -1,16 +1,15 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ +#include "progress.h" #include #include #include #include -#include "progress.h" - static double _end; static int _current; @@ -49,3 +48,18 @@ void stopProgress() printf("\n"); fflush(stdout); } + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; +} + +void writeResidual(FILE* fp, double ts, double res) { fprintf(fp, "%f, %f\n", ts, res); } \ No newline at end of file diff --git a/BasicSolver/3D-mpi/src/progress.h b/BasicSolver/3D-mpi/src/progress.h index 9ef2d96..d612ebe 100644 --- a/BasicSolver/3D-mpi/src/progress.h +++ b/BasicSolver/3D-mpi/src/progress.h @@ -1,14 +1,17 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ + +#include + #ifndef __PROGRESS_H_ #define __PROGRESS_H_ - extern void initProgress(double); extern void printProgress(double); -extern void stopProgress(); - +extern void stopProgress(void); +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/BasicSolver/3D-mpi/src/solver-mg.c b/BasicSolver/3D-mpi/src/solver-mg.c new file mode 100644 index 0000000..27b1b18 --- /dev/null +++ b/BasicSolver/3D-mpi/src/solver-mg.c @@ -0,0 +1,403 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +#define S(i, j, k) \ + s[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define E(i, j, k) \ + e[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define R(i, j, k) \ + r[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define OLD(i, j, k) \ + old[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +static void restrictMG(Solver* s, int level, Comm* comm) +{ + double* r = s->r[level + 1]; + double* old = s->r[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + commExchange(comm, old); + + for (int k = 1; k < comm->kmaxLocal + 1; k++) { + for (int j = 1; j < comm->jmaxLocal + 1; j++) { + for (int i = 1; i < comm->imaxLocal + 1; ++i) { + R(i, j, k) = (OLD(2 * i - 1, 2 * j - 1, 2 * k) + + OLD(2 * i, 2 * j - 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k) + + OLD(2 * i - 1, 2 * j, 2 * k) * 2 + + OLD(2 * i, 2 * j, 2 * k) * 8 + + OLD(2 * i + 1, 2 * j, 2 * k) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k) + + OLD(2 * i, 2 * j + 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i, 2 * j - 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i - 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i, 2 * j, 2 * k - 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k - 1) + + OLD(2 * i, 2 * j + 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k - 1) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i, 2 * j - 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i - 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i, 2 * j, 2 * k + 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k + 1) + + OLD(2 * i, 2 * j + 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k + 1)) / + 64.0; + } + } + } +} + +static void prolongate(Solver* s, int level, Comm* comm) +{ + double* old = s->r[level + 1]; + double* e = s->r[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + for (int k = 2; k < kmaxLocal + 1; k += 2) { + for (int j = 2; j < jmaxLocal + 1; j += 2) { + for (int i = 2; i < imaxLocal + 1; i += 2) { + E(i, j, k) = OLD(i / 2, j / 2, k / 2); + } + } + } +} + +static void correct(Solver* s, double* p, int level, Comm* comm) +{ + double* e = s->e[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + for (int k = 1; k < kmaxLocal + 1; ++k) { + for (int j = 1; j < jmaxLocal + 1; ++j) { + for (int i = 1; i < imaxLocal + 1; ++i) { + P(i, j, k) += E(i, j, k); + } + } + } +} + +static void setBoundaryCondition( + Solver* s, double* p, int imaxLocal, int jmaxLocal, int kmaxLocal) +{ +#ifdef _MPI + if (commIsBoundary(s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + } + } + } + + if (commIsBoundary(s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + } + + if (commIsBoundary(s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + } + } + } + + if (commIsBoundary(s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + } + + if (commIsBoundary(s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + } + } + } + + if (commIsBoundary(s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } + } +#else + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } +#endif +} + +static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + commExchange(comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + P(i, j, k) -= + factor * + (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + commExchange(comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + R(i, + j, + k) = (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * + idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + + res += (R(i, j, k) * R(i, j, k)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + commReduction(&res, SUM); + + res = res / (double)(imaxLocal * jmaxLocal * kmaxLocal); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + return res; +} + +static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, comm); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + } + + res = calculateResidual(s, p, rhs, level, comm); + + // restrict + restrictMG(s, level, comm); + + Comm newcomm; + commUpdateDatatypes(s->comm, + &newcomm, + imaxLocal, + jmaxLocal, + kmaxLocal); + + // MGSolver on residual and error. + multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm); + + commFreeCommunicator(&newcomm); + + // prolongate + prolongate(s, level, comm); + + // correct p on finer level using residual + correct(s, p, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + s->comm = &d->comm; + s->problem = p->name; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + int levels = s->levels; + if (commIsMaster(s->comm)) printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * (kmax + 2); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size * sizeof(double)); + s->e[j] = allocate(64, size * sizeof(double)); + + for (size_t i = 0; i < size; i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->comm); + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Residuum: %.6f\n", res); + } +#endif + + return res; +} diff --git a/BasicSolver/3D-mpi/src/solver-rb.c b/BasicSolver/3D-mpi/src/solver-rb.c new file mode 100644 index 0000000..d64fecd --- /dev/null +++ b/BasicSolver/3D-mpi/src/solver-rb.c @@ -0,0 +1,175 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + s->comm = &d->comm; + s->problem = p->name; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double epssq = eps * eps; + int it = 0; + double res = 1.0; + int pass, ksw, jsw, isw; + + while ((res >= epssq) && (it < itermax)) { + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + commExchange(s->comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + double r = + RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2); + + P(i, j, k) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +#ifdef _MPI + if (commIsBoundary(s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + } + } + } + + if (commIsBoundary(s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + } + + if (commIsBoundary(s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + } + } + } + + if (commIsBoundary(s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + } + + if (commIsBoundary(s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + } + } + } + + if (commIsBoundary(s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } + } +#else + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } +#endif + commReduction(&res, SUM); + res = res / (double)(imax * jmax * kmax); +#ifdef DEBUG + if (commIsMaster(&s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + commExchange(s->comm, p); + it++; + } + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + +return res; +} \ No newline at end of file diff --git a/BasicSolver/3D-mpi/src/solver.c b/BasicSolver/3D-mpi/src/solver.c deleted file mode 100644 index 2a49504..0000000 --- a/BasicSolver/3D-mpi/src/solver.c +++ /dev/null @@ -1,845 +0,0 @@ -/* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. - * All rights reserved. This file is part of nusif-solver. - * Use of this source code is governed by a MIT style - * license that can be found in the LICENSE file. - */ -#include -#include -#include -#include -#include - -#include "allocate.h" -#include "comm.h" -#include "parameter.h" -#include "solver.h" -#include "util.h" - -#define P(i, j, k) \ - p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] -#define F(i, j, k) \ - f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] -#define G(i, j, k) \ - g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] -#define H(i, j, k) \ - h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] -#define U(i, j, k) \ - u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] -#define V(i, j, k) \ - v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] -#define W(i, j, k) \ - w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] -#define RHS(i, j, k) \ - rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] - -static void printConfig(Solver* s) -{ - if (commIsMaster(&s->comm)) { - printf("Parameters for #%s#\n", s->problem); - printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n", - s->bcLeft, - s->bcRight, - s->bcBottom, - s->bcTop, - s->bcFront, - s->bcBack); - printf("\tReynolds number: %.2f\n", s->re); - printf("\tGx Gy: %.2f %.2f %.2f\n", s->gx, s->gy, s->gz); - printf("Geometry data:\n"); - printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", - s->grid.xlength, - s->grid.ylength, - s->grid.zlength); - printf("\tCells (x, y, z): %d, %d, %d\n", - s->grid.imax, - s->grid.jmax, - s->grid.kmax); - printf("\tCell size (dx, dy, dz): %f, %f, %f\n", - s->grid.dx, - s->grid.dy, - s->grid.dz); - printf("Timestep parameters:\n"); - printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te); - printf("\tdt bound: %.6f\n", s->dtBound); - printf("\tTau factor: %.2f\n", s->tau); - printf("Iterative parameters:\n"); - printf("\tMax iterations: %d\n", s->itermax); - printf("\tepsilon (stopping tolerance) : %f\n", s->eps); - printf("\tgamma factor: %f\n", s->gamma); - printf("\tomega (SOR relaxation): %f\n", s->omega); - } - commPrintConfig(&s->comm); -} - -void initSolver(Solver* s, Parameter* params) -{ - s->problem = params->name; - s->bcLeft = params->bcLeft; - s->bcRight = params->bcRight; - s->bcBottom = params->bcBottom; - s->bcTop = params->bcTop; - s->bcFront = params->bcFront; - s->bcBack = params->bcBack; - - s->grid.imax = params->imax; - s->grid.jmax = params->jmax; - s->grid.kmax = params->kmax; - s->grid.xlength = params->xlength; - s->grid.ylength = params->ylength; - s->grid.zlength = params->zlength; - s->grid.dx = params->xlength / params->imax; - s->grid.dy = params->ylength / params->jmax; - s->grid.dz = params->zlength / params->kmax; - - s->eps = params->eps; - s->omega = params->omg; - s->itermax = params->itermax; - s->re = params->re; - s->gx = params->gx; - s->gy = params->gy; - s->gz = params->gz; - s->dt = params->dt; - s->te = params->te; - s->tau = params->tau; - s->gamma = params->gamma; - - commInit(&s->comm, s->grid.kmax, s->grid.jmax, s->grid.imax); - /* allocate arrays */ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - size_t size = (imaxLocal + 2) * (jmaxLocal + 2) * (kmaxLocal + 2); - - s->u = allocate(64, size * sizeof(double)); - s->v = allocate(64, size * sizeof(double)); - s->w = allocate(64, size * sizeof(double)); - s->p = allocate(64, size * sizeof(double)); - s->rhs = allocate(64, size * sizeof(double)); - s->f = allocate(64, size * sizeof(double)); - s->g = allocate(64, size * sizeof(double)); - s->h = allocate(64, size * sizeof(double)); - - for (int i = 0; i < size; i++) { - s->u[i] = params->u_init; - s->v[i] = params->v_init; - s->w[i] = params->w_init; - s->p[i] = params->p_init; - s->rhs[i] = 0.0; - s->f[i] = 0.0; - s->g[i] = 0.0; - s->h[i] = 0.0; - } - - double dx = s->grid.dx; - double dy = s->grid.dy; - double dz = s->grid.dz; - - double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz); - s->dtBound = 0.5 * s->re * 1.0 / invSqrSum; - -#ifdef VERBOSE - printConfig(s); -#endif /* VERBOSE */ -} - -void computeRHS(Solver* s) -{ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - - double idx = 1.0 / s->grid.dx; - double idy = 1.0 / s->grid.dy; - double idz = 1.0 / s->grid.dz; - double idt = 1.0 / s->dt; - - double* rhs = s->rhs; - double* f = s->f; - double* g = s->g; - double* h = s->h; - - commShift(&s->comm, f, g, h); - - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx + - (G(i, j, k) - G(i, j - 1, k)) * idy + - (H(i, j, k) - H(i, j, k - 1)) * idz) * - idt; - } - } - } -} - -void solve(Solver* s) -{ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - - int imax = s->grid.imax; - int jmax = s->grid.jmax; - int kmax = s->grid.kmax; - - double eps = s->eps; - int itermax = s->itermax; - double dx2 = s->grid.dx * s->grid.dx; - double dy2 = s->grid.dy * s->grid.dy; - double dz2 = s->grid.dz * s->grid.dz; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double idz2 = 1.0 / dz2; - - double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / - (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); - double* p = s->p; - double* rhs = s->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - commExchange(&s->comm, p); - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - - double r = RHS(i, j, k) - - ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * - idx2 + - (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * - idy2 + - (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * - idz2); - - P(i, j, k) -= (factor * r); - res += (r * r); - } - } - } - - if (commIsBoundary(&s->comm, FRONT)) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, j, 0) = P(i, j, 1); - } - } - } - - if (commIsBoundary(&s->comm, BACK)) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); - } - } - } - - if (commIsBoundary(&s->comm, BOTTOM)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, 0, k) = P(i, 1, k); - } - } - } - - if (commIsBoundary(&s->comm, TOP)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); - } - } - } - - if (commIsBoundary(&s->comm, LEFT)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - P(0, j, k) = P(1, j, k); - } - } - } - - if (commIsBoundary(&s->comm, RIGHT)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); - } - } - } - - commReduction(&res, SUM); - res = res / (double)(imax * jmax * kmax); -#ifdef DEBUG - if (commIsMaster(&s->comm)) { - printf("%d Residuum: %e\n", it, res); - } -#endif - commExchange(&s->comm, p); - it++; - } - -#ifdef VERBOSE - if (commIsMaster(&s->comm)) { - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); - } -#endif -} - -static double maxElement(Solver* s, double* m) -{ - int size = (s->comm.imaxLocal + 2) * (s->comm.jmaxLocal + 2) * - (s->comm.kmaxLocal + 2); - double maxval = DBL_MIN; - - for (int i = 0; i < size; i++) { - maxval = MAX(maxval, fabs(m[i])); - } - commReduction(&maxval, MAX); - return maxval; -} - -void normalizePressure(Solver* s) -{ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - - double* p = s->p; - double avgP = 0.0; - - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - avgP += P(i, j, k); - } - } - } - commReduction(&avgP, SUM); - avgP /= (s->grid.imax * s->grid.jmax * s->grid.kmax); - - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - P(i, j, k) = P(i, j, k) - avgP; - } - } - } -} - -void computeTimestep(Solver* s) -{ - double dt = s->dtBound; - double dx = s->grid.dx; - double dy = s->grid.dy; - double dz = s->grid.dz; - - double umax = maxElement(s, s->u); - double vmax = maxElement(s, s->v); - double wmax = maxElement(s, s->w); - - if (umax > 0) { - dt = (dt > dx / umax) ? dx / umax : dt; - } - if (vmax > 0) { - dt = (dt > dy / vmax) ? dy / vmax : dt; - } - if (wmax > 0) { - dt = (dt > dz / wmax) ? dz / wmax : dt; - } - - s->dt = dt * s->tau; -} - -void setBoundaryConditions(Solver* s) -{ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - - double* u = s->u; - double* v = s->v; - double* w = s->w; - - if (commIsBoundary(&s->comm, TOP)) { - switch (s->bcTop) { - case NOSLIP: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, jmaxLocal + 1, k) = -U(i, jmaxLocal, k); - V(i, jmaxLocal, k) = 0.0; - W(i, jmaxLocal + 1, k) = -W(i, jmaxLocal, k); - } - } - break; - case SLIP: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k); - V(i, jmaxLocal, k) = 0.0; - W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k); - } - } - break; - case OUTFLOW: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k); - V(i, jmaxLocal, k) = V(i, jmaxLocal - 1, k); - W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k); - } - } - break; - case PERIODIC: - break; - } - } - - if (commIsBoundary(&s->comm, BOTTOM)) { - switch (s->bcBottom) { - case NOSLIP: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, 0, k) = -U(i, 1, k); - V(i, 0, k) = 0.0; - W(i, 0, k) = -W(i, 1, k); - } - } - break; - case SLIP: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, 0, k) = U(i, 1, k); - V(i, 0, k) = 0.0; - W(i, 0, k) = W(i, 1, k); - } - } - break; - case OUTFLOW: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, 0, k) = U(i, 1, k); - V(i, 0, k) = V(i, 1, k); - W(i, 0, k) = W(i, 1, k); - } - } - break; - case PERIODIC: - break; - } - } - - if (commIsBoundary(&s->comm, LEFT)) { - switch (s->bcLeft) { - case NOSLIP: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j, k) = 0.0; - V(0, j, k) = -V(1, j, k); - W(0, j, k) = -W(1, j, k); - } - } - break; - case SLIP: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j, k) = 0.0; - V(0, j, k) = V(1, j, k); - W(0, j, k) = W(1, j, k); - } - } - break; - case OUTFLOW: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j, k) = U(1, j, k); - V(0, j, k) = V(1, j, k); - W(0, j, k) = W(1, j, k); - } - } - break; - case PERIODIC: - break; - } - } - - if (commIsBoundary(&s->comm, RIGHT)) { - switch (s->bcRight) { - case NOSLIP: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imaxLocal, j, k) = 0.0; - V(imaxLocal + 1, j, k) = -V(imaxLocal, j, k); - W(imaxLocal + 1, j, k) = -W(imaxLocal, j, k); - } - } - break; - case SLIP: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imaxLocal, j, k) = 0.0; - V(imaxLocal + 1, j, k) = V(imaxLocal, j, k); - W(imaxLocal + 1, j, k) = W(imaxLocal, j, k); - } - } - break; - case OUTFLOW: - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - U(imaxLocal, j, k) = U(imaxLocal - 1, j, k); - V(imaxLocal + 1, j, k) = V(imaxLocal, j, k); - W(imaxLocal + 1, j, k) = W(imaxLocal, j, k); - } - } - break; - case PERIODIC: - break; - } - } - - if (commIsBoundary(&s->comm, FRONT)) { - switch (s->bcFront) { - case NOSLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, j, 0) = -U(i, j, 1); - V(i, j, 0) = -V(i, j, 1); - W(i, j, 0) = 0.0; - } - } - break; - case SLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, j, 0) = U(i, j, 1); - V(i, j, 0) = V(i, j, 1); - W(i, j, 0) = 0.0; - } - } - break; - case OUTFLOW: - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, j, 0) = U(i, j, 1); - V(i, j, 0) = V(i, j, 1); - W(i, j, 0) = W(i, j, 1); - } - } - break; - case PERIODIC: - break; - } - } - - if (commIsBoundary(&s->comm, BACK)) { - switch (s->bcBack) { - case NOSLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, j, kmaxLocal + 1) = -U(i, j, kmaxLocal); - V(i, j, kmaxLocal + 1) = -V(i, j, kmaxLocal); - W(i, j, kmaxLocal) = 0.0; - } - } - break; - case SLIP: - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal); - V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal); - W(i, j, kmaxLocal) = 0.0; - } - } - break; - case OUTFLOW: - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal); - V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal); - W(i, j, kmaxLocal) = W(i, j, kmaxLocal - 1); - } - } - break; - case PERIODIC: - break; - } - } -} - -void setSpecialBoundaryCondition(Solver* s) -{ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - - double* u = s->u; - - if (strcmp(s->problem, "dcavity") == 0) { - if (commIsBoundary(&s->comm, TOP)) { - for (int k = 1; k < kmaxLocal; k++) { - for (int i = 1; i < imaxLocal; i++) { - U(i, jmaxLocal + 1, k) = 2.0 - U(i, jmaxLocal, k); - } - } - } - } else if (strcmp(s->problem, "canal") == 0) { - if (commIsBoundary(&s->comm, LEFT)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - U(0, j, k) = 2.0; - } - } - } - } -} - -void computeFG(Solver* s) -{ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - - double* u = s->u; - double* v = s->v; - double* w = s->w; - double* f = s->f; - double* g = s->g; - double* h = s->h; - - double gx = s->gx; - double gy = s->gy; - double gz = s->gz; - double dt = s->dt; - - double gamma = s->gamma; - double inverseRe = 1.0 / s->re; - double inverseDx = 1.0 / s->grid.dx; - double inverseDy = 1.0 / s->grid.dy; - double inverseDz = 1.0 / s->grid.dz; - double du2dx, dv2dy, dw2dz; - double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz; - double du2dx2, du2dy2, du2dz2; - double dv2dx2, dv2dy2, dv2dz2; - double dw2dx2, dw2dy2, dw2dz2; - - commExchange(&s->comm, u); - commExchange(&s->comm, v); - commExchange(&s->comm, w); - - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - du2dx = inverseDx * 0.25 * - ((U(i, j, k) + U(i + 1, j, k)) * - (U(i, j, k) + U(i + 1, j, k)) - - (U(i, j, k) + U(i - 1, j, k)) * - (U(i, j, k) + U(i - 1, j, k))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j, k) + U(i + 1, j, k)) * - (U(i, j, k) - U(i + 1, j, k)) + - fabs(U(i, j, k) + U(i - 1, j, k)) * - (U(i, j, k) - U(i - 1, j, k))); - - duvdy = inverseDy * 0.25 * - ((V(i, j, k) + V(i + 1, j, k)) * - (U(i, j, k) + U(i, j + 1, k)) - - (V(i, j - 1, k) + V(i + 1, j - 1, k)) * - (U(i, j, k) + U(i, j - 1, k))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j, k) + V(i + 1, j, k)) * - (U(i, j, k) - U(i, j + 1, k)) + - fabs(V(i, j - 1, k) + V(i + 1, j - 1, k)) * - (U(i, j, k) - U(i, j - 1, k))); - - duwdz = inverseDz * 0.25 * - ((W(i, j, k) + W(i + 1, j, k)) * - (U(i, j, k) + U(i, j, k + 1)) - - (W(i, j, k - 1) + W(i + 1, j, k - 1)) * - (U(i, j, k) + U(i, j, k - 1))) + - gamma * inverseDz * 0.25 * - (fabs(W(i, j, k) + W(i + 1, j, k)) * - (U(i, j, k) - U(i, j, k + 1)) + - fabs(W(i, j, k - 1) + W(i + 1, j, k - 1)) * - (U(i, j, k) - U(i, j, k - 1))); - - du2dx2 = inverseDx * inverseDx * - (U(i + 1, j, k) - 2.0 * U(i, j, k) + U(i - 1, j, k)); - du2dy2 = inverseDy * inverseDy * - (U(i, j + 1, k) - 2.0 * U(i, j, k) + U(i, j - 1, k)); - du2dz2 = inverseDz * inverseDz * - (U(i, j, k + 1) - 2.0 * U(i, j, k) + U(i, j, k - 1)); - F(i, j, k) = U(i, j, k) + dt * (inverseRe * (du2dx2 + du2dy2 + du2dz2) - - du2dx - duvdy - duwdz + gx); - - duvdx = inverseDx * 0.25 * - ((U(i, j, k) + U(i, j + 1, k)) * - (V(i, j, k) + V(i + 1, j, k)) - - (U(i - 1, j, k) + U(i - 1, j + 1, k)) * - (V(i, j, k) + V(i - 1, j, k))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j, k) + U(i, j + 1, k)) * - (V(i, j, k) - V(i + 1, j, k)) + - fabs(U(i - 1, j, k) + U(i - 1, j + 1, k)) * - (V(i, j, k) - V(i - 1, j, k))); - - dv2dy = inverseDy * 0.25 * - ((V(i, j, k) + V(i, j + 1, k)) * - (V(i, j, k) + V(i, j + 1, k)) - - (V(i, j, k) + V(i, j - 1, k)) * - (V(i, j, k) + V(i, j - 1, k))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j, k) + V(i, j + 1, k)) * - (V(i, j, k) - V(i, j + 1, k)) + - fabs(V(i, j, k) + V(i, j - 1, k)) * - (V(i, j, k) - V(i, j - 1, k))); - - dvwdz = inverseDz * 0.25 * - ((W(i, j, k) + W(i, j + 1, k)) * - (V(i, j, k) + V(i, j, k + 1)) - - (W(i, j, k - 1) + W(i, j + 1, k - 1)) * - (V(i, j, k) + V(i, j, k + 1))) + - gamma * inverseDz * 0.25 * - (fabs(W(i, j, k) + W(i, j + 1, k)) * - (V(i, j, k) - V(i, j, k + 1)) + - fabs(W(i, j, k - 1) + W(i, j + 1, k - 1)) * - (V(i, j, k) - V(i, j, k + 1))); - - dv2dx2 = inverseDx * inverseDx * - (V(i + 1, j, k) - 2.0 * V(i, j, k) + V(i - 1, j, k)); - dv2dy2 = inverseDy * inverseDy * - (V(i, j + 1, k) - 2.0 * V(i, j, k) + V(i, j - 1, k)); - dv2dz2 = inverseDz * inverseDz * - (V(i, j, k + 1) - 2.0 * V(i, j, k) + V(i, j, k - 1)); - G(i, j, k) = V(i, j, k) + dt * (inverseRe * (dv2dx2 + dv2dy2 + dv2dz2) - - duvdx - dv2dy - dvwdz + gy); - - duwdx = inverseDx * 0.25 * - ((U(i, j, k) + U(i, j, k + 1)) * - (W(i, j, k) + W(i + 1, j, k)) - - (U(i - 1, j, k) + U(i - 1, j, k + 1)) * - (W(i, j, k) + W(i - 1, j, k))) + - gamma * inverseDx * 0.25 * - (fabs(U(i, j, k) + U(i, j, k + 1)) * - (W(i, j, k) - W(i + 1, j, k)) + - fabs(U(i - 1, j, k) + U(i - 1, j, k + 1)) * - (W(i, j, k) - W(i - 1, j, k))); - - dvwdy = inverseDy * 0.25 * - ((V(i, j, k) + V(i, j, k + 1)) * - (W(i, j, k) + W(i, j + 1, k)) - - (V(i, j - 1, k + 1) + V(i, j - 1, k)) * - (W(i, j, k) + W(i, j - 1, k))) + - gamma * inverseDy * 0.25 * - (fabs(V(i, j, k) + V(i, j, k + 1)) * - (W(i, j, k) - W(i, j + 1, k)) + - fabs(V(i, j - 1, k + 1) + V(i, j - 1, k)) * - (W(i, j, k) - W(i, j - 1, k))); - - dw2dz = inverseDz * 0.25 * - ((W(i, j, k) + W(i, j, k + 1)) * - (W(i, j, k) + W(i, j, k + 1)) - - (W(i, j, k) + W(i, j, k - 1)) * - (W(i, j, k) + W(i, j, k - 1))) + - gamma * inverseDz * 0.25 * - (fabs(W(i, j, k) + W(i, j, k + 1)) * - (W(i, j, k) - W(i, j, k + 1)) + - fabs(W(i, j, k) + W(i, j, k - 1)) * - (W(i, j, k) - W(i, j, k - 1))); - - dw2dx2 = inverseDx * inverseDx * - (W(i + 1, j, k) - 2.0 * W(i, j, k) + W(i - 1, j, k)); - dw2dy2 = inverseDy * inverseDy * - (W(i, j + 1, k) - 2.0 * W(i, j, k) + W(i, j - 1, k)); - dw2dz2 = inverseDz * inverseDz * - (W(i, j, k + 1) - 2.0 * W(i, j, k) + W(i, j, k - 1)); - H(i, j, k) = W(i, j, k) + dt * (inverseRe * (dw2dx2 + dw2dy2 + dw2dz2) - - duwdx - dvwdy - dw2dz + gz); - } - } - } - - /* ----------------------------- boundary of F --------------------------- - */ - if (commIsBoundary(&s->comm, LEFT)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - F(0, j, k) = U(0, j, k); - } - } - } - - if (commIsBoundary(&s->comm, RIGHT)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - F(imaxLocal, j, k) = U(imaxLocal, j, k); - } - } - } - - /* ----------------------------- boundary of G --------------------------- - */ - if (commIsBoundary(&s->comm, BOTTOM)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - G(i, 0, k) = V(i, 0, k); - } - } - } - - if (commIsBoundary(&s->comm, TOP)) { - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int i = 1; i < imaxLocal + 1; i++) { - G(i, jmaxLocal, k) = V(i, jmaxLocal, k); - } - } - } - - /* ----------------------------- boundary of H --------------------------- - */ - if (commIsBoundary(&s->comm, FRONT)) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - H(i, j, 0) = W(i, j, 0); - } - } - } - - if (commIsBoundary(&s->comm, BACK)) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - H(i, j, kmaxLocal) = W(i, j, kmaxLocal); - } - } - } -} - -void adaptUV(Solver* s) -{ - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - - double* p = s->p; - double* u = s->u; - double* v = s->v; - double* w = s->w; - double* f = s->f; - double* g = s->g; - double* h = s->h; - - double factorX = s->dt / s->grid.dx; - double factorY = s->dt / s->grid.dy; - double factorZ = s->dt / s->grid.dz; - - for (int k = 1; k < kmaxLocal + 1; k++) { - for (int j = 1; j < jmaxLocal + 1; j++) { - for (int i = 1; i < imaxLocal + 1; i++) { - U(i, j, k) = F(i, j, k) - (P(i + 1, j, k) - P(i, j, k)) * factorX; - V(i, j, k) = G(i, j, k) - (P(i, j + 1, k) - P(i, j, k)) * factorY; - W(i, j, k) = H(i, j, k) - (P(i, j, k + 1) - P(i, j, k)) * factorZ; - } - } - } -} diff --git a/BasicSolver/3D-mpi/src/solver.h b/BasicSolver/3D-mpi/src/solver.h index ac9f450..d9a0c39 100644 --- a/BasicSolver/3D-mpi/src/solver.h +++ b/BasicSolver/3D-mpi/src/solver.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -9,12 +9,11 @@ #include "comm.h" #include "grid.h" #include "parameter.h" - -enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; +#include "discretization.h" typedef struct { /* geometry and grid information */ - Grid grid; + Grid* grid; /* arrays */ double *p, *rhs; double *f, *g, *h; @@ -30,16 +29,11 @@ typedef struct { char* problem; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; /* communication */ - Comm comm; + double **r, **e; + int levels, presmooth, postsmooth; + Comm* comm; } Solver; -void initSolver(Solver*, Parameter*); -void computeRHS(Solver*); -void solve(Solver*); -void normalizePressure(Solver*); -void computeTimestep(Solver*); -void setBoundaryConditions(Solver*); -void setSpecialBoundaryCondition(Solver*); -void computeFG(Solver*); -void adaptUV(Solver*); +extern double solve(Solver* , double* , double* ); +extern void initSolver(Solver*, Discretization*, Parameter*); #endif diff --git a/BasicSolver/3D-mpi/src/test.c b/BasicSolver/3D-mpi/src/test.c index 1665ce6..ba6bc78 100644 --- a/BasicSolver/3D-mpi/src/test.c +++ b/BasicSolver/3D-mpi/src/test.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -14,10 +14,10 @@ void testInit(Solver* s) { - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - int myrank = s->comm.rank; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; + int myrank = s->comm->rank; double* p = s->p; double* f = s->f; double* g = s->g; @@ -26,10 +26,21 @@ void testInit(Solver* s) for (int k = 0; k < kmaxLocal + 2; k++) { for (int j = 0; j < jmaxLocal + 2; j++) { for (int i = 0; i < imaxLocal + 2; i++) { - G(p, i, j, k) = myrank; - G(f, i, j, k) = myrank; - G(g, i, j, k) = myrank; - G(h, i, j, k) = myrank; + G(p, i, j, k) = 10.0; + G(f, i, j, k) = myrank + 1.0; + G(g, i, j, k) = myrank + 1.0; + G(h, i, j, k) = myrank + 1.0; + } + } + } + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(p, i, j, k) = myrank + 1.0; + G(f, i, j, k) = myrank + 1.0; + G(g, i, j, k) = myrank + 1.0; + G(h, i, j, k) = myrank + 1.0; } } } @@ -65,11 +76,11 @@ static char* direction2String(Direction dir) static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir) { - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; char filename[50]; - snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm.rank); + snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm->rank); FILE* fh = fopen(filename, "w"); for (int y = 0; y < ymax; y++) { @@ -105,9 +116,9 @@ static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir) void testPrintHalo(Solver* s, double* a) { - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, BOTTOM); printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, TOP); diff --git a/BasicSolver/3D-mpi/src/test.h b/BasicSolver/3D-mpi/src/test.h index ecefe4e..779b509 100644 --- a/BasicSolver/3D-mpi/src/test.h +++ b/BasicSolver/3D-mpi/src/test.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. diff --git a/BasicSolver/3D-mpi/src/timing.c b/BasicSolver/3D-mpi/src/timing.c index c4025a4..78b01c4 100644 --- a/BasicSolver/3D-mpi/src/timing.c +++ b/BasicSolver/3D-mpi/src/timing.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,18 +7,16 @@ #include #include -double getTimeStamp() +double getTimeStamp(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } -double getTimeResolution() +double getTimeResolution(void) { struct timespec ts; clock_getres(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } - -double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/3D-mpi/src/timing.h b/BasicSolver/3D-mpi/src/timing.h index db95329..ed05a8c 100644 --- a/BasicSolver/3D-mpi/src/timing.h +++ b/BasicSolver/3D-mpi/src/timing.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,8 +7,7 @@ #ifndef __TIMING_H_ #define __TIMING_H_ -extern double getTimeStamp(); -extern double getTimeResolution(); -extern double getTimeStamp_(); +extern double getTimeStamp(void); +extern double getTimeResolution(void); #endif // __TIMING_H_ diff --git a/BasicSolver/3D-mpi/src/util.h b/BasicSolver/3D-mpi/src/util.h index 657b009..68b87f0 100644 --- a/BasicSolver/3D-mpi/src/util.h +++ b/BasicSolver/3D-mpi/src/util.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -19,4 +19,13 @@ #define ABS(a) ((a) >= 0 ? (a) : -(a)) #endif +#define P(i, j, k) p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define F(i, j, k) f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define G(i, j, k) g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define H(i, j, k) h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define U(i, j, k) u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define V(i, j, k) v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define W(i, j, k) w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define RHS(i, j, k) rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + #endif // __UTIL_H_ diff --git a/BasicSolver/3D-mpi-io/src/vtkWriter.c b/BasicSolver/3D-mpi/src/vtkWriter-mpi.c similarity index 77% rename from BasicSolver/3D-mpi-io/src/vtkWriter.c rename to BasicSolver/3D-mpi/src/vtkWriter-mpi.c index fe9dc3f..d9670d5 100644 --- a/BasicSolver/3D-mpi-io/src/vtkWriter.c +++ b/BasicSolver/3D-mpi/src/vtkWriter-mpi.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -14,59 +14,63 @@ #include "comm.h" #include "vtkWriter.h" +// reset fileview for output of string headers static void resetFileview(VtkOptions* o) { - // reset fileview for output of string header MPI_Offset disp; + MPI_File_sync(o->fh); MPI_Barrier(o->comm.comm); MPI_File_get_size(o->fh, &disp); MPI_File_set_view(o->fh, disp, MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL); - // printf("Rank %d disp %lld\n", o->comm.rank, disp); +} + +static void writeVersion(VtkOptions* o) +{ + char header[50] = "# vtk DataFile Version 3.0\n"; + // always overwrite exiting files + MPI_File_set_view(o->fh, 0, MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL); + + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + } } static void writeHeader(VtkOptions* o) { - const size_t MAX_HEADER = 200; - - char* header = (char*)malloc(MAX_HEADER); + char header[400]; char* cursor = header; - cursor += sprintf(cursor, "# vtk DataFile Version 3.0\n"); cursor += sprintf(cursor, "PAMPI cfd solver output\n"); cursor += sprintf(cursor, "BINARY\n"); cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n"); cursor += sprintf(cursor, "DIMENSIONS %d %d %d\n", - o->grid.imax, - o->grid.jmax, - o->grid.kmax); + o->grid->imax, + o->grid->jmax, + o->grid->kmax); cursor += sprintf(cursor, "ORIGIN %f %f %f\n", - o->grid.dx * 0.5, - o->grid.dy * 0.5, - o->grid.dz * 0.5); - cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz); + o->grid->dx * 0.5, + o->grid->dy * 0.5, + o->grid->dz * 0.5); + cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz); cursor += sprintf(cursor, "POINT_DATA %d\n", - o->grid.imax * o->grid.jmax * o->grid.kmax); + o->grid->imax * o->grid->jmax * o->grid->kmax); if (commIsMaster(&o->comm)) { - MPI_File_write(o->fh, header, strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); } - - free(header); } -// TODO Check inputs for length void vtkOpen(VtkOptions* o, char* problem) { char filename[50]; - snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size); MPI_File_open(o->comm.comm, filename, - MPI_MODE_WRONLY | MPI_MODE_CREATE | MPI_MODE_EXCL, + MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &o->fh); @@ -74,40 +78,36 @@ void vtkOpen(VtkOptions* o, char* problem) printf("Writing VTK output for %s\n", problem); } - resetFileview(o); + writeVersion(o); writeHeader(o); } void vtkScalar(VtkOptions* o, char* name, double* s) { resetFileview(o); - if (commIsMaster(&o->comm)) printf("Register scalar %s\n", name); - const size_t MAX_HEADER = 100; - char* header = (char*)malloc(MAX_HEADER); + char header[100]; char* cursor = header; - cursor += sprintf(cursor, "SCALARS %s double 1\n", name); - cursor += sprintf(cursor, "LOOKUP_TABLE default\n"); + cursor += sprintf(cursor, "SCALARS %s double\n", name); if (commIsMaster(&o->comm)) { - MPI_File_write(o->fh, header, strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); } - free(header); - int offsets[NDIMS]; - commGetOffsets(&o->comm, offsets, o->grid.imax, o->grid.jmax, o->grid.kmax); + commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax); // set global view in file MPI_Offset disp; MPI_Datatype fileViewType; + MPI_File_sync(o->fh); MPI_Barrier(o->comm.comm); MPI_File_get_size(o->fh, &disp); MPI_Type_create_subarray(NDIMS, - (int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax }, + (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax }, (int[NDIMS]) { o->comm.kmaxLocal, o->comm.jmaxLocal, o->comm.imaxLocal }, offsets, MPI_ORDER_C, @@ -172,15 +172,16 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec) resetFileview(o); if (commIsMaster(&o->comm)) { - MPI_File_write(o->fh, header, strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); } int offsets[NDIMS]; - commGetOffsets(&o->comm, offsets, o->grid.imax, o->grid.jmax, o->grid.kmax); + commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax); // set global view in file MPI_Offset disp; MPI_Datatype fileViewType, vectorType; + MPI_File_sync(o->fh); MPI_Barrier(o->comm.comm); MPI_File_get_size(o->fh, &disp); @@ -188,7 +189,7 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec) MPI_Type_commit(&vectorType); MPI_Type_create_subarray(NDIMS, - (int[NDIMS]) { o->grid.kmax, o->grid.jmax, o->grid.imax }, + (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax }, (int[NDIMS]) { kmaxLocal, jmaxLocal, imaxLocal }, offsets, MPI_ORDER_C, diff --git a/BasicSolver/3D-mpi/src/vtkWriter.c b/BasicSolver/3D-mpi/src/vtkWriter-seq.c similarity index 62% rename from BasicSolver/3D-mpi/src/vtkWriter.c rename to BasicSolver/3D-mpi/src/vtkWriter-seq.c index e96e773..4bb4fbb 100644 --- a/BasicSolver/3D-mpi/src/vtkWriter.c +++ b/BasicSolver/3D-mpi/src/vtkWriter-seq.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -41,37 +41,32 @@ static void writeHeader(VtkOptions* o) } fprintf(o->fh, "DATASET STRUCTURED_POINTS\n"); - fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid.imax, o->grid.jmax, o->grid.kmax); + fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid->imax, o->grid->jmax, o->grid->kmax); fprintf(o->fh, "ORIGIN %f %f %f\n", - o->grid.dx * 0.5, - o->grid.dy * 0.5, - o->grid.dz * 0.5); - fprintf(o->fh, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz); - fprintf(o->fh, "POINT_DATA %d\n", o->grid.imax * o->grid.jmax * o->grid.kmax); + o->grid->dx * 0.5, + o->grid->dy * 0.5, + o->grid->dz * 0.5); + fprintf(o->fh, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz); + fprintf(o->fh, "POINT_DATA %d\n", o->grid->imax * o->grid->jmax * o->grid->kmax); } void vtkOpen(VtkOptions* o, char* problem) { char filename[50]; - if (o->mode == UNIX) { - if (commIsMaster(&o->comm)) { - snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size); - o->fh = fopen(filename, "w"); - writeHeader(o); - } - } else if (o->mode == MPI) { - } + snprintf(filename, 50, "%s.vtk", problem); + o->fh = fopen(filename, "w"); + writeHeader(o); - if (commIsMaster(&o->comm)) printf("Writing VTK output for %s\n", problem); + printf("Writing VTK output for %s\n", problem); } static void writeScalar(VtkOptions* o, double* s) { - int imax = o->grid.imax; - int jmax = o->grid.jmax; - int kmax = o->grid.kmax; + int imax = o->grid->imax; + int jmax = o->grid->jmax; + int kmax = o->grid->kmax; for (int k = 0; k < kmax; k++) { for (int j = 0; j < jmax; j++) { @@ -101,24 +96,18 @@ static bool isInitialized(FILE* ptr) void vtkScalar(VtkOptions* o, char* name, double* s) { - if (commIsMaster(&o->comm)) printf("Register scalar %s\n", name); - - if (o->mode == UNIX) { - if (commIsMaster(&o->comm)) { - if (!isInitialized(o->fh)) return; - fprintf(o->fh, "SCALARS %s double 1\n", name); - fprintf(o->fh, "LOOKUP_TABLE default\n"); - writeScalar(o, s); - } - } else if (o->mode == MPI) { - } + printf("Register scalar %s\n", name); + if (!isInitialized(o->fh)) return; + fprintf(o->fh, "SCALARS %s double 1\n", name); + fprintf(o->fh, "LOOKUP_TABLE default\n"); + writeScalar(o, s); } static void writeVector(VtkOptions* o, VtkVector vec) { - int imax = o->grid.imax; - int jmax = o->grid.jmax; - int kmax = o->grid.kmax; + int imax = o->grid->imax; + int jmax = o->grid->jmax; + int kmax = o->grid->kmax; for (int k = 0; k < kmax; k++) { for (int j = 0; j < jmax; j++) { @@ -145,25 +134,14 @@ static void writeVector(VtkOptions* o, VtkVector vec) void vtkVector(VtkOptions* o, char* name, VtkVector vec) { - if (commIsMaster(&o->comm)) printf("Register vector %s\n", name); - - if (o->mode == UNIX) { - if (commIsMaster(&o->comm)) { - if (!isInitialized(o->fh)) return; - fprintf(o->fh, "VECTORS %s double\n", name); - writeVector(o, vec); - } - } else if (o->mode == MPI) { - } + printf("Register vector %s\n", name); + if (!isInitialized(o->fh)) return; + fprintf(o->fh, "VECTORS %s double\n", name); + writeVector(o, vec); } void vtkClose(VtkOptions* o) { - if (o->mode == UNIX) { - if (commIsMaster(&o->comm)) { - fclose(o->fh); - o->fh = NULL; - } - } else if (o->mode == MPI) { - } + fclose(o->fh); + o->fh = NULL; } diff --git a/BasicSolver/3D-mpi/src/vtkWriter.h b/BasicSolver/3D-mpi/src/vtkWriter.h index 7a91dbb..e93835c 100644 --- a/BasicSolver/3D-mpi/src/vtkWriter.h +++ b/BasicSolver/3D-mpi/src/vtkWriter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -12,13 +12,15 @@ #include "grid.h" typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat; -typedef enum VtkMode { UNIX = 0, MPI } VtkMode; typedef struct VtkOptions { - VtkFormat fmt; - VtkMode mode; - Grid grid; + Grid* grid; +#ifdef _VTK_WRITER_MPI + MPI_File fh; +#else FILE* fh; + VtkFormat fmt; +#endif // _VTK_WRITER_MPI Comm comm; } VtkOptions; diff --git a/BasicSolver/3D-seq/Makefile b/BasicSolver/3D-seq/Makefile index 57f99f4..65e6d8d 100644 --- a/BasicSolver/3D-seq/Makefile +++ b/BasicSolver/3D-seq/Makefile @@ -1,5 +1,5 @@ #======================================================================================= -# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. # All rights reserved. # Use of this source code is governed by a MIT-style # license that can be found in the LICENSE file. @@ -18,9 +18,10 @@ include $(MAKE_DIR)/include_$(TAG).mk INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) VPATH = $(SRC_DIR) -SRC = $(wildcard $(SRC_DIR)/*.c) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) @@ -39,6 +40,18 @@ $(BUILD_DIR)/%.s: %.c .PHONY: clean distclean tags info asm format +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.vtk + @rm -f *.png + +clean: vis_clean + clean: $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @@ -47,6 +60,8 @@ clean: distclean: clean $(info ===> DIST CLEAN) @rm -f $(TARGET) + @rm -f *.dat + @rm -f *.png info: $(info $(CFLAGS)) diff --git a/BasicSolver/3D-seq/canal.par b/BasicSolver/3D-seq/canal.par index 30e1f41..597c5b5 100644 --- a/BasicSolver/3D-seq/canal.par +++ b/BasicSolver/3D-seq/canal.par @@ -38,10 +38,17 @@ kmax 50 # number of interior cells in z-direction # Time Data: # --------- -te 100.0 # final time +te 60.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- diff --git a/BasicSolver/3D-seq/config.mk b/BasicSolver/3D-seq/config.mk index af5f1a0..2cf33f9 100644 --- a/BasicSolver/3D-seq/config.mk +++ b/BasicSolver/3D-seq/config.mk @@ -1,12 +1,12 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg +# Run in debug settings +DEBUG ?= false #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 OPTIONS += -DVERBOSE #OPTIONS += -DDEBUG -#OPTIONS += -DBOUNDCHECK -#OPTIONS += -DVERBOSE_AFFINITY -#OPTIONS += -DVERBOSE_DATASIZE -#OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/3D-seq/dcavity.par b/BasicSolver/3D-seq/dcavity.par index 6d1ef6d..b668f49 100644 --- a/BasicSolver/3D-seq/dcavity.par +++ b/BasicSolver/3D-seq/dcavity.par @@ -38,15 +38,23 @@ kmax 128 # number of interior cells in z-direction # Time Data: # --------- -te 2.0 # final time +te 10.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- itermax 1000 # maximal number of pressure iteration in one time step eps 0.001 # stopping tolerance for pressure iteration +rho 0.5 omg 1.7 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma #=============================================================================== diff --git a/BasicSolver/3D-seq/include_CLANG.mk b/BasicSolver/3D-seq/include_CLANG.mk index 17b26f3..a124053 100644 --- a/BasicSolver/3D-seq/include_CLANG.mk +++ b/BasicSolver/3D-seq/include_CLANG.mk @@ -2,16 +2,18 @@ CC = clang GCC = cc LINKER = $(CC) -ifeq ($(ENABLE_OPENMP),true) +ifeq ($(strip $(ENABLE_OPENMP)),true) OPENMP = -fopenmp #OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp LIBS = # -lomp endif +ifeq ($(strip $(DEBUG)),true) +CFLAGS = -O0 -g -std=c17 +else +CFLAGS = -O3 -std=c17 $(OPENMP) +endif VERSION = --version -# CFLAGS = -O3 -std=c17 $(OPENMP) -CFLAGS = -Ofast -std=c17 -Weverything -#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG LFLAGS = $(OPENMP) -lm -DEFINES = -D_GNU_SOURCE# -DDEBUG +DEFINES = -D_GNU_SOURCE INCLUDES = diff --git a/BasicSolver/3D-seq/residual.plot b/BasicSolver/3D-seq/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/BasicSolver/3D-seq/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/3D-seq/src/allocate.c b/BasicSolver/3D-seq/src/allocate.c index 6af6c7f..cf2efd6 100644 --- a/BasicSolver/3D-seq/src/allocate.c +++ b/BasicSolver/3D-seq/src/allocate.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. diff --git a/BasicSolver/3D-seq/src/allocate.h b/BasicSolver/3D-seq/src/allocate.h index 1537eb2..77f4ba0 100644 --- a/BasicSolver/3D-seq/src/allocate.h +++ b/BasicSolver/3D-seq/src/allocate.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. diff --git a/BasicSolver/3D-seq/src/solver.c b/BasicSolver/3D-seq/src/discretization.c similarity index 60% rename from BasicSolver/3D-seq/src/solver.c rename to BasicSolver/3D-seq/src/discretization.c index c741ece..8821e7e 100644 --- a/BasicSolver/3D-seq/src/solver.c +++ b/BasicSolver/3D-seq/src/discretization.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -7,132 +7,123 @@ #include #include #include -#include #include #include "allocate.h" +#include "discretization.h" #include "parameter.h" -#include "solver.h" #include "util.h" -#define P(i, j, k) p[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] -#define F(i, j, k) f[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] -#define G(i, j, k) g[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] -#define H(i, j, k) h[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] -#define U(i, j, k) u[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] -#define V(i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] -#define W(i, j, k) w[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] -#define RHS(i, j, k) rhs[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] - -static void printConfig(Solver* s) +static void printConfig(Discretization* d) { - printf("Parameters for #%s#\n", s->problem); + printf("Parameters for #%s#\n", d->problem); printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n", - s->bcLeft, - s->bcRight, - s->bcBottom, - s->bcTop, - s->bcFront, - s->bcBack); - printf("\tReynolds number: %.2f\n", s->re); - printf("\tGx Gy: %.2f %.2f %.2f\n", s->gx, s->gy, s->gz); + d->bcLeft, + d->bcRight, + d->bcBottom, + d->bcTop, + d->bcFront, + d->bcBack); + printf("\tReynolds number: %.2f\n", d->re); + printf("\tGx Gy: %.2f %.2f %.2f\n", d->gx, d->gy, d->gz); printf("Geometry data:\n"); printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", - s->grid.xlength, - s->grid.ylength, - s->grid.zlength); - printf("\tCells (x, y, z): %d, %d, %d\n", s->grid.imax, s->grid.jmax, s->grid.kmax); - printf("\tCell size (dx, dy, dz): %f, %f, %f\n", s->grid.dx, s->grid.dy, s->grid.dz); + d->grid.xlength, + d->grid.ylength, + d->grid.zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", d->grid.imax, d->grid.jmax, d->grid.kmax); + printf("\tCell size (dx, dy, dz): %f, %f, %f\n", d->grid.dx, d->grid.dy, d->grid.dz); printf("Timestep parameters:\n"); - printf("\tDefault stepsize: %.2f, Final time %.2f\n", s->dt, s->te); - printf("\tdt bound: %.6f\n", s->dtBound); - printf("\tTau factor: %.2f\n", s->tau); - printf("Iterative s parameters:\n"); - printf("\tMax iterations: %d\n", s->itermax); - printf("\tepsilon (stopping tolerance) : %f\n", s->eps); - printf("\tgamma factor: %f\n", s->gamma); - printf("\tomega (SOR relaxation): %f\n", s->omega); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te); + printf("\tdt bound: %.6f\n", d->dtBound); + printf("\tTau factor: %.2f\n", d->tau); + printf("Iterative parameters:\n"); + printf("\tepsilon (stopping tolerance) : %f\n", d->eps); + printf("\tgamma factor: %f\n", d->gamma); + printf("\tomega (SOR relaxation): %f\n", d->omega); } -void initSolver(Solver* s, Parameter* params) +void initDiscretization(Discretization* d, Parameter* p) { - s->problem = params->name; - s->bcLeft = params->bcLeft; - s->bcRight = params->bcRight; - s->bcBottom = params->bcBottom; - s->bcTop = params->bcTop; - s->bcFront = params->bcFront; - s->bcBack = params->bcBack; - s->grid.imax = params->imax; - s->grid.jmax = params->jmax; - s->grid.kmax = params->kmax; - s->grid.xlength = params->xlength; - s->grid.ylength = params->ylength; - s->grid.zlength = params->zlength; - s->grid.dx = params->xlength / params->imax; - s->grid.dy = params->ylength / params->jmax; - s->grid.dz = params->zlength / params->kmax; - s->eps = params->eps; - s->omega = params->omg; - s->itermax = params->itermax; - s->re = params->re; - s->gx = params->gx; - s->gy = params->gy; - s->gz = params->gz; - s->dt = params->dt; - s->te = params->te; - s->tau = params->tau; - s->gamma = params->gamma; + d->problem = p->name; + d->bcLeft = p->bcLeft; + d->bcRight = p->bcRight; + d->bcBottom = p->bcBottom; + d->bcTop = p->bcTop; + d->bcFront = p->bcFront; + d->bcBack = p->bcBack; - int imax = s->grid.imax; - int jmax = s->grid.jmax; - int kmax = s->grid.kmax; + d->grid.imax = p->imax; + d->grid.jmax = p->jmax; + d->grid.kmax = p->kmax; + d->grid.xlength = p->xlength; + d->grid.ylength = p->ylength; + d->grid.zlength = p->zlength; + d->grid.dx = p->xlength / p->imax; + d->grid.dy = p->ylength / p->jmax; + d->grid.dz = p->zlength / p->kmax; + + d->eps = p->eps; + d->omega = p->omg; + d->re = p->re; + d->gx = p->gx; + d->gy = p->gy; + d->gz = p->gz; + d->dt = p->dt; + d->te = p->te; + d->tau = p->tau; + d->gamma = p->gamma; + + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; size_t bytesize = (imax + 2) * (jmax + 2) * (kmax + 2) * sizeof(double); - s->u = allocate(64, bytesize); - s->v = allocate(64, bytesize); - s->w = allocate(64, bytesize); - s->p = allocate(64, bytesize); - s->rhs = allocate(64, bytesize); - s->f = allocate(64, bytesize); - s->g = allocate(64, bytesize); - s->h = allocate(64, bytesize); + d->u = allocate(64, bytesize); + d->v = allocate(64, bytesize); + d->w = allocate(64, bytesize); + d->p = allocate(64, bytesize); + d->rhs = allocate(64, bytesize); + d->f = allocate(64, bytesize); + d->g = allocate(64, bytesize); + d->h = allocate(64, bytesize); for (int i = 0; i < (imax + 2) * (jmax + 2) * (kmax + 2); i++) { - s->u[i] = params->u_init; - s->v[i] = params->v_init; - s->w[i] = params->w_init; - s->p[i] = params->p_init; - s->rhs[i] = 0.0; - s->f[i] = 0.0; - s->g[i] = 0.0; - s->h[i] = 0.0; + d->u[i] = p->u_init; + d->v[i] = p->v_init; + d->w[i] = p->w_init; + d->p[i] = p->p_init; + d->rhs[i] = 0.0; + d->f[i] = 0.0; + d->g[i] = 0.0; + d->h[i] = 0.0; } - double dx = s->grid.dx; - double dy = s->grid.dy; - double dz = s->grid.dz; + double dx = d->grid.dx; + double dy = d->grid.dy; + double dz = d->grid.dz; double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz); - s->dtBound = 0.5 * s->re * 1.0 / invSqrSum; + d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; #ifdef VERBOSE - printConfig(s); + printConfig(d); #endif /* VERBOSE */ } -void computeRHS(Solver* s) +void computeRHS(Discretization* d) { - int imax = s->grid.imax; - int jmax = s->grid.jmax; - int kmax = s->grid.kmax; - double idx = 1.0 / s->grid.dx; - double idy = 1.0 / s->grid.dy; - double idz = 1.0 / s->grid.dz; - double idt = 1.0 / s->dt; - double* rhs = s->rhs; - double* f = s->f; - double* g = s->g; - double* h = s->h; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + double idx = 1.0 / d->grid.dx; + double idy = 1.0 / d->grid.dy; + double idz = 1.0 / d->grid.dz; + double idt = 1.0 / d->dt; + + double* rhs = d->rhs; + double* f = d->f; + double* g = d->g; + double* h = d->h; for (int k = 1; k < kmax + 1; k++) { for (int j = 1; j < jmax + 1; j++) { @@ -146,169 +137,9 @@ void computeRHS(Solver* s) } } -void solve(Solver* s) +static double maxElement(Discretization* d, double* m) { - int imax = s->grid.imax; - int jmax = s->grid.jmax; - int kmax = s->grid.kmax; - double eps = s->eps; - int itermax = s->itermax; - double dx2 = s->grid.dx * s->grid.dx; - double dy2 = s->grid.dy * s->grid.dy; - double dz2 = s->grid.dz * s->grid.dz; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double idz2 = 1.0 / dz2; - double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / - (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); - double* p = s->p; - double* rhs = s->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - - for (int k = 1; k < kmax + 1; k++) { - for (int j = 1; j < jmax + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - - double r = RHS(i, j, k) - - ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * - idx2 + - (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * - idy2 + - (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * - idz2); - - P(i, j, k) -= (factor * r); - res += (r * r); - } - } - } - - for (int j = 1; j < jmax + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - P(i, j, 0) = P(i, j, 1); - P(i, j, kmax + 1) = P(i, j, kmax); - } - } - - for (int k = 1; k < kmax + 1; k++) { - for (int i = 1; i < imax + 1; i++) { - P(i, 0, k) = P(i, 1, k); - P(i, jmax + 1, k) = P(i, jmax, k); - } - } - - for (int k = 1; k < kmax + 1; k++) { - for (int j = 1; j < jmax + 1; j++) { - P(0, j, k) = P(1, j, k); - P(imax + 1, j, k) = P(imax, j, k); - } - } - - res = res / (double)(imax * jmax * kmax); -#ifdef DEBUG - printf("%d Residuum: %e\n", it, res); -#endif - it++; - } - -#ifdef VERBOSE - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); -#endif -} - -void solveRB(Solver* s) -{ - int imax = s->grid.imax; - int jmax = s->grid.jmax; - int kmax = s->grid.kmax; - double eps = s->eps; - int itermax = s->itermax; - double dx2 = s->grid.dx * s->grid.dx; - double dy2 = s->grid.dy * s->grid.dy; - double dz2 = s->grid.dz * s->grid.dz; - double idx2 = 1.0 / dx2; - double idy2 = 1.0 / dy2; - double idz2 = 1.0 / dz2; - double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / - (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); - double* p = s->p; - double* rhs = s->rhs; - double epssq = eps * eps; - int it = 0; - double res = 1.0; - int pass, ksw, jsw, isw; - - while ((res >= epssq) && (it < itermax)) { - res = 0.0; - ksw = 1; - - for (pass = 0; pass < 2; pass++) { - jsw = ksw; - - for (int k = 1; k < kmax + 1; k++) { - isw = jsw; - for (int j = 1; j < jmax + 1; j++) { - for (int i = isw; i < imax + 1; i += 2) { - - double r = - RHS(i, j, k) - - ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + - (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * - idy2 + - (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * - idz2); - - P(i, j, k) -= (factor * r); - res += (r * r); - } - isw = 3 - isw; - } - jsw = 3 - jsw; - } - ksw = 3 - ksw; - } - - for (int j = 1; j < jmax + 1; j++) { - for (int i = 1; i < imax + 1; i++) { - P(i, j, 0) = P(i, j, 1); - P(i, j, kmax + 1) = P(i, j, kmax); - } - } - - for (int k = 1; k < kmax + 1; k++) { - for (int i = 1; i < imax + 1; i++) { - P(i, 0, k) = P(i, 1, k); - P(i, jmax + 1, k) = P(i, jmax, k); - } - } - - for (int k = 1; k < kmax + 1; k++) { - for (int j = 1; j < jmax + 1; j++) { - P(0, j, k) = P(1, j, k); - P(imax + 1, j, k) = P(imax, j, k); - } - } - - res = res / (double)(imax * jmax * kmax); -#ifdef DEBUG - printf("%d Residuum: %e\n", it, res); -#endif - it++; - } - -#ifdef VERBOSE - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); -#endif -} - -static double maxElement(Solver* s, double* m) -{ - int size = (s->grid.imax + 2) * (s->grid.jmax + 2) * (s->grid.kmax + 2); + int size = (d->grid.imax + 2) * (d->grid.jmax + 2) * (d->grid.kmax + 2); double maxval = DBL_MIN; for (int i = 0; i < size; i++) { @@ -318,10 +149,10 @@ static double maxElement(Solver* s, double* m) return maxval; } -void normalizePressure(Solver* s) +void normalizePressure(Discretization* d) { - int size = (s->grid.imax + 2) * (s->grid.jmax + 2) * (s->grid.kmax + 2); - double* p = s->p; + int size = (d->grid.imax + 2) * (d->grid.jmax + 2) * (d->grid.kmax + 2); + double* p = d->p; double avgP = 0.0; for (int i = 0; i < size; i++) { @@ -334,15 +165,16 @@ void normalizePressure(Solver* s) } } -void computeTimestep(Solver* s) +void computeTimestep(Discretization* d) { - double dt = s->dtBound; - double dx = s->grid.dx; - double dy = s->grid.dy; - double dz = s->grid.dz; - double umax = maxElement(s, s->u); - double vmax = maxElement(s, s->v); - double wmax = maxElement(s, s->w); + double dt = d->dtBound; + double dx = d->grid.dx; + double dy = d->grid.dy; + double dz = d->grid.dz; + + double umax = maxElement(d, d->u); + double vmax = maxElement(d, d->v); + double wmax = maxElement(d, d->w); if (umax > 0) { dt = (dt > dx / umax) ? dx / umax : dt; @@ -354,20 +186,20 @@ void computeTimestep(Solver* s) dt = (dt > dz / wmax) ? dz / wmax : dt; } - s->dt = dt * s->tau; + d->dt = dt * d->tau; } -void setBoundaryConditions(Solver* s) +void setBoundaryConditions(Discretization* d) { - int imax = s->grid.imax; - int jmax = s->grid.jmax; - int kmax = s->grid.kmax; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; - double* u = s->u; - double* v = s->v; - double* w = s->w; + double* u = d->u; + double* v = d->v; + double* w = d->w; - switch (s->bcTop) { + switch (d->bcTop) { case NOSLIP: for (int k = 1; k < kmax + 1; k++) { for (int i = 1; i < imax + 1; i++) { @@ -399,7 +231,7 @@ void setBoundaryConditions(Solver* s) break; } - switch (s->bcBottom) { + switch (d->bcBottom) { case NOSLIP: for (int k = 1; k < kmax + 1; k++) { for (int i = 1; i < imax + 1; i++) { @@ -431,7 +263,7 @@ void setBoundaryConditions(Solver* s) break; } - switch (s->bcLeft) { + switch (d->bcLeft) { case NOSLIP: for (int k = 1; k < kmax + 1; k++) { for (int j = 1; j < jmax + 1; j++) { @@ -463,7 +295,7 @@ void setBoundaryConditions(Solver* s) break; } - switch (s->bcRight) { + switch (d->bcRight) { case NOSLIP: for (int k = 1; k < kmax + 1; k++) { for (int j = 1; j < jmax + 1; j++) { @@ -495,7 +327,7 @@ void setBoundaryConditions(Solver* s) break; } - switch (s->bcFront) { + switch (d->bcFront) { case NOSLIP: for (int j = 1; j < jmax + 1; j++) { for (int i = 1; i < imax + 1; i++) { @@ -527,7 +359,7 @@ void setBoundaryConditions(Solver* s) break; } - switch (s->bcBack) { + switch (d->bcBack) { case NOSLIP: for (int j = 1; j < jmax + 1; j++) { for (int i = 1; i < imax + 1; i++) { @@ -560,23 +392,23 @@ void setBoundaryConditions(Solver* s) } } -void setSpecialBoundaryCondition(Solver* s) +void setSpecialBoundaryCondition(Discretization* d) { - int imax = s->grid.imax; - int jmax = s->grid.jmax; - int kmax = s->grid.kmax; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; - double mDy = s->grid.dy; - double* u = s->u; + double mDy = d->grid.dy; + double* u = d->u; - if (strcmp(s->problem, "dcavity") == 0) { + if (strcmp(d->problem, "dcavity") == 0) { for (int k = 1; k < kmax; k++) { for (int i = 1; i < imax; i++) { U(i, jmax + 1, k) = 2.0 - U(i, jmax, k); } } - } else if (strcmp(s->problem, "canal") == 0) { - double ylength = s->grid.ylength; + } else if (strcmp(d->problem, "canal") == 0) { + double ylength = d->grid.ylength; double y; for (int k = 1; k < kmax + 1; k++) { @@ -588,29 +420,29 @@ void setSpecialBoundaryCondition(Solver* s) } } -void computeFG(Solver* s) +void computeFG(Discretization* d) { - int imax = s->grid.imax; - int jmax = s->grid.jmax; - int kmax = s->grid.kmax; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; - double* u = s->u; - double* v = s->v; - double* w = s->w; - double* f = s->f; - double* g = s->g; - double* h = s->h; + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* f = d->f; + double* g = d->g; + double* h = d->h; - double gx = s->gx; - double gy = s->gy; - double gz = s->gz; - double gamma = s->gamma; - double dt = s->dt; + double gx = d->gx; + double gy = d->gy; + double gz = d->gz; + double dt = d->dt; - double inverseRe = 1.0 / s->re; - double inverseDx = 1.0 / s->grid.dx; - double inverseDy = 1.0 / s->grid.dy; - double inverseDz = 1.0 / s->grid.dz; + double gamma = d->gamma; + double inverseRe = 1.0 / d->re; + double inverseDx = 1.0 / d->grid.dx; + double inverseDy = 1.0 / d->grid.dy; + double inverseDz = 1.0 / d->grid.dz; double du2dx, dv2dy, dw2dz; double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz; double du2dx2, du2dy2, du2dz2; @@ -777,23 +609,23 @@ void computeFG(Solver* s) } } -void adaptUV(Solver* s) +void adaptUV(Discretization* d) { - int imax = s->grid.imax; - int jmax = s->grid.jmax; - int kmax = s->grid.kmax; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; - double* p = s->p; - double* u = s->u; - double* v = s->v; - double* w = s->w; - double* f = s->f; - double* g = s->g; - double* h = s->h; + double* p = d->p; + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* f = d->f; + double* g = d->g; + double* h = d->h; - double factorX = s->dt / s->grid.dx; - double factorY = s->dt / s->grid.dy; - double factorZ = s->dt / s->grid.dz; + double factorX = d->dt / d->grid.dx; + double factorY = d->dt / d->grid.dy; + double factorZ = d->dt / d->grid.dz; for (int k = 1; k < kmax + 1; k++) { for (int j = 1; j < jmax + 1; j++) { diff --git a/BasicSolver/3D-seq/src/discretization.h b/BasicSolver/3D-seq/src/discretization.h new file mode 100644 index 0000000..17f97fd --- /dev/null +++ b/BasicSolver/3D-seq/src/discretization.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ + +#include "grid.h" +#include "parameter.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g, *h; + double *u, *v, *w; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy, gz; + /* time stepping */ + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; +} Discretization; + +extern void initDiscretization(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +#endif diff --git a/BasicSolver/3D-seq/src/grid.h b/BasicSolver/3D-seq/src/grid.h index c963429..0689ebc 100644 --- a/BasicSolver/3D-seq/src/grid.h +++ b/BasicSolver/3D-seq/src/grid.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. diff --git a/BasicSolver/3D-seq/src/main.c b/BasicSolver/3D-seq/src/main.c index 73f9df2..f453fdd 100644 --- a/BasicSolver/3D-seq/src/main.c +++ b/BasicSolver/3D-seq/src/main.c @@ -1,16 +1,15 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ -#include -#include #include #include #include #include "allocate.h" +#include "discretization.h" #include "parameter.h" #include "progress.h" #include "solver.h" @@ -19,7 +18,8 @@ #define G(v, i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] -static void createBulkArrays(Solver* s, double* pg, double* ug, double* vg, double* wg) +static void createBulkArrays( + Discretization* s, double* pg, double* ug, double* vg, double* wg) { int imax = s->grid.imax; int jmax = s->grid.jmax; @@ -68,63 +68,79 @@ static void createBulkArrays(Solver* s, double* pg, double* ug, double* vg, doub int main(int argc, char** argv) { double timeStart, timeStop; - Parameter params; + Parameter p; + Discretization d; Solver s; - initParameter(¶ms); + initParameter(&p); + + FILE* fp; + fp = initResidualWriter(); if (argc != 2) { printf("Usage: %s \n", argv[0]); exit(EXIT_SUCCESS); } - readParameter(¶ms, argv[1]); - printParameter(¶ms); - initSolver(&s, ¶ms); + readParameter(&p, argv[1]); + printParameter(&p); + initDiscretization(&d, &p); + initSolver(&s, &d, &p); #ifndef VERBOSE - initProgress(s.te); + initProgress(d.te); #endif - double tau = s.tau; - double te = s.te; + double tau = d.tau; + double te = d.te; double t = 0.0; + int nt = 0; + double res = 0.0; timeStart = getTimeStamp(); while (t <= te) { - if (tau > 0.0) computeTimestep(&s); - setBoundaryConditions(&s); - setSpecialBoundaryCondition(&s); - computeFG(&s); - computeRHS(&s); - solveRB(&s); - adaptUV(&s); - t += s.dt; + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + computeFG(&d); + computeRHS(&d); + if (nt % 100 == 0) normalizePressure(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + + writeResidual(fp, t, res); + + t += d.dt; + nt++; #ifdef VERBOSE - printf("TIME %f , TIMESTEP %f\n", t, s.dt); + printf("TIME %f , TIMESTEP %f\n", t, d.dt); #else printProgress(t); #endif } timeStop = getTimeStamp(); -#ifndef VERBOSE stopProgress(); -#endif printf("Solution took %.2fs\n", timeStop - timeStart); + timeStart = getTimeStamp(); double *pg, *ug, *vg, *wg; - size_t bytesize = (size_t)(s.grid.imax * s.grid.jmax * s.grid.kmax) * sizeof(double); + size_t bytesize = (size_t)(d.grid.imax * d.grid.jmax * d.grid.kmax) * sizeof(double); pg = allocate(64, bytesize); ug = allocate(64, bytesize); vg = allocate(64, bytesize); wg = allocate(64, bytesize); - - createBulkArrays(&s, pg, ug, vg, wg); - VtkOptions opts = { .grid = s.grid }; - vtkOpen(&opts, s.problem); + + fclose(fp); + createBulkArrays(&d, pg, ug, vg, wg); + VtkOptions opts = { .grid = d.grid }; + vtkOpen(&opts, d.problem); vtkScalar(&opts, "pressure", pg); vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); vtkClose(&opts); + + timeStop = getTimeStamp(); + printf("Result output took %.2fs\n", timeStop - timeStart); + return EXIT_SUCCESS; } diff --git a/BasicSolver/3D-seq/src/parameter.c b/BasicSolver/3D-seq/src/parameter.c index d4ea5b6..13a2d8a 100644 --- a/BasicSolver/3D-seq/src/parameter.c +++ b/BasicSolver/3D-seq/src/parameter.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -14,18 +14,21 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->zlength = 1.0; - param->imax = 100; - param->jmax = 100; - param->kmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.7; - param->re = 100.0; - param->gamma = 0.9; - param->tau = 0.5; + param->xlength = 1.0; + param->ylength = 1.0; + param->zlength = 1.0; + param->imax = 100; + param->jmax = 100; + param->kmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) @@ -65,6 +68,7 @@ void readParameter(Parameter* param, const char* filename) PARSE_INT(jmax); PARSE_INT(kmax); PARSE_INT(itermax); + PARSE_INT(levels); PARSE_REAL(eps); PARSE_REAL(omg); PARSE_REAL(re); @@ -123,4 +127,5 @@ void printParameter(Parameter* param) printf("\tepsilon (stopping tolerance) : %f\n", param->eps); printf("\tgamma (stopping tolerance) : %f\n", param->gamma); printf("\tomega (SOR relaxation): %f\n", param->omg); + printf("\tMultiGrid levels : %d\n", param->levels); } diff --git a/BasicSolver/3D-seq/src/parameter.h b/BasicSolver/3D-seq/src/parameter.h index 6dddf5f..eb2b753 100644 --- a/BasicSolver/3D-seq/src/parameter.h +++ b/BasicSolver/3D-seq/src/parameter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -10,14 +10,15 @@ typedef struct { int imax, jmax, kmax; double xlength, ylength, zlength; - int itermax; - double eps, omg; + int itermax, levels; + double eps, omg, rho; double re, tau, gamma; double te, dt; double gx, gy, gz; char* name; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; double u_init, v_init, w_init, p_init; + int presmooth, postsmooth; } Parameter; void initParameter(Parameter*); diff --git a/BasicSolver/3D-seq/src/progress.c b/BasicSolver/3D-seq/src/progress.c index a9b82bd..3523426 100644 --- a/BasicSolver/3D-seq/src/progress.c +++ b/BasicSolver/3D-seq/src/progress.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -49,3 +49,22 @@ void stopProgress() printf("\n"); fflush(stdout); } + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} \ No newline at end of file diff --git a/BasicSolver/3D-seq/src/progress.h b/BasicSolver/3D-seq/src/progress.h index 4d02cdb..240c279 100644 --- a/BasicSolver/3D-seq/src/progress.h +++ b/BasicSolver/3D-seq/src/progress.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -10,5 +10,6 @@ extern void initProgress(double); extern void printProgress(double); extern void stopProgress(void); - +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/BasicSolver/3D-seq/src/solver-mg.c b/BasicSolver/3D-seq/src/solver-mg.c new file mode 100644 index 0000000..743292c --- /dev/null +++ b/BasicSolver/3D-seq/src/solver-mg.c @@ -0,0 +1,304 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +#define S(i, j, k) s[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define E(i, j, k) e[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define R(i, j, k) r[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define OLD(i, j, k) old[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +static void restrictMG(Solver* s, int level, int imax, int jmax, int kmax) +{ + double* r = s->r[level + 1]; + double* old = s->r[level]; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; ++i) { + R(i, j, k) = (OLD(2 * i - 1, 2 * j - 1, 2 * k) + + OLD(2 * i, 2 * j - 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k) + + OLD(2 * i - 1, 2 * j, 2 * k) * 2 + + OLD(2 * i, 2 * j, 2 * k) * 8 + + OLD(2 * i + 1, 2 * j, 2 * k) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k) + + OLD(2 * i, 2 * j + 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i, 2 * j - 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i - 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i, 2 * j, 2 * k - 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k - 1) + + OLD(2 * i, 2 * j + 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k - 1) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i, 2 * j - 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i - 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i, 2 * j, 2 * k + 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k + 1) + + OLD(2 * i, 2 * j + 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k + 1)) / + 64.0; + } + } + } +} + +static void prolongate(Solver* s, int level, int imax, int jmax, int kmax) +{ + double* old = s->r[level + 1]; + double* e = s->r[level]; + + for (int k = 2; k < kmax + 1; k += 2) { + for (int j = 2; j < jmax + 1; j += 2) { + for (int i = 2; i < imax + 1; i += 2) { + E(i, j, k) = OLD(i / 2, j / 2, k / 2); + } + } + } +} + +static void correct(Solver* s, double* p, int level, int imax, int jmax, int kmax) +{ + double* e = s->e[level]; + + for (int k = 1; k < kmax + 1; ++k) { + for (int j = 1; j < jmax + 1; ++j) { + for (int i = 1; i < imax + 1; ++i) { + P(i, j, k) += E(i, j, k); + } + } + } +} + +static void setBoundaryCondition(double* p, int imax, int jmax, int kmax) +{ + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } +} + +static double smooth( + Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax) +{ + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + P(i, j, k) -= + factor * + (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +} + +static double calculateResidual( + Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax) +{ + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + R(i, + j, + k) = (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * + idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + + res += (R(i, j, k) * R(i, j, k)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + res = res / (double)(imax * jmax * kmax); + + return res; +} + +static double multiGrid( + Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax) +{ + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, imax, jmax, kmax); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, imax, jmax, kmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); + } + + res = calculateResidual(s, p, rhs, level, imax, jmax, kmax); + + // restrict + restrictMG(s, level, imax, jmax, kmax); + + // MGSolver on residual and error. + multiGrid(s, + s->e[level + 1], + s->r[level + 1], + level + 1, + imax / 2, + jmax / 2, + kmax / 2); + + // prolongate + prolongate(s, level, imax, jmax, kmax); + + // correct p on finer level using residual + correct(s, p, level, imax, jmax, kmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, imax, jmax, kmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + int levels = s->levels; + printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * (kmax + 2); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size * sizeof(double)); + s->e[j] = allocate(64, size * sizeof(double)); + + for (size_t i = 0; i < size; i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax, s->grid->kmax); + +#ifdef VERBOSE + printf("Residuum: %.6f\n", res); +#endif + +return res; +} diff --git a/BasicSolver/3D-seq/src/solver-rb.c b/BasicSolver/3D-seq/src/solver-rb.c new file mode 100644 index 0000000..6332cce --- /dev/null +++ b/BasicSolver/3D-seq/src/solver-rb.c @@ -0,0 +1,101 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->itermax = p->itermax; + s->eps = p->eps; + s->omega = p->omg; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double epssq = eps * eps; + int it = 0; + double res = 1.0; + int pass, ksw, jsw, isw; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + double r = + RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2); + + P(i, j, k) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } + + res = res / (double)(imax * jmax * kmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif + +return res; +} diff --git a/BasicSolver/3D-seq/src/solver.h b/BasicSolver/3D-seq/src/solver.h index 4cec259..e5f1b38 100644 --- a/BasicSolver/3D-seq/src/solver.h +++ b/BasicSolver/3D-seq/src/solver.h @@ -1,45 +1,27 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ #ifndef __SOLVER_H_ #define __SOLVER_H_ - +#include "discretization.h" #include "grid.h" #include "parameter.h" -enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; - typedef struct { /* geometry and grid information */ - Grid grid; - /* arrays */ - double *p, *rhs; - double *f, *g, *h; - double *u, *v, *w; + Grid* grid; /* parameters */ - double eps, omega; - double re, tau, gamma; - double gx, gy, gz; - /* time stepping */ + double eps, omega, rho; int itermax; - double dt, te; - double dtBound; - char* problem; - int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + int levels; + double **r, **e; + int presmooth, postsmooth; } Solver; -extern void initSolver(Solver*, Parameter*); -extern void computeRHS(Solver*); -extern void solve(Solver*); -extern void solveRB(Solver*); -extern void normalizePressure(Solver*); -extern void computeTimestep(Solver*); -extern void setBoundaryConditions(Solver*); -extern void setSpecialBoundaryCondition(Solver*); -extern void computeFG(Solver*); -extern void adaptUV(Solver*); -extern void writeResult(Solver*); +extern void initSolver(Solver*, Discretization*, Parameter*); +extern double solve(Solver*, double*, double*); + #endif diff --git a/BasicSolver/3D-seq/src/timing.c b/BasicSolver/3D-seq/src/timing.c index e578f18..78b01c4 100644 --- a/BasicSolver/3D-seq/src/timing.c +++ b/BasicSolver/3D-seq/src/timing.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. diff --git a/BasicSolver/3D-seq/src/timing.h b/BasicSolver/3D-seq/src/timing.h index 58fb5ac..ed05a8c 100644 --- a/BasicSolver/3D-seq/src/timing.h +++ b/BasicSolver/3D-seq/src/timing.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. diff --git a/BasicSolver/3D-seq/src/util.h b/BasicSolver/3D-seq/src/util.h index 657b009..1131237 100644 --- a/BasicSolver/3D-seq/src/util.h +++ b/BasicSolver/3D-seq/src/util.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -19,4 +19,13 @@ #define ABS(a) ((a) >= 0 ? (a) : -(a)) #endif +#define P(i, j, k) p[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define F(i, j, k) f[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define G(i, j, k) g[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define H(i, j, k) h[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define U(i, j, k) u[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define V(i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define W(i, j, k) w[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define RHS(i, j, k) rhs[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + #endif // __UTIL_H_ diff --git a/BasicSolver/3D-seq/src/vtkWriter.c b/BasicSolver/3D-seq/src/vtkWriter.c index 2f7e8c5..2b8f92f 100644 --- a/BasicSolver/3D-seq/src/vtkWriter.c +++ b/BasicSolver/3D-seq/src/vtkWriter.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -69,7 +69,7 @@ void vtkScalar(VtkOptions* o, char* name, double* s) printf("vtkWriter not initialize! Call vtkOpen first!\n"); exit(EXIT_FAILURE); } - fprintf(o->fh, "SCALARS %s float 1\n", name); + fprintf(o->fh, "SCALARS %s float\n", name); fprintf(o->fh, "LOOKUP_TABLE default\n"); for (int k = 0; k < kmax; k++) { diff --git a/BasicSolver/3D-seq/src/vtkWriter.h b/BasicSolver/3D-seq/src/vtkWriter.h index d166f62..9168e57 100644 --- a/BasicSolver/3D-seq/src/vtkWriter.h +++ b/BasicSolver/3D-seq/src/vtkWriter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. diff --git a/BasicSolver/README.md b/BasicSolver/README.md index 036e49f..ccc1e05 100644 --- a/BasicSolver/README.md +++ b/BasicSolver/README.md @@ -1,54 +1,53 @@ -# Introduction +# BasicSolver Variants + +## Introduction + This folder contains variants of the NuSiF basic solver. The basic solver does not allow obstacles within the domain. All basic solver variants include two test cases for validation: + * dcavity - Lid driven cavity * canal - Channel flow -# 2D solver variants -## Sequential solver (2D-seq) +## 2D solver variants + +### Sequential solver (2D-seq) + This is the basic sequential version. Gnuplot result visualization. -## Sequential solver with particle tracing (2D-seq-pt) -This version adds particle tracing and streak lines to the sequential basic solver. -Gnuplot result visualization. +### MPI parallel solver (2D-mpi) -## Simple MPI parallel solver (2D-mpi-v1) The simplest possible MPI parallelization with domain decomposition in one direction and communication just based on simple send and recv calls. Gnuplot result visualization. -## MPI parallel solver with 2D domain decomposition (2D-mpi-v2) A MPI parallelization with two-dimensional domain decomposition using MPI virtual topologies. Gnuplot result visualization. -## MPI parallel solver using MPI-3 neighborhood collectives (2D-mpi-v3) A MPI parallelization with two-dimensional domain decomposition using neighborhood collective call instead of send and recv calls. Gnuplot result visualization. -## Refactored MPI parallel solver (2D-mpi) The final version of the 2D MPI parallel solver. All MPI calls are contained in a single communication module. The rest of the code does not depend on MPI. This version is prepared to also compile and run without MPI. VTK result visualization. -# 3D solver variants +## 3D solver variants + +### Sequential solver (3D-seq) -## Sequential solver (3D-seq) This is the basic sequential version. VTK result visualization. -## MPI parallel solver (3D-mpi) +### MPI parallel solver (3D-mpi) + A MPI parallel solver with 3D domain decomposition using MPI virtual topologies and neighborhood collectives. All MPI calls are contained in a single communication module. The rest of the code does not depend on MPI. This version is prepared to also compile and run without MPI. VTK result visualization. -## MPI parallel solver with MPI-IO (3D-mpi-io) Identical to the 3D-MPI variant but using MPI-IO for VTK result file output. - - diff --git a/BasicSolver/2D-mpi-v3/Makefile b/EnhancedSolver/2D-mpi/Makefile similarity index 71% rename from BasicSolver/2D-mpi-v3/Makefile rename to EnhancedSolver/2D-mpi/Makefile index 57f99f4..22ed824 100644 --- a/BasicSolver/2D-mpi-v3/Makefile +++ b/EnhancedSolver/2D-mpi/Makefile @@ -1,5 +1,5 @@ #======================================================================================= -# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. # All rights reserved. # Use of this source code is governed by a MIT-style # license that can be found in the LICENSE file. @@ -18,9 +18,11 @@ include $(MAKE_DIR)/include_$(TAG).mk INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) VPATH = $(SRC_DIR) -SRC = $(wildcard $(SRC_DIR)/*.c) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/comm-$(COMM_TYPE).o +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) @@ -37,9 +39,22 @@ $(BUILD_DIR)/%.s: %.c $(info ===> GENERATE ASM $@) $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ -.PHONY: clean distclean tags info asm format +.PHONY: clean distclean vis_clean vis tags info asm format -clean: +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='pressure.dat'" ./surface.plot + @gnuplot -e "filename='velocity.dat'" ./vector.plot + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.png + @rm -f ./vis_files/*.dat + @rm -f ./vis_files/*.gif + +clean: vis_clean $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @rm -f tags @@ -47,6 +62,8 @@ clean: distclean: clean $(info ===> DIST CLEAN) @rm -f $(TARGET) + @rm -f *.dat + @rm -f *.png info: $(info $(CFLAGS)) diff --git a/BasicSolver/2D-mpi-v1/README.md b/EnhancedSolver/2D-mpi/README.md similarity index 100% rename from BasicSolver/2D-mpi-v1/README.md rename to EnhancedSolver/2D-mpi/README.md diff --git a/EnhancedSolver/2D-mpi/backstep.par b/EnhancedSolver/2D-mpi/backstep.par new file mode 100644 index 0000000..7063358 --- /dev/null +++ b/EnhancedSolver/2D-mpi/backstep.par @@ -0,0 +1,79 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name backstep # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 36000.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 1.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 7.0 # domain size in x-direction +ylength 1.5 # domain size in y-direction +imax 200 # number of interior cells in x-direction +jmax 40 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 60.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 55 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 500 +startTime 0 #if you want to see particles trapped in recirculation zone, startTime should be set to 0 +injectTimePeriod 1.0 +writeTimePeriod 0.5 + +x1 0.0 +y1 0.5 +x2 0.0 +y2 1.5 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 1 +xCenter 0.0 +yCenter 0.0 +xRectLength 2.0 +yRectLength 1.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/BasicSolver/2D-mpi-v2/canal.par b/EnhancedSolver/2D-mpi/canal.par similarity index 56% rename from BasicSolver/2D-mpi-v2/canal.par rename to EnhancedSolver/2D-mpi/canal.par index 80dfa19..7642ace 100644 --- a/BasicSolver/2D-mpi-v2/canal.par +++ b/EnhancedSolver/2D-mpi/canal.par @@ -7,10 +7,10 @@ name canal # name of flow setup -bcN 1 # flags for boundary conditions -bcE 3 # 1 = no-slip 3 = outflow -bcS 1 # 2 = free-slip 4 = periodic -bcW 3 # +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # gx 0.0 # Body forces (e.g. gravity) gy 0.0 # @@ -27,20 +27,52 @@ p_init 0.0 # initial value for pressure xlength 30.0 # domain size in x-direction ylength 4.0 # domain size in y-direction imax 200 # number of interior cells in x-direction -jmax 50 # number of interior cells in y-direction +jmax 40 # number of interior cells in y-direction # Time Data: # --------- -te 100.0 # final time +te 60.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + # Pressure Iteration Data: # ----------------------- itermax 500 # maximal number of pressure iteration in one time step eps 0.00001 # stopping tolerance for pressure iteration -omg 1.8 # relaxation parameter for SOR iteration +omg 1.7 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 60 +startTime 10.0 +injectTimePeriod 4.0 +writeTimePeriod 1.0 + +x1 1.0 +y1 0.0 +x2 1.0 +y2 4.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 10.0 +yCenter 2 +xRectLength 6.0 +yRectLength 1.0 +circleRadius 1.0 + #=============================================================================== diff --git a/BasicSolver/2D-mpi-v3/config.mk b/EnhancedSolver/2D-mpi/config.mk similarity index 58% rename from BasicSolver/2D-mpi-v3/config.mk rename to EnhancedSolver/2D-mpi/config.mk index 496668e..0b7938e 100644 --- a/BasicSolver/2D-mpi-v3/config.mk +++ b/EnhancedSolver/2D-mpi/config.mk @@ -1,10 +1,17 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC +# Supported: true, false +ENABLE_MPI ?= true ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg +# Run in debug settings ?= mg +COMM_TYPE ?= v3 #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 OPTIONS += -DVERBOSE +# OPTIONS += -DTEST #OPTIONS += -DVERBOSE_AFFINITY #OPTIONS += -DVERBOSE_DATASIZE #OPTIONS += -DVERBOSE_TIMER diff --git a/BasicSolver/2D-seq-pt/dcavity.par b/EnhancedSolver/2D-mpi/dcavity.par similarity index 60% rename from BasicSolver/2D-seq-pt/dcavity.par rename to EnhancedSolver/2D-mpi/dcavity.par index 4241393..bc91616 100644 --- a/BasicSolver/2D-seq-pt/dcavity.par +++ b/EnhancedSolver/2D-mpi/dcavity.par @@ -17,7 +17,7 @@ gy 0.0 # re 10.0 # Reynolds number -u_init 0.0 # initial value for velocity in x-direction +u_init 1.0 # initial value for velocity in x-direction v_init 0.0 # initial value for velocity in y-direction p_init 0.0 # initial value for pressure @@ -26,8 +26,8 @@ p_init 0.0 # initial value for pressure xlength 1.0 # domain size in x-direction ylength 1.0 # domain size in y-direction -imax 40 # number of interior cells in x-direction -jmax 40 # number of interior cells in y-direction +imax 128 # number of interior cells in x-direction +jmax 128 # number of interior cells in y-direction # Time Data: # --------- @@ -41,6 +41,39 @@ tau 0.5 # safety factor for time stepsize control (<0 constant delt) itermax 1000 # maximal number of pressure iteration in one time step eps 0.001 # stopping tolerance for pressure iteration +rho 0.5 omg 1.7 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 2.0 +injectTimePeriod 0.5 +writeTimePeriod 0.2 + +x1 0.1 +y1 0.9 +x2 0.9 +y2 0.9 + + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 0.5 +yCenter 0.5 +xRectLength 0.5 +yRectLength 0.5 +circleRadius 0.5 #=============================================================================== diff --git a/BasicSolver/3D-mpi-io/include_CLANG.mk b/EnhancedSolver/2D-mpi/include_CLANG.mk similarity index 63% rename from BasicSolver/3D-mpi-io/include_CLANG.mk rename to EnhancedSolver/2D-mpi/include_CLANG.mk index 889fa93..3641cad 100644 --- a/BasicSolver/3D-mpi-io/include_CLANG.mk +++ b/EnhancedSolver/2D-mpi/include_CLANG.mk @@ -1,4 +1,10 @@ +ifeq ($(ENABLE_MPI),true) CC = mpicc +DEFINES = -D_MPI +else +CC = cc +endif + GCC = cc LINKER = $(CC) @@ -9,9 +15,7 @@ LIBS = # -lomp endif VERSION = --version -# CFLAGS = -O3 -std=c17 $(OPENMP) CFLAGS = -Ofast -std=c17 -#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG LFLAGS = $(OPENMP) -lm -DEFINES = -D_GNU_SOURCE# -DDEBUG -INCLUDES = +DEFINES += -D_GNU_SOURCE# -DDEBUG +INCLUDES = -I/opt/homebrew/include diff --git a/BasicSolver/2D-mpi-v3/include_GCC.mk b/EnhancedSolver/2D-mpi/include_GCC.mk similarity index 68% rename from BasicSolver/2D-mpi-v3/include_GCC.mk rename to EnhancedSolver/2D-mpi/include_GCC.mk index 427e798..cf1e2aa 100644 --- a/BasicSolver/2D-mpi-v3/include_GCC.mk +++ b/EnhancedSolver/2D-mpi/include_GCC.mk @@ -1,4 +1,10 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpicc +DEFINES = -D_MPI +else CC = gcc +endif + GCC = gcc LINKER = $(CC) @@ -9,6 +15,6 @@ endif VERSION = --version CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE +DEFINES += -D_GNU_SOURCE INCLUDES = LIBS = diff --git a/BasicSolver/2D-mpi-v1/include_ICC.mk b/EnhancedSolver/2D-mpi/include_ICC.mk similarity index 69% rename from BasicSolver/2D-mpi-v1/include_ICC.mk rename to EnhancedSolver/2D-mpi/include_ICC.mk index f85d836..6bedf55 100644 --- a/BasicSolver/2D-mpi-v1/include_ICC.mk +++ b/EnhancedSolver/2D-mpi/include_ICC.mk @@ -1,4 +1,10 @@ +ifeq ($(ENABLE_MPI),true) CC = mpiicc +DEFINES = -D_MPI +else +CC = icc +endif + GCC = gcc LINKER = $(CC) @@ -9,6 +15,6 @@ endif VERSION = --version CFLAGS = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP) LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE +DEFINES += -D_GNU_SOURCE# -DDEBUG INCLUDES = LIBS = diff --git a/EnhancedSolver/2D-mpi/karman.par b/EnhancedSolver/2D-mpi/karman.par new file mode 100644 index 0000000..a22eb34 --- /dev/null +++ b/EnhancedSolver/2D-mpi/karman.par @@ -0,0 +1,79 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name karman # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 5050.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 8.0 # domain size in y-direction +imax 400 # number of interior cells in x-direction +jmax 200 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 150.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 200 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.75 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 15 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 50 +injectTimePeriod 1.0 +writeTimePeriod 0.5 + +x1 0.0 +y1 3.8 +x2 0.0 +y2 4.1 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 2 +xCenter 5.0 +yCenter 4.0 +xRectLength 2.0 +yRectLength 1.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/EnhancedSolver/2D-mpi/residual.plot b/EnhancedSolver/2D-mpi/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/EnhancedSolver/2D-mpi/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/2D-mpi-v3/src/affinity.c b/EnhancedSolver/2D-mpi/src/affinity.c similarity index 95% rename from BasicSolver/2D-mpi-v3/src/affinity.c rename to EnhancedSolver/2D-mpi/src/affinity.c index b501665..ef8355a 100644 --- a/BasicSolver/2D-mpi-v3/src/affinity.c +++ b/EnhancedSolver/2D-mpi/src/affinity.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. diff --git a/BasicSolver/2D-mpi-v2/src/affinity.h b/EnhancedSolver/2D-mpi/src/affinity.h similarity index 83% rename from BasicSolver/2D-mpi-v2/src/affinity.h rename to EnhancedSolver/2D-mpi/src/affinity.h index d844fe5..84bf733 100644 --- a/BasicSolver/2D-mpi-v2/src/affinity.h +++ b/EnhancedSolver/2D-mpi/src/affinity.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. diff --git a/BasicSolver/2D-mpi-v1/src/allocate.c b/EnhancedSolver/2D-mpi/src/allocate.c similarity index 83% rename from BasicSolver/2D-mpi-v1/src/allocate.c rename to EnhancedSolver/2D-mpi/src/allocate.c index 81e1e9d..cf2efd6 100644 --- a/BasicSolver/2D-mpi-v1/src/allocate.c +++ b/EnhancedSolver/2D-mpi/src/allocate.c @@ -1,14 +1,17 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ #include +#include #include #include -void* allocate(int alignment, size_t bytesize) +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) { int errorCode; void* ptr; diff --git a/BasicSolver/2D-mpi-v3/src/allocate.h b/EnhancedSolver/2D-mpi/src/allocate.h similarity index 64% rename from BasicSolver/2D-mpi-v3/src/allocate.h rename to EnhancedSolver/2D-mpi/src/allocate.h index 54cfe06..77f4ba0 100644 --- a/BasicSolver/2D-mpi-v3/src/allocate.h +++ b/EnhancedSolver/2D-mpi/src/allocate.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -8,6 +8,6 @@ #define __ALLOCATE_H_ #include -extern void* allocate(int alignment, size_t bytesize); +extern void* allocate(size_t alignment, size_t bytesize); #endif diff --git a/EnhancedSolver/2D-mpi/src/comm-v1.c b/EnhancedSolver/2D-mpi/src/comm-v1.c new file mode 100644 index 0000000..9be7e88 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/comm-v1.c @@ -0,0 +1,233 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include + +#include "comm.h" + +#ifdef _MPI +// subroutines local to this module +static int sum(int* sizes, int position) +{ + int sum = 0; + + for (int i = 0; i < position; i += position) { + sum += sizes[i]; + } + + return sum; +} + +static void gatherArray( + Comm* c, int cnt, int* rcvCounts, int* displs, double* src, double* dst) +{ + double* sendbuffer = src + (c->imaxLocal + 2); + + if (c->rank == 0) { + sendbuffer = src; + } + + MPI_Gatherv(sendbuffer, + cnt, + MPI_DOUBLE, + dst, + rcvCounts, + displs, + MPI_DOUBLE, + 0, + MPI_COMM_WORLD); +} +#endif // defined _MPI + +// exported subroutines +int commIsBoundary(Comm* c, int direction) +{ +#ifdef _MPI + switch (direction) { + case L: + return 1; + break; + case R: + return 1; + break; + case B: + return c->rank == 0; + break; + case T: + return c->rank == (c->size - 1); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ +#ifdef _MPI + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* exchange ghost cells with top neighbor */ + if (c->rank + 1 < c->size) { + int top = c->rank + 1; + double* src = grid + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + double* dst = grid + (c->jmaxLocal + 1) * (c->imaxLocal + 2) + 1; + + MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, top, 1, MPI_COMM_WORLD, &requests[0]); + MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, top, 2, MPI_COMM_WORLD, &requests[1]); + } + + /* exchange ghost cells with bottom neighbor */ + if (c->rank > 0) { + int bottom = c->rank - 1; + double* src = grid + (c->imaxLocal + 2) + 1; + double* dst = grid + 1; + + MPI_Isend(src, c->imaxLocal, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]); + MPI_Irecv(dst, c->imaxLocal, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]); + } + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commShift(Comm* c, double* f, double* g) +{ +#ifdef _MPI + MPI_Request requests[2] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + if (c->rank > 0) { + int bottom = c->rank - 1; + MPI_Irecv(g + 1, + c->imaxLocal, + MPI_DOUBLE, + bottom, + 0, + MPI_COMM_WORLD, + &requests[0]); + } + + if (c->rank + 1 < c->size) { + int top = c->rank + 1; + double* buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + /* send ghost cells to top neighbor */ + MPI_Isend(buf, c->imaxLocal, MPI_DOUBLE, top, 0, MPI_COMM_WORLD, &requests[1]); + } + + MPI_Waitall(2, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int jmax, + int imax) +{ +#ifdef _MPI + int *rcvCounts, *displs; + int cnt = c->jmaxLocal * (imax + 2); + + if (c->rank == 0) { + rcvCounts = (int*)malloc(c->size * sizeof(int)); + displs = (int*)malloc(c->size * sizeof(int)); + } + + if (c->rank == 0 && c->size == 1) { + cnt = (c->jmaxLocal + 2) * (imax + 2); + } else if (c->rank == 0 || c->rank == (c->size - 1)) { + cnt = (c->jmaxLocal + 1) * (imax + 2); + } + + MPI_Gather(&cnt, 1, MPI_INTEGER, rcvCounts, 1, MPI_INTEGER, 0, MPI_COMM_WORLD); + + if (c->rank == 0) { + displs[0] = 0; + int cursor = rcvCounts[0]; + + for (int i = 1; i < c->size; i++) { + displs[i] = cursor; + cursor += rcvCounts[i]; + } + } + + gatherArray(c, cnt, rcvCounts, displs, p, pg); + gatherArray(c, cnt, rcvCounts, displs, u, ug); + gatherArray(c, cnt, rcvCounts, displs, v, vg); +#endif +} + +void commPartition(Comm* c, int jmax, int imax) +{ +#ifdef _MPI + c->imaxLocal = imax; + c->jmaxLocal = sizeOfRank(c->coords[JDIM], c->size, jmax); + + c->neighbours[BOTTOM] = c->rank == 0 ? -1 : c->rank - 1; + c->neighbours[TOP] = c->rank == (c->size - 1) ? -1 : c->rank + 1; + c->neighbours[LEFT] = -1; + c->neighbours[RIGHT] = -1; + + c->coords[IDIM] = 0; + c->coords[JDIM] = c->rank; + + c->dims[IDIM] = 1; + c->dims[JDIM] = c->size; +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; +#endif +} + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ + +#if defined _MPI + newcomm->comm = MPI_COMM_NULL; + int result = MPI_Comm_dup(MPI_COMM_WORLD, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + + newcomm->neighbours[BOTTOM] = newcomm->rank == 0 ? -1 : newcomm->rank - 1; + newcomm->neighbours[TOP] = newcomm->rank == (newcomm->size - 1) ? -1 : newcomm->rank + 1; + newcomm->neighbours[LEFT] = -1; + newcomm->neighbours[RIGHT] = -1; + + newcomm->coords[IDIM] = 0; + newcomm->coords[JDIM] = newcomm->rank; + + newcomm->dims[IDIM] = 1; + newcomm->dims[JDIM] = newcomm->size; + + +#endif + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/comm-v2.c b/EnhancedSolver/2D-mpi/src/comm-v2.c new file mode 100644 index 0000000..ef1777c --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/comm-v2.c @@ -0,0 +1,342 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "comm.h" +#include +#include +#include + +#ifdef _MPI +// subroutines local to this module +static int sum(int* sizes, int init, int offset, int coord) +{ + int sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax) +{ + MPI_Request* requests; + int numRequests = 1; + + if (c->rank == 0) { + numRequests = c->size + 1; + } else { + numRequests = 1; + } + + requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); + + /* all ranks send their bulk array, including the external boundary layer */ + MPI_Datatype bulkType; + int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; + int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; + int starts[NDIMS] = { 1, 1 }; + + if (commIsBoundary(c, L)) { + newSizes[CIDIM] += 1; + starts[CIDIM] = 0; + } + if (commIsBoundary(c, R)) { + newSizes[CIDIM] += 1; + } + if (commIsBoundary(c, B)) { + newSizes[CJDIM] += 1; + starts[CJDIM] = 0; + } + if (commIsBoundary(c, T)) { + newSizes[CJDIM] += 1; + } + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]); + + int newSizesI[c->size]; + int newSizesJ[c->size]; + MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD); + + /* rank 0 assembles the subdomains */ + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + MPI_Datatype domainType; + int oldSizes[NDIMS] = { jmax + 2, imax + 2 }; + int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] }; + int coords[NDIMS]; + MPI_Cart_coords(c->comm, i, NDIMS, coords); + int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]), + sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) }; + printf( + "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d " + "Starts(i,j): %d %d\n", + i, + coords[IDIM], + coords[JDIM], + oldSizes[CIDIM], + oldSizes[CJDIM], + newSizes[CIDIM], + newSizes[CJDIM], + starts[CIDIM], + starts[CJDIM]); + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} +#endif // defined _MPI + +// exported subroutines +int commIsBoundary(Comm* c, int direction) +{ +#ifdef _MPI + switch (direction) { + case L: + return c->coords[IDIM] == 0; + break; + case R: + return c->coords[IDIM] == (c->dims[IDIM] - 1); + break; + case B: + return c->coords[JDIM] == 0; + break; + case T: + return c->coords[JDIM] == (c->dims[JDIM] - 1); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ +#ifdef _MPI + MPI_Request requests[8]; + for (int i = 0; i < 8; i++) + requests[i] = MPI_REQUEST_NULL; + + for (int i = 0; i < NDIRS; i++) { + double* sbuf = grid + c->sdispls[i]; + double* rbuf = grid + c->rdispls[i]; + + int tag = 0; + if (c->neighbours[i] != MPI_PROC_NULL) { + // printf("DEBUG: Rank %d - SendRecv with %d\n", c->rank, c->neighbours[i]); + tag = c->neighbours[i]; + } + MPI_Irecv(rbuf, + 1, + c->bufferTypes[i], + c->neighbours[i], + tag, + c->comm, + &requests[i * 2]); + MPI_Isend(sbuf, + 1, + c->bufferTypes[i], + c->neighbours[i], + c->rank, + c->comm, + &requests[i * 2 + 1]); + } + + MPI_Waitall(8, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commShift(Comm* c, double* f, double* g) +{ +#ifdef _MPI + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + double* buf = g + 1; + MPI_Irecv(buf, + 1, + c->bufferTypes[B], + c->neighbours[B], + 0, + c->comm, + &requests[0]); + + /* send ghost cells to top neighbor */ + buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + MPI_Isend(buf, 1, c->bufferTypes[T], c->neighbours[T], 0, c->comm, &requests[1]); + + /* shift F */ + /* receive ghost cells from left neighbor */ + buf = f + (c->imaxLocal + 2); + MPI_Irecv(buf, + 1, + c->bufferTypes[L], + c->neighbours[L], + 1, + c->comm, + &requests[2]); + + /* send ghost cells to right neighbor */ + buf = f + (c->imaxLocal + 2) + (c->imaxLocal); + MPI_Isend(buf, + 1, + c->bufferTypes[R], + c->neighbours[R], + 1, + c->comm, + &requests[3]); + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int imax, + int jmax) +{ +#ifdef _MPI + /* collect P */ + assembleResult(c, p, pg, imax, jmax); + + /* collect U */ + assembleResult(c, u, ug, imax, jmax); + + /* collect V */ + assembleResult(c, v, vg, imax, jmax); +#endif +} + +void commPartition(Comm* c, int jmax, int imax) +{ +#ifdef _MPI + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + MPI_Dims_create(c->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm); + MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[L], &c->neighbours[R]); + MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[B], &c->neighbours[T]); + MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords); + + int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax); + int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax); + + c->imaxLocal = imaxLocal; + c->jmaxLocal = jmaxLocal; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + c->bufferTypes[L] = iBufferType; + c->bufferTypes[R] = iBufferType; + c->bufferTypes[B] = jBufferType; + c->bufferTypes[T] = jBufferType; + + c->sdispls[L] = (imaxLocal + 2) + 1; + c->sdispls[R] = (imaxLocal + 2) + imaxLocal; + c->sdispls[B] = (imaxLocal + 2) + 1; + c->sdispls[T] = jmaxLocal * (imaxLocal + 2) + 1; + + c->rdispls[L] = (imaxLocal + 2); + c->rdispls[R] = (imaxLocal + 2) + (imaxLocal + 1); + c->rdispls[B] = 1; + c->rdispls[T] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; +#endif +} + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ +#if defined _MPI + newcomm->comm = MPI_COMM_NULL; + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + memcpy(&newcomm->neighbours, &oldcomm->neighbours, sizeof(oldcomm->neighbours)); + memcpy(&newcomm->coords, &oldcomm->coords, sizeof(oldcomm->coords)); + memcpy(&newcomm->dims, &oldcomm->dims, sizeof(oldcomm->dims)); + + newcomm->imaxLocal = imaxLocal/2; + newcomm->jmaxLocal = jmaxLocal/2; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + newcomm->bufferTypes[L] = iBufferType; + newcomm->bufferTypes[R] = iBufferType; + newcomm->bufferTypes[B] = jBufferType; + newcomm->bufferTypes[T] = jBufferType; + + newcomm->sdispls[L] = (imaxLocal + 2) + 1; + newcomm->sdispls[R] = (imaxLocal + 2) + imaxLocal; + newcomm->sdispls[B] = (imaxLocal + 2) + 1; + newcomm->sdispls[T] = jmaxLocal * (imaxLocal + 2) + 1; + + newcomm->rdispls[L] = (imaxLocal + 2); + newcomm->rdispls[R] = (imaxLocal + 2) + (imaxLocal + 1); + newcomm->rdispls[B] = 1; + newcomm->rdispls[T] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ + #ifdef _MPI + MPI_Comm_free(&comm->comm); + #endif +} \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/comm-v3.c b/EnhancedSolver/2D-mpi/src/comm-v3.c new file mode 100644 index 0000000..18cd186 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/comm-v3.c @@ -0,0 +1,320 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "comm.h" + +#ifdef _MPI +// subroutines local to this module +static int sum(int* sizes, int init, int offset, int coord) +{ + int sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +static void assembleResult(Comm* c, double* src, double* dst, int imax, int jmax) +{ + MPI_Request* requests; + int numRequests = 1; + + if (c->rank == 0) { + numRequests = c->size + 1; + } else { + numRequests = 1; + } + + requests = (MPI_Request*)malloc(numRequests * sizeof(MPI_Request)); + + /* all ranks send their bulk array, including the external boundary layer */ + MPI_Datatype bulkType; + int oldSizes[NDIMS] = { c->jmaxLocal + 2, c->imaxLocal + 2 }; + int newSizes[NDIMS] = { c->jmaxLocal, c->imaxLocal }; + int starts[NDIMS] = { 1, 1 }; + + if (commIsBoundary(c, L)) { + newSizes[CIDIM] += 1; + starts[CIDIM] = 0; + } + if (commIsBoundary(c, R)) { + newSizes[CIDIM] += 1; + } + if (commIsBoundary(c, B)) { + newSizes[CJDIM] += 1; + starts[CJDIM] = 0; + } + if (commIsBoundary(c, T)) { + newSizes[CJDIM] += 1; + } + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + MPI_Isend(src, 1, bulkType, 0, 0, c->comm, &requests[0]); + + int newSizesI[c->size]; + int newSizesJ[c->size]; + MPI_Gather(&newSizes[CIDIM], 1, MPI_INT, newSizesI, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&newSizes[CJDIM], 1, MPI_INT, newSizesJ, 1, MPI_INT, 0, MPI_COMM_WORLD); + + /* rank 0 assembles the subdomains */ + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + MPI_Datatype domainType; + int oldSizes[NDIMS] = { jmax + 2, imax + 2 }; + int newSizes[NDIMS] = { newSizesJ[i], newSizesI[i] }; + int coords[NDIMS]; + MPI_Cart_coords(c->comm, i, NDIMS, coords); + int starts[NDIMS] = { sum(newSizesJ, i, 1, coords[JDIM]), + sum(newSizesI, i, c->dims[JDIM], coords[IDIM]) }; + printf( + "Rank: %d, Coords(i,j): %d %d, Size(i,j): %d %d, Target Size(i,j): %d %d " + "Starts(i,j): %d %d\n", + i, + coords[IDIM], + coords[JDIM], + oldSizes[CIDIM], + oldSizes[CJDIM], + newSizes[CIDIM], + newSizes[CJDIM], + starts[CIDIM], + starts[CJDIM]); + + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} +#endif // defined _MPI + +// exported subroutines +int commIsBoundary(Comm* c, int direction) +{ +#ifdef _MPI + switch (direction) { + case L: + return c->coords[IDIM] == 0; + break; + case R: + return c->coords[IDIM] == (c->dims[IDIM] - 1); + break; + case B: + return c->coords[JDIM] == 0; + break; + case T: + return c->coords[JDIM] == (c->dims[JDIM] - 1); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ +#ifdef _MPI + int counts[NDIRS] = { 1, 1, 1, 1 }; + MPI_Neighbor_alltoallw(grid, + counts, + c->sdispls, + c->bufferTypes, + grid, + counts, + c->rdispls, + c->bufferTypes, + c->comm); +#endif +} + +void commShift(Comm* c, double* f, double* g) +{ +#ifdef _MPI + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + double* buf = g + 1; + MPI_Irecv(buf, + 1, + c->bufferTypes[B], + c->neighbours[B], + 0, + c->comm, + &requests[0]); + + /* send ghost cells to top neighbor */ + buf = g + (c->jmaxLocal) * (c->imaxLocal + 2) + 1; + MPI_Isend(buf, 1, c->bufferTypes[T], c->neighbours[T], 0, c->comm, &requests[1]); + + /* shift F */ + /* receive ghost cells from left neighbor */ + buf = f + (c->imaxLocal + 2); + MPI_Irecv(buf, + 1, + c->bufferTypes[L], + c->neighbours[L], + 1, + c->comm, + &requests[2]); + + /* send ghost cells to right neighbor */ + buf = f + (c->imaxLocal + 2) + (c->imaxLocal); + MPI_Isend(buf, + 1, + c->bufferTypes[R], + c->neighbours[R], + 1, + c->comm, + &requests[3]); + + MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int imax, + int jmax) +{ +#ifdef _MPI + /* collect P */ + assembleResult(c, p, pg, imax, jmax); + + /* collect U */ + assembleResult(c, u, ug, imax, jmax); + + /* collect V */ + assembleResult(c, v, vg, imax, jmax); +#endif +} + +void commPartition(Comm* c, int jmax, int imax) +{ +#ifdef _MPI + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + MPI_Dims_create(c->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NDIMS, dims, periods, 0, &c->comm); + MPI_Cart_shift(c->comm, IDIM, 1, &c->neighbours[L], &c->neighbours[R]); + MPI_Cart_shift(c->comm, JDIM, 1, &c->neighbours[B], &c->neighbours[T]); + MPI_Cart_get(c->comm, NDIMS, c->dims, periods, c->coords); + + int imaxLocal = sizeOfRank(c->coords[IDIM], dims[IDIM], imax); + int jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JDIM], jmax); + + c->imaxLocal = imaxLocal; + c->jmaxLocal = jmaxLocal; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + c->bufferTypes[L] = iBufferType; + c->bufferTypes[R] = iBufferType; + c->bufferTypes[B] = jBufferType; + c->bufferTypes[T] = jBufferType; + + size_t dblsize = sizeof(double); + c->sdispls[L] = ((imaxLocal + 2) + 1) * dblsize; + c->sdispls[R] = ((imaxLocal + 2) + imaxLocal) * dblsize; + c->sdispls[B] = ((imaxLocal + 2) + 1) * dblsize; + c->sdispls[T] = (jmaxLocal * (imaxLocal + 2) + 1) * dblsize; + + c->rdispls[L] = (imaxLocal + 2) * dblsize; + c->rdispls[R] = ((imaxLocal + 2) + (imaxLocal + 1)) * dblsize; + c->rdispls[B] = 1 * dblsize; + c->rdispls[T] = ((jmaxLocal + 1) * (imaxLocal + 2) + 1) * dblsize; +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; +#endif +} + +void commUpdateDatatypes(Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal) +{ +#if defined _MPI + + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + + MPI_Datatype jBufferType; + MPI_Type_contiguous(imaxLocal, MPI_DOUBLE, &jBufferType); + MPI_Type_commit(&jBufferType); + + MPI_Datatype iBufferType; + MPI_Type_vector(jmaxLocal, 1, imaxLocal + 2, MPI_DOUBLE, &iBufferType); + MPI_Type_commit(&iBufferType); + + newcomm->bufferTypes[L] = iBufferType; + newcomm->bufferTypes[R] = iBufferType; + newcomm->bufferTypes[B] = jBufferType; + newcomm->bufferTypes[T] = jBufferType; + + newcomm->sdispls[L] = (imaxLocal + 2) + 1; + newcomm->sdispls[R] = (imaxLocal + 2) + imaxLocal; + newcomm->sdispls[B] = (imaxLocal + 2) + 1; + newcomm->sdispls[T] = jmaxLocal * (imaxLocal + 2) + 1; + + newcomm->rdispls[L] = (imaxLocal + 2); + newcomm->rdispls[R] = (imaxLocal + 2) + (imaxLocal + 1); + newcomm->rdispls[B] = 1; + newcomm->rdispls[T] = (jmaxLocal + 1) * (imaxLocal + 2) + 1; +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ + #ifdef _MPI + MPI_Comm_free(&comm->comm); + #endif +} \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/comm.c b/EnhancedSolver/2D-mpi/src/comm.c new file mode 100644 index 0000000..7904a9b --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/comm.c @@ -0,0 +1,129 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "comm.h" + +// subroutines local to this module +int sizeOfRank(int rank, int size, int N) +{ + return N / size + ((N % size > rank) ? 1 : 0); +} + +void commReduction(double* v, int op) +{ +#ifdef _MPI + if (op == MAX) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + } else if (op == SUM) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + } +#endif +} + +void commPrintConfig(Comm* c) +{ +#ifdef _MPI + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + if (commIsMaster(c)) { + printf("Communication setup:\n"); + } + + for (int i = 0; i < c->size; i++) { + if (i == c->rank) { + printf("\tRank %d of %d\n", c->rank, c->size); + printf("\tNeighbours (bottom, top, left, right): %d %d, %d, %d\n", + c->neighbours[B], + c->neighbours[T], + c->neighbours[L], + c->neighbours[R]); + printf("\tIs boundary:\n"); + printf("\t\tLEFT: %d\n", commIsBoundary(c, L)); + printf("\t\tRIGHT: %d\n", commIsBoundary(c, R)); + printf("\t\tBOTTOM: %d\n", commIsBoundary(c, B)); + printf("\t\tTOP: %d\n", commIsBoundary(c, T)); + printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]); + printf("\tDims (i,j) %d %d\n", c->dims[IDIM], c->dims[JDIM]); + printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal); + fflush(stdout); + } + MPI_Barrier(MPI_COMM_WORLD); + } +#endif +} + +void commInit(Comm* c, int argc, char** argv) +{ +#ifdef _MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank)); + MPI_Comm_size(MPI_COMM_WORLD, &(c->size)); +#else + c->rank = 0; + c->size = 1; +#endif +} + +void commTestInit(Comm* c, double* p, double* f, double* g) +{ + int imax = c->imaxLocal; + int jmax = c->jmaxLocal; + int rank = c->rank; + + for (int j = 0; j < jmax + 2; j++) { + for (int i = 0; i < imax + 2; i++) { + p[j * (imax + 2) + i] = rank; + f[j * (imax + 2) + i] = rank; + g[j * (imax + 2) + i] = rank; + } + } +} + +static void testWriteFile(char* filename, double* grid, int imax, int jmax) +{ + FILE* fp = fopen(filename, "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 0; j < jmax + 2; j++) { + for (int i = 0; i < imax + 2; i++) { + fprintf(fp, "%.2f ", grid[j * (imax + 2) + i]); + } + fprintf(fp, "\n"); + } + + fclose(fp); +} + +void commTestWrite(Comm* c, double* p, double* f, double* g) +{ + int imax = c->imaxLocal; + int jmax = c->jmaxLocal; + int rank = c->rank; + + char filename[30]; + snprintf(filename, 30, "ptest-%d.dat", rank); + testWriteFile(filename, p, imax, jmax); + + snprintf(filename, 30, "ftest-%d.dat", rank); + testWriteFile(filename, f, imax, jmax); + + snprintf(filename, 30, "gtest-%d.dat", rank); + testWriteFile(filename, g, imax, jmax); +} + +void commFinalize(Comm* c) +{ +#ifdef _MPI + MPI_Finalize(); +#endif +} diff --git a/EnhancedSolver/2D-mpi/src/comm.h b/EnhancedSolver/2D-mpi/src/comm.h new file mode 100644 index 0000000..04d2a8a --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/comm.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __COMM_H_ +#define __COMM_H_ +#if defined(_MPI) +#include +#endif + +enum direction { L = 0, R, B, T, NDIRS }; // L = Left, R = Right, B = Bottom, T =Top +enum dimension { IDIM = 0, JDIM, NDIMS }; +enum cdimension { CJDIM = 0, CIDIM }; +enum layer { HALO = 0, BULK }; +enum op { MAX = 0, SUM }; + +typedef struct { + int rank; + int size; +#if defined(_MPI) + MPI_Comm comm; + MPI_Datatype bufferTypes[NDIRS]; + MPI_Aint sdispls[NDIRS]; + MPI_Aint rdispls[NDIRS]; +#endif + int neighbours[NDIRS]; + int coords[NDIMS], dims[NDIMS]; + int imaxLocal, jmaxLocal; +} Comm; + +extern int sizeOfRank(int rank, int size, int N); +extern void commInit(Comm* c, int argc, char** argv); +extern void commTestInit(Comm* c, double* p, double* f, double* g); +extern void commTestWrite(Comm* c, double* p, double* f, double* g); +extern void commFinalize(Comm* c); +extern void commPartition(Comm* c, int jmax, int imax); +extern void commPrintConfig(Comm*); +extern void commExchange(Comm*, double*); +extern void commShift(Comm* c, double* f, double* g); +extern void commReduction(double* v, int op); +extern int commIsBoundary(Comm* c, int direction); +extern void commUpdateDatatypes(Comm*, Comm*, int, int); +extern void commFreeCommunicator(Comm*); +extern void commCollectResult(Comm* c, + double* ug, + double* vg, + double* pg, + double* u, + double* v, + double* p, + int jmax, + int imax); + +static inline int commIsMaster(Comm* c) { return c->rank == 0; } +#endif // __COMM_H_ diff --git a/EnhancedSolver/2D-mpi/src/discretization.c b/EnhancedSolver/2D-mpi/src/discretization.c new file mode 100644 index 0000000..94c6acb --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/discretization.c @@ -0,0 +1,724 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "discretization.h" +#include "parameter.h" +#include "util.h" + +static double distance(double i, double j, double iCenter, double jCenter) +{ + return sqrt(pow(iCenter - i, 2) + pow(jCenter - j, 2) * 1.0); +} + +double sumOffset(double* sizes, int init, int offset, int coord) +{ + double sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +void print(Discretization* d, double* grid) +{ + int imaxLocal = d->comm.imaxLocal; + + for (int i = 0; i < d->comm.size; i++) { + if (i == d->comm.rank) { + sleep(1 * d->comm.rank); + printf("### RANK %d LVL " + "###################################################### #\n ", + d->comm.rank); + for (int j = 0; j < d->comm.jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < d->comm.imaxLocal + 2; i++) { + printf("%2.2f ", grid[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + } +} + +static void printConfig(Discretization* d) +{ + if (commIsMaster(&d->comm)) { + printf("Parameters for #%s#\n", d->problem); + printf("BC Left:%d Right:%d Bottom:%d Top:%d\n", + d->bcLeft, + d->bcRight, + d->bcBottom, + d->bcTop); + printf("\tReynolds number: %.2f\n", d->re); + printf("\tGx Gy: %.2f %.2f\n", d->gx, d->gy); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", + d->grid.xlength, + d->grid.ylength); + printf("\tCells (x, y): %d, %d\n", d->grid.imax, d->grid.jmax); + printf("\tCell size (dx, dy): %f, %f\n", d->grid.dx, d->grid.dy); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te); + printf("\tdt bound: %.6f\n", d->dtBound); + printf("\tTau factor: %.2f\n", d->tau); + printf("Iterative s parameters:\n"); + printf("\tgamma factor: %f\n", d->gamma); + } + commPrintConfig(&d->comm); +} + +void initDiscretiztion(Discretization* d, Parameter* params) +{ + d->problem = params->name; + d->bcLeft = params->bcLeft; + d->bcRight = params->bcRight; + d->bcBottom = params->bcBottom; + d->bcTop = params->bcTop; + d->grid.imax = params->imax; + d->grid.jmax = params->jmax; + d->grid.xlength = params->xlength; + d->grid.ylength = params->ylength; + d->grid.dx = params->xlength / params->imax; + d->grid.dy = params->ylength / params->jmax; + d->re = params->re; + d->gx = params->gx; + d->gy = params->gy; + d->dt = params->dt; + d->te = params->te; + d->tau = params->tau; + d->gamma = params->gamma; + + /* allocate arrays */ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + size_t size = (imaxLocal + 2) * (jmaxLocal + 2); + + d->u = allocate(64, size * sizeof(double)); + d->v = allocate(64, size * sizeof(double)); + d->p = allocate(64, size * sizeof(double)); + d->rhs = allocate(64, size * sizeof(double)); + d->f = allocate(64, size * sizeof(double)); + d->g = allocate(64, size * sizeof(double)); + d->grid.s = allocate(64, size * sizeof(double)); + + for (int i = 0; i < size; i++) { + d->u[i] = params->u_init; + d->v[i] = params->v_init; + d->p[i] = params->p_init; + d->rhs[i] = 0.0; + d->f[i] = 0.0; + d->g[i] = 0.0; + d->grid.s[i] = FLUID; + } + + double dx = d->grid.dx; + double dy = d->grid.dy; + + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy); + d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; + + d->xLocal = d->comm.imaxLocal * d->grid.dx; + d->yLocal = d->comm.jmaxLocal * d->grid.dy; + + double xLocal[d->comm.size]; + double yLocal[d->comm.size]; + +#ifdef _MPI + MPI_Allgather(&d->xLocal, 1, MPI_DOUBLE, xLocal, 1, MPI_DOUBLE, d->comm.comm); + MPI_Allgather(&d->yLocal, 1, MPI_DOUBLE, yLocal, 1, MPI_DOUBLE, d->comm.comm); + + d->xOffset = sumOffset(xLocal, + d->comm.rank, + d->comm.dims[JDIM], + d->comm.coords[IDIM]); + d->yOffset = sumOffset(yLocal, d->comm.rank, 1, d->comm.coords[JDIM]); + d->xOffsetEnd = d->xOffset + d->xLocal; + d->yOffsetEnd = d->yOffset + d->yLocal; +#else + + d->xOffset = 0; + d->yOffset = 0; + d->xOffsetEnd = d->xOffset + d->xLocal; + d->yOffsetEnd = d->yOffset + d->yLocal; +#endif + + printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : " + "%.2f\n", + d->comm.rank, + d->xOffset, + d->yOffset, + d->xOffsetEnd, + d->yOffsetEnd); + + double* s = d->grid.s; + int iOffset = 0, jOffset = 0; + + double xCenter = 0, yCenter = 0, radius = 0; + double x1 = 0, x2 = 0, y1 = 0, y2 = 0; + + switch (params->shape) { + case NOSHAPE: + break; + case RECT: + x1 = params->xCenter - params->xRectLength / 2; + x2 = params->xCenter + params->xRectLength / 2; + y1 = params->yCenter - params->yRectLength / 2; + y2 = params->yCenter + params->yRectLength / 2; + + iOffset = d->xOffset / dx; + jOffset = d->yOffset / dy; + + for (int j = 1; j < jmaxLocal + 1; ++j) { + for (int i = 1; i < imaxLocal + 1; ++i) { + if ((x1 <= ((i + iOffset) * dx)) && (((i + iOffset) * dx) <= x2) && + (y1 <= ((j + jOffset) * dy)) && (((j + jOffset) * dy) <= y2)) { + S(i, j) = OBSTACLE; + } + } + } + + break; + case CIRCLE: + xCenter = params->xCenter; + yCenter = params->yCenter; + radius = params->circleRadius; + + iOffset = d->xOffset / dx; + jOffset = d->yOffset / dy; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if (distance(((i + iOffset) * dx), + ((j + jOffset) * dy), + xCenter, + yCenter) <= radius) { + S(i, j) = OBSTACLE; + } + } + } + + break; + default: + break; + } + +#ifdef _MPI + commExchange(&d->comm, s); +#endif + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if (S(i, j - 1) == FLUID && S(i, j + 1) == OBSTACLE && S(i, j) == OBSTACLE) + S(i, j) = BOTTOM; // BOTTOM + if (S(i - 1, j) == FLUID && S(i + 1, j) == OBSTACLE && S(i, j) == OBSTACLE) + S(i, j) = LEFT; // LEFT + if (S(i + 1, j) == FLUID && S(i - 1, j) == OBSTACLE && S(i, j) == OBSTACLE) + S(i, j) = RIGHT; // RIGHT + if (S(i, j + 1) == FLUID && S(i, j - 1) == OBSTACLE && S(i, j) == OBSTACLE) + S(i, j) = TOP; // TOP + if (S(i - 1, j - 1) == FLUID && S(i, j - 1) == FLUID && + S(i - 1, j) == FLUID && S(i + 1, j + 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == LEFT || S(i, j) == BOTTOM)) + S(i, j) = BOTTOMLEFT; // BOTTOMLEFT + if (S(i + 1, j - 1) == FLUID && S(i, j - 1) == FLUID && + S(i + 1, j) == FLUID && S(i - 1, j + 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == RIGHT || S(i, j) == BOTTOM)) + S(i, j) = BOTTOMRIGHT; // BOTTOMRIGHT + if (S(i - 1, j + 1) == FLUID && S(i - 1, j) == FLUID && + S(i, j + 1) == FLUID && S(i + 1, j - 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == LEFT || S(i, j) == TOP)) + S(i, j) = TOPLEFT; // TOPLEFT + if (S(i + 1, j + 1) == FLUID && S(i + 1, j) == FLUID && + S(i, j + 1) == FLUID && S(i - 1, j - 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == RIGHT || S(i, j) == TOP)) + S(i, j) = TOPRIGHT; // TOPRIGHT + } + } + +#ifdef VERBOSE + printConfig(d); +#endif +} + +void computeRHS(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + double idx = 1.0 / d->grid.dx; + double idy = 1.0 / d->grid.dy; + double idt = 1.0 / d->dt; + double* rhs = d->rhs; + double* f = d->f; + double* g = d->g; + + commShift(&d->comm, f, g); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + RHS(i, j) = ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy) * + idt; + } + } +} + +static double maxElement(Discretization* d, double* m) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int size = (imaxLocal + 2) * (jmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + commReduction(&maxval, MAX); + return maxval; +} + +void computeTimestep(Discretization* d) +{ + double dt = d->dtBound; + double dx = d->grid.dx; + double dy = d->grid.dy; + double umax = maxElement(d, d->u); + double vmax = maxElement(d, d->v); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + + d->dt = dt * d->tau; +} + +void setBoundaryConditions(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + double* u = d->u; + double* v = d->v; + + if (commIsBoundary(&d->comm, T)) { + switch (d->bcTop) { + case NOSLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = -U(i, jmaxLocal); + } + break; + case SLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, jmaxLocal) = 0.0; + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + } + break; + case OUTFLOW: + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1) = U(i, jmaxLocal); + V(i, jmaxLocal) = V(i, jmaxLocal - 1); + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, B)) { + switch (d->bcBottom) { + case NOSLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = -U(i, 1); + } + break; + case SLIP: + for (int i = 1; i < imaxLocal + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = U(i, 1); + } + break; + case OUTFLOW: + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0) = U(i, 1); + V(i, 0) = V(i, 1); + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, R)) { + switch (d->bcRight) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = 0.0; + V(imaxLocal + 1, j) = -V(imaxLocal, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = 0.0; + V(imaxLocal + 1, j) = V(imaxLocal, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j) = U(imaxLocal - 1, j); + V(imaxLocal + 1, j) = V(imaxLocal, j); + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, L)) { + switch (d->bcLeft) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = -V(1, j); + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 0.0; + V(0, j) = V(1, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = U(1, j); + V(0, j) = V(1, j); + } + break; + case PERIODIC: + break; + } + } +} + +void setSpecialBoundaryCondition(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + double* u = d->u; + double* s = d->grid.s; + + if (strcmp(d->problem, "dcavity") == 0) { + if (commIsBoundary(&d->comm, T)) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1) = 2.0 - U(i, jmaxLocal); + } + } + } else if (strcmp(d->problem, "canal") == 0) { + if (commIsBoundary(&d->comm, L)) { + + double ylength = d->grid.ylength; + double dy = d->grid.dy; + int rest = d->grid.jmax % d->comm.dims[JDIM]; + int yc = d->comm.rank * (d->grid.jmax / d->comm.dims[JDIM]) + + MIN(rest, d->comm.rank); + double ys = dy * (yc + 0.5); + double y; + + // printf("RANK %d yc: %d ys: %f\n",d->comm.rank, yc, ys); + + for (int j = 1; j < jmaxLocal + 1; j++) { + y = ys + dy * (j - 0.5); + U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } + } else if (strcmp(d->problem, "backstep") == 0) { + for (int j = 1; j < jmaxLocal + 1; j++) { + if (S(0, j) == FLUID) U(0, j) = 1.0; + } + } else if (strcmp(d->problem, "karman") == 0) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j) = 1.0; + } + } + /* print(solver, solver->u); */ +} + +void setObjectBoundaryCondition(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + double* u = d->u; + double* v = d->v; + double* s = d->grid.s; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + switch ((int)S(i, j)) { + case TOP: + U(i, j) = -U(i, j + 1); + U(i - 1, j) = -U(i - 1, j + 1); + V(i, j) = 0.0; + break; + case BOTTOM: + U(i, j) = -U(i, j - 1); + U(i - 1, j) = -U(i - 1, j - 1); + V(i, j) = 0.0; + break; + case LEFT: + U(i - 1, j) = 0.0; + V(i, j) = -V(i - 1, j); + V(i, j - 1) = -V(i - 1, j - 1); + break; + case RIGHT: + U(i, j) = 0.0; + V(i, j) = -V(i + 1, j); + V(i, j - 1) = -V(i + 1, j - 1); + break; + case TOPLEFT: + U(i, j) = -U(i, j + 1); + U(i - 1, j) = 0.0; + V(i, j) = 0.0; + V(i, j - 1) = -V(i - 1, j - 1); + break; + case TOPRIGHT: + U(i, j) = 0.0; + U(i - 1, j) = -U(i - 1, j + 1); + V(i, j) = 0.0; + V(i, j - 1) = -V(i + 1, j - 1); + break; + case BOTTOMLEFT: + U(i, j) = -U(i, j - 1); + U(i - 1, j) = 0.0; + V(i, j) = -V(i - 1, j); + V(i, j - 1) = 0.0; + break; + case BOTTOMRIGHT: + U(i, j) = 0.0; + U(i - 1, j) = -U(i - 1, j - 1); + V(i, j) = -V(i, j + 1); + V(i, j - 1) = 0.0; + break; + } + } + } +} + +void computeFG(Discretization* d) +{ + double* u = d->u; + double* v = d->v; + double* f = d->f; + double* g = d->g; + double* s = d->grid.s; + + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + + double gx = d->gx; + double gy = d->gy; + double gamma = d->gamma; + double dt = d->dt; + double inverseRe = 1.0 / d->re; + double inverseDx = 1.0 / d->grid.dx; + double inverseDy = 1.0 / d->grid.dy; + double du2dx, dv2dy, duvdx, duvdy; + double du2dx2, du2dy2, dv2dx2, dv2dy2; + + commExchange(&d->comm, u); + commExchange(&d->comm, v); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if (S(i, j) == FLUID) { + du2dx = inverseDx * 0.25 * + ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - + (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + + fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); + + duvdy = inverseDy * 0.25 * + ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - + (V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) + U(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + + fabs(V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) - U(i, j - 1))); + + du2dx2 = inverseDx * inverseDx * + (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); + du2dy2 = inverseDy * inverseDy * + (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); + F(i, j) = U(i, j) + + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - + (U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) + V(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + + fabs(U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) - V(i - 1, j))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - + (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + + fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); + + dv2dx2 = inverseDx * inverseDx * + (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); + dv2dy2 = inverseDy * inverseDy * + (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); + G(i, j) = V(i, j) + + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); + } else { + switch ((int)S(i, j)) { + case TOP: + G(i, j) = V(i, j); + break; + case BOTTOM: + G(i, j - 1) = V(i, j - 1); + break; + case LEFT: + F(i - 1, j) = U(i - 1, j); + break; + case RIGHT: + F(i, j) = U(i, j); + break; + case TOPLEFT: + F(i - 1, j) = U(i - 1, j); + G(i, j) = V(i, j); + break; + case TOPRIGHT: + F(i, j) = U(i, j); + G(i, j) = V(i, j); + break; + case BOTTOMLEFT: + F(i - 1, j) = U(i - 1, j); + G(i, j - 1) = V(i, j - 1); + break; + case BOTTOMRIGHT: + F(i, j) = U(i, j); + G(i, j - 1) = V(i, j - 1); + break; + } + } + } + } + + /* ----------------------------- boundary of F --------------------------- */ + if (commIsBoundary(&d->comm, L)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(0, j) = U(0, j); + } + } + + if (commIsBoundary(&d->comm, R)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(imaxLocal, j) = U(imaxLocal, j); + } + } + + /* ----------------------------- boundary of G --------------------------- */ + if (commIsBoundary(&d->comm, B)) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, 0) = V(i, 0); + } + } + + if (commIsBoundary(&d->comm, T)) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, jmaxLocal) = V(i, jmaxLocal); + } + } +} + +void adaptUV(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + + double* p = d->p; + double* u = d->u; + double* v = d->v; + double* f = d->f; + double* g = d->g; + + double factorX = d->dt / d->grid.dx; + double factorY = d->dt / d->grid.dy; + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; + V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; + } + } +} + +void writeResult(Discretization* d, double* u, double* v, double* p) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double dx = d->grid.dx; + double dy = d->grid.dy; + double x = 0.0, y = 0.0; + + FILE* fp; + fp = fopen("pressure.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j <= jmax; j++) { + y = (double)(j - 0.5) * dy; + for (int i = 1; i <= imax; i++) { + x = (double)(i - 0.5) * dx; + fprintf(fp, "%.2f %.2f %f\n", x, y, p[j * (imax + 2) + i]); + } + fprintf(fp, "\n"); + } + + fclose(fp); + + fp = fopen("velocity.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j <= jmax; j++) { + y = dy * (j - 0.5); + for (int i = 1; i <= imax; i++) { + x = dx * (i - 0.5); + double velU = (u[j * (imax + 2) + i] + u[j * (imax + 2) + (i - 1)]) / 2.0; + double velV = (v[j * (imax + 2) + i] + v[(j - 1) * (imax + 2) + i]) / 2.0; + double len = sqrt((velU * velU) + (velV * velV)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, velU, velV, len); + } + } + + fclose(fp); +} diff --git a/EnhancedSolver/2D-mpi/src/discretization.h b/EnhancedSolver/2D-mpi/src/discretization.h new file mode 100644 index 0000000..5765a2e --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/discretization.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ +#include "comm.h" +#include "grid.h" +#include "parameter.h" +#include + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +enum OBJECTBOUNDARY { + FLUID = 0, + TOP, + BOTTOM, + LEFT, + RIGHT, + TOPLEFT, + BOTTOMLEFT, + TOPRIGHT, + BOTTOMRIGHT, + OBSTACLE +}; + +enum SHAPE { NOSHAPE = 0, RECT, CIRCLE }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double re, tau, gamma; + double gx, gy; + /* time stepping */ + double dt, te; + double dtBound; + char* problem; + + double xLocal, yLocal, xOffset, yOffset, xOffsetEnd, yOffsetEnd; + + + int bcLeft, bcRight, bcBottom, bcTop; + /* communication */ + Comm comm; +} Discretization; + +extern void initDiscretiztion(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void setObjectBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +extern void writeResult(Discretization* s, double* u, double* v, double* p); +extern double sumOffset(double* , int , int , int ); +extern void print(Discretization* , double* ); +#endif diff --git a/EnhancedSolver/2D-mpi/src/grid.h b/EnhancedSolver/2D-mpi/src/grid.h new file mode 100644 index 0000000..d326641 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/grid.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __GRID_H_ +#define __GRID_H_ + +typedef struct { + double dx, dy; + int imax, jmax; + double xlength, ylength; + double* s; +} Grid; + +#endif // __GRID_H_ diff --git a/BasicSolver/2D-mpi-v2/src/likwid-marker.h b/EnhancedSolver/2D-mpi/src/likwid-marker.h similarity index 100% rename from BasicSolver/2D-mpi-v2/src/likwid-marker.h rename to EnhancedSolver/2D-mpi/src/likwid-marker.h diff --git a/EnhancedSolver/2D-mpi/src/main.c b/EnhancedSolver/2D-mpi/src/main.c new file mode 100644 index 0000000..1f3f79c --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/main.c @@ -0,0 +1,125 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "discretization.h" +#include "grid.h" +#include "parameter.h" +#include "particletracing.h" +#include "progress.h" +#include "timing.h" + +static void writeResults(Discretization* s) +{ +#ifdef _MPI + size_t bytesize = (s->grid.imax + 2) * (s->grid.jmax + 2) * sizeof(double); + + double* ug = allocate(64, bytesize); + double* vg = allocate(64, bytesize); + double* pg = allocate(64, bytesize); + + commCollectResult(&s->comm, ug, vg, pg, s->u, s->v, s->p, s->grid.imax, s->grid.jmax); + if (commIsMaster(&s->comm)) { + writeResult(s, ug, vg, pg); + } + + free(ug); + free(vg); + free(pg); +#else + writeResult(s, s->u, s->v, s->p); +#endif +} + +int main(int argc, char** argv) +{ + int rank; + double timeStart, timeStop; + Parameter p; + Discretization d; + Solver s; + ParticleTracer particletracer; + + commInit(&d.comm, argc, argv); + initParameter(&p); + FILE* fp; + fp = initResidualWriter(); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(&p, argv[1]); + commPartition(&d.comm, p.jmax, p.imax); + if (commIsMaster(&d.comm)) { + printParameter(&p); + } + + initDiscretiztion(&d, &p); + initSolver(&s, &d, &p); + initParticleTracer(&particletracer, &d, &p); + +#ifdef TEST + commPrintConfig(&d.comm); + commTestInit(&d.comm, d.p, d.f, d.g); + commExchange(&d.comm, d.p); + commShift(&d.comm, d.f, d.g); + commTestWrite(&d.comm, d.p, d.f, d.g); + writeResults(&d); + commFinalize(&d.comm); + exit(EXIT_SUCCESS); +#endif +#ifndef VERBOSE + initProgress(d.te); +#endif + + double tau = d.tau; + double te = d.te; + double t = 0.0; + double res = 0.0; + timeStart = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + setObjectBoundaryCondition(&d); + computeFG(&d); + computeRHS(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + trace(&particletracer, &d, t); + + writeResidual(fp, t, res); + + t += d.dt; + +#ifdef VERBOSE + if (commIsMaster(s.comm)) { + printf("TIME %f , TIMESTEP %f\n", t, d.dt); + } +#else + printProgress(t); +#endif + } + timeStop = getTimeStamp(); +#ifndef VERBOSE + stopProgress(); +#endif + if (commIsMaster(s.comm)) { + printf("Solution took %.2fs\n", timeStop - timeStart); + } + fclose(fp); + freeParticles(&particletracer); + writeResults(&d); + commFinalize(s.comm); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/2D-mpi-v1/src/parameter.c b/EnhancedSolver/2D-mpi/src/parameter.c similarity index 67% rename from BasicSolver/2D-mpi-v1/src/parameter.c rename to EnhancedSolver/2D-mpi/src/parameter.c index d691627..3f388c4 100644 --- a/BasicSolver/2D-mpi-v1/src/parameter.c +++ b/EnhancedSolver/2D-mpi/src/parameter.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -14,16 +14,16 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->imax = 100; - param->jmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.7; - param->re = 100.0; - param->gamma = 0.9; - param->tau = 0.5; + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.8; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) @@ -75,9 +75,30 @@ void readParameter(Parameter* param, const char* filename) PARSE_INT(bcRight); PARSE_INT(bcBottom); PARSE_INT(bcTop); + PARSE_INT(levels); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); PARSE_REAL(u_init); PARSE_REAL(v_init); PARSE_REAL(p_init); + + /* Added new particle tracing parameters */ + PARSE_INT(numberOfParticles); + PARSE_REAL(startTime); + PARSE_REAL(injectTimePeriod); + PARSE_REAL(writeTimePeriod); + PARSE_REAL(x1); + PARSE_REAL(y1); + PARSE_REAL(x2); + PARSE_REAL(y2); + + /* Added obstacle geometry parameters */ + PARSE_INT(shape); + PARSE_REAL(xCenter); + PARSE_REAL(yCenter); + PARSE_REAL(xRectLength); + PARSE_REAL(yRectLength); + PARSE_REAL(circleRadius); } } @@ -108,4 +129,15 @@ void printParameter(Parameter* param) printf("\tepsilon (stopping tolerance) : %f\n", param->eps); printf("\tgamma (stopping tolerance) : %f\n", param->gamma); printf("\tomega (SOR relaxation): %f\n", param->omg); + printf("Particle tracer parameters:\n"); + printf("\tNumber of particles: %d\n", param->numberOfParticles); + printf("\tstartTime : %f\n", param->startTime); + printf("\tinjecTimePeriod : %f\n", param->injectTimePeriod); + printf("\twriteTimePeriod: %f\n", param->writeTimePeriod); + printf("\tx1 : %f, x2 : %f, y1 : %f, y2 : %f\n", + param->x1, + param->x2, + param->y1, + param->y2); + printf("\tShape : %d\n", param->shape); } diff --git a/BasicSolver/2D-mpi-v3/src/parameter.h b/EnhancedSolver/2D-mpi/src/parameter.h similarity index 66% rename from BasicSolver/2D-mpi-v3/src/parameter.h rename to EnhancedSolver/2D-mpi/src/parameter.h index f4c331a..3085db3 100644 --- a/BasicSolver/2D-mpi-v3/src/parameter.h +++ b/EnhancedSolver/2D-mpi/src/parameter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -18,6 +18,18 @@ typedef struct { char* name; int bcLeft, bcRight, bcBottom, bcTop; double u_init, v_init, p_init; + + int levels, presmooth, postsmooth; + + int numberOfParticles; + double startTime, injectTimePeriod, writeTimePeriod; + + double x1, y1, x2, y2; + + int shape; + double xCenter, yCenter, xRectLength, yRectLength, circleRadius; + + } Parameter; void initParameter(Parameter*); diff --git a/EnhancedSolver/2D-mpi/src/particletracing.c b/EnhancedSolver/2D-mpi/src/particletracing.c new file mode 100644 index 0000000..f4bb874 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/particletracing.c @@ -0,0 +1,675 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "particletracing.h" + +#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] +#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] +#define S(i, j) s[(j) * (imaxLocal + 2) + (i)] + +static int ts = 0; + +#define IDIM 0 +#define JDIM 1 + +#define XOFFSET 0 +#define YOFFSET 1 +#define XOFFSETEND 2 +#define YOFFSETEND 3 + +static double sum(int* sizes, int size) +{ + double sum = 0; + + for (int i = 0; i < size; ++i) { + sum += sizes[i]; + } + + return sum; +} + +void printUV(ParticleTracer* particletracer, double* u, double* v) +{ + int imaxLocal = particletracer->imaxLocal; + + for (int i = 0; i < particletracer->size; i++) { + if (i == particletracer->rank) { + printf( + "\n### RANK %d #######################################################\n", + particletracer->rank); + printf("\nGrid U : \n"); + for (int j = 0; j < particletracer->jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < particletracer->imaxLocal + 2; i++) { + printf("%4.2f ", u[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + printf("\nGrid V : \n"); + for (int j = 0; j < particletracer->jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < particletracer->imaxLocal + 2; i++) { + printf("%4.2f ", v[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + +#ifdef _MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + } +} + +void initParticleTracer( + ParticleTracer* particletracer, Discretization* d, Parameter* params) +{ + int dims[NDIMS] = { 0, 0 }; + int periods[NDIMS] = { 0, 0 }; + + /* initializing local properties from params */ + particletracer->rank = d->comm.rank; + particletracer->size = d->comm.size; + + particletracer->numberOfParticles = params->numberOfParticles; + particletracer->startTime = params->startTime; + particletracer->injectTimePeriod = params->injectTimePeriod; + particletracer->writeTimePeriod = params->writeTimePeriod; + + particletracer->dt = params->dt; + particletracer->dx = params->xlength / params->imax; + particletracer->dy = params->ylength / params->jmax; + + particletracer->xlength = params->xlength; + particletracer->ylength = params->ylength; + + particletracer->x1 = params->x1; + particletracer->y1 = params->y1; + particletracer->x2 = params->x2; + particletracer->y2 = params->y2; + + particletracer->lastInjectTime = params->startTime; + particletracer->lastUpdateTime = params->startTime; + particletracer->lastWriteTime = params->startTime; + + particletracer->pointer = 0; + particletracer->totalParticles = 0; + particletracer->removedParticles = 0; + + particletracer->imax = params->imax; + particletracer->jmax = params->jmax; + + particletracer->imaxLocal = d->comm.imaxLocal; + particletracer->jmaxLocal = d->comm.jmaxLocal; + particletracer->removedParticles = 0; + + // Estimating the number of particles over the number of timesteps that could be + // required. + particletracer->estimatedNumParticles = (particletracer->imaxLocal * + particletracer->jmaxLocal); + + // Allocating memory for the estimated particles over the timesteps. + particletracer->particlePool = malloc( + sizeof(Particle) * particletracer->estimatedNumParticles); + + // Initializing the number of particles to 0 and turning OFF all of the particles. + // Contain information in x and y length metric, not i and j discretization metrics. + for (int i = 0; i < particletracer->estimatedNumParticles; ++i) { + particletracer->particlePool[i].x = 0.0; + particletracer->particlePool[i].y = 0.0; + particletracer->particlePool[i].flag = false; + } + + // Creating a linearly spaced particle line. + particletracer->linSpaceLine = malloc( + sizeof(Particle) * particletracer->numberOfParticles); + + // Creating an array for each rank that will hold information about + // offsets from other ranks. Holds each ranks x and y length metrics. + particletracer->offset = (double*)malloc(sizeof(double) * 4 * particletracer->size); + + // Calculating each ranks local x and y length metrics. + + double offset[4][particletracer->size]; + + // Calculating each ranks x and y local lengths. + particletracer->xLocal = d->xLocal; + particletracer->yLocal = d->yLocal; + + double xLocal[particletracer->size]; + double yLocal[particletracer->size]; + + // Calculate own x and y length metric offset based on other ranks offset data. + particletracer->xOffset = d->xOffset; + particletracer->yOffset = d->yOffset; + particletracer->xOffsetEnd = d->xOffsetEnd; + particletracer->yOffsetEnd = d->yOffsetEnd; + + printf("Rank : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, yOffsetEnd : " + "%.2f\n", + particletracer->rank, + particletracer->xOffset, + particletracer->yOffset, + particletracer->xOffsetEnd, + particletracer->yOffsetEnd); + +#ifdef _MPI + // Gather each ranks x and y length metric that marks each ranks own territory. + // Once the boundary leaves local domain, then it needs to know which ranks to send. + // And to know whos boundary it is, we need to know the rank. + MPI_Allgather(&particletracer->xOffset, + 1, + MPI_DOUBLE, + offset[0], + 1, + MPI_DOUBLE, + d->comm.comm); + MPI_Allgather(&particletracer->yOffset, + 1, + MPI_DOUBLE, + offset[1], + 1, + MPI_DOUBLE, + d->comm.comm); + MPI_Allgather(&particletracer->xOffsetEnd, + 1, + MPI_DOUBLE, + offset[2], + 1, + MPI_DOUBLE, + d->comm.comm); + MPI_Allgather(&particletracer->yOffsetEnd, + 1, + MPI_DOUBLE, + offset[3], + 1, + MPI_DOUBLE, + d->comm.comm); +#endif + + memcpy(particletracer->offset, offset, sizeof(offset)); + + particleRandomizer(particletracer); + +#ifdef _MPI + // Create the mpi_particle datatype + MPI_Datatype mpi_particle; + int lengths[3] = { 1, 1, 1 }; + + MPI_Aint displacements[3]; + Particle dummy_particle; + MPI_Aint base_address; + MPI_Get_address(&dummy_particle, &base_address); + MPI_Get_address(&dummy_particle.x, &displacements[0]); + MPI_Get_address(&dummy_particle.y, &displacements[1]); + MPI_Get_address(&dummy_particle.flag, &displacements[2]); + displacements[0] = MPI_Aint_diff(displacements[0], base_address); + displacements[1] = MPI_Aint_diff(displacements[1], base_address); + displacements[2] = MPI_Aint_diff(displacements[2], base_address); + + MPI_Datatype types[3] = { MPI_DOUBLE, MPI_DOUBLE, MPI_C_BOOL }; + MPI_Type_create_struct(3, + lengths, + displacements, + types, + &particletracer->mpi_particle); + MPI_Type_commit(&particletracer->mpi_particle); +#endif +} + +void printParticles(ParticleTracer* particletracer) +{ + for (int i = 0; i < particletracer->totalParticles; ++i) { + printf("Rank : %d Particle position X : %.2f, Y : %.2f, flag : %d, total pt : " + "%d, pointer : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, " + "yOffsetEnd : %.2f\n", + particletracer->rank, + particletracer->particlePool[i].x, + particletracer->particlePool[i].y, + particletracer->particlePool[i].flag, + particletracer->totalParticles, + particletracer->pointer, + particletracer->xOffset, + particletracer->yOffset, + particletracer->xOffsetEnd, + particletracer->yOffsetEnd); + } +} +void injectParticles(ParticleTracer* particletracer, double* s) +{ + double x, y; + compress(particletracer); + particleRandomizer(particletracer); + + int imaxLocal = particletracer->imaxLocal; + int jmaxLocal = particletracer->jmaxLocal; + + for (int i = 0; i < particletracer->numberOfParticles; ++i) { + x = particletracer->linSpaceLine[i].x; + y = particletracer->linSpaceLine[i].y; + if (x >= particletracer->xOffset && y >= particletracer->yOffset && + x <= particletracer->xOffsetEnd && y <= particletracer->yOffsetEnd) { + + particletracer->particlePool[particletracer->pointer].x = x; + particletracer->particlePool[particletracer->pointer].y = y; + + int i = particletracer->particlePool[particletracer->pointer].x / + particletracer->dx; + int j = particletracer->particlePool[particletracer->pointer].y / + particletracer->dy; + + int iOffset = particletracer->xOffset / particletracer->dx, + jOffset = particletracer->yOffset / particletracer->dy; + + if (S(i - iOffset, j - jOffset) == FLUID) { + particletracer->particlePool[particletracer->pointer].flag = true; + ++(particletracer->pointer); + ++(particletracer->totalParticles); + } + } + } +} + +void advanceParticles(ParticleTracer* particletracer, + double* restrict u, + double* restrict v, + double* restrict s, + Comm* comm, + double time) +{ + + int imax = particletracer->imax; + int jmax = particletracer->jmax; + int imaxLocal = particletracer->imaxLocal; + int jmaxLocal = particletracer->jmaxLocal; + + double dx = particletracer->dx; + double dy = particletracer->dy; + + double xlength = particletracer->xlength; + double ylength = particletracer->ylength; + + Particle buff[particletracer->size][(particletracer->estimatedNumParticles)]; + memset(buff, 0, sizeof(buff)); + Particle recvbuff[particletracer->size][(particletracer->estimatedNumParticles)]; + memset(buff, 0, sizeof(recvbuff)); + + int particleBufIndex[particletracer->size], + recvparticleBufIndex[particletracer->size]; + + memset(particleBufIndex, 0, sizeof(particleBufIndex)); + memset(recvparticleBufIndex, 0, sizeof(recvparticleBufIndex)); + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (particletracer->particlePool[i].flag == true) { + double xTemp = particletracer->particlePool[i].x; + double yTemp = particletracer->particlePool[i].y; + + double x = xTemp - particletracer->xOffset; + double y = yTemp - particletracer->yOffset; + + int iCoord = (int)(x / dx) + 1; + int jCoord = (int)((y + 0.5 * dy) / dy) + 1; + + double x1 = (double)(iCoord - 1) * dx; + double y1 = ((double)(jCoord - 1) - 0.5) * dy; + double x2 = (double)iCoord * dx; + double y2 = ((double)jCoord - 0.5) * dy; + + double u_n = (1.0 / (dx * dy)) * + ((x2 - x) * (y2 - y) * U(iCoord - 1, jCoord - 1) + + (x - x1) * (y2 - y) * U(iCoord, jCoord - 1) + + (x2 - x) * (y - y1) * U(iCoord - 1, jCoord) + + (x - x1) * (y - y1) * U(iCoord, jCoord)); + + double new_x = (x + particletracer->xOffset) + particletracer->dt * u_n; + particletracer->particlePool[i].x = new_x; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)(y / dy) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = (double)(jCoord - 1) * dy; + x2 = ((double)iCoord - 0.5) * dx; + y2 = (double)jCoord * dy; + + double v_n = (1.0 / (dx * dy)) * + ((x2 - x) * (y2 - y) * V(iCoord - 1, jCoord - 1) + + (x - x1) * (y2 - y) * V(iCoord, jCoord - 1) + + (x2 - x) * (y - y1) * V(iCoord - 1, jCoord) + + (x - x1) * (y - y1) * V(iCoord, jCoord)); + + double new_y = (y + particletracer->yOffset) + particletracer->dt * v_n; + particletracer->particlePool[i].y = new_y; + + if (((new_x < particletracer->xOffset) || + (new_x > particletracer->xOffsetEnd) || + (new_y < particletracer->yOffset) || + (new_y > particletracer->yOffsetEnd))) { + // New logic to transfer particles to neighbouring ranks or discard the + // particle. +#ifdef _MPI + for (int i = 0; i < particletracer->size; ++i) { + if ((new_x >= + particletracer->offset[i + particletracer->size * XOFFSET]) && + (new_x <= particletracer + ->offset[i + particletracer->size * XOFFSETEND]) && + (new_y >= + particletracer->offset[i + particletracer->size * YOFFSET]) && + (new_y <= particletracer + ->offset[i + particletracer->size * YOFFSETEND]) && + i != particletracer->rank) { + buff[i][particleBufIndex[i]].x = new_x; + buff[i][particleBufIndex[i]].y = new_y; + buff[i][particleBufIndex[i]].flag = true; + ++particleBufIndex[i]; + } + } +#endif + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + + int i_new = new_x / dx, j_new = new_y / dy; + int iOffset = particletracer->xOffset / dx, + jOffset = particletracer->yOffset / dy; + + if (S(i_new - iOffset, j_new - jOffset) != FLUID) { + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + } + } + +#ifdef _MPI + for (int i = 0; i < particletracer->size; ++i) { + if (i != comm->rank) { + MPI_Send(&particleBufIndex[i], 1, MPI_INT, i, 0, comm->comm); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Recv(&recvparticleBufIndex[i], + 1, + MPI_INT, + i, + 0, + comm->comm, + MPI_STATUS_IGNORE); + + // if (0 !=recvparticleBufIndex[i]) { + // printf("Rank %d will receive %d particles from rank %d\n", + // particletracer->rank, + // recvparticleBufIndex[i], + // i); + // } + } + } + + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Send(buff[i], + particleBufIndex[i], + particletracer->mpi_particle, + i, + 0, + comm->comm); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Recv(recvbuff[i], + recvparticleBufIndex[i], + particletracer->mpi_particle, + i, + 0, + comm->comm, + MPI_STATUS_IGNORE); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + for (int j = 0; j < recvparticleBufIndex[i]; ++j) { + particletracer->particlePool[particletracer->pointer].x = recvbuff[i][j] + .x; + particletracer->particlePool[particletracer->pointer].y = recvbuff[i][j] + .y; + particletracer->particlePool[particletracer->pointer].flag = true; + ++(particletracer->pointer); + ++(particletracer->totalParticles); + } + } + } +#endif +} + +void freeParticles(ParticleTracer* particletracer) +{ + free(particletracer->particlePool); + free(particletracer->linSpaceLine); + free(particletracer->offset); +} + +void writeParticles(ParticleTracer* particletracer, Comm* comm) +{ + int collectedBuffIndex[particletracer->size]; + compress(particletracer); +#ifdef _MPI + + MPI_Gather(&particletracer->totalParticles, + 1, + MPI_INT, + collectedBuffIndex, + 1, + MPI_INT, + 0, + comm->comm); + + if (particletracer->rank != 0) { + Particle buff[particletracer->totalParticles]; + for (int i = 0; i < particletracer->totalParticles; ++i) { + buff[i].x = particletracer->particlePool[i].x; + buff[i].y = particletracer->particlePool[i].y; + buff[i].flag = particletracer->particlePool[i].flag; + // printf("Rank : %d sending to rank 0 X : %.2f, Y : %.2f with totalpt : + // %d\n", particletracer->rank, buff[i].x, buff[i].y, + // particletracer->totalParticles); + } + MPI_Send(buff, + particletracer->totalParticles, + particletracer->mpi_particle, + 0, + 1, + comm->comm); + } +#endif + + if (particletracer->rank == 0) { + char filename[50]; + FILE* fp; + + snprintf(filename, 50, "vis_files/particles_%d.dat", ts); + fp = fopen(filename, "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + // fprintf(fp, "# vtk DataFile Version 3.0\n"); + // fprintf(fp, "PAMPI cfd solver particle tracing file\n"); + // fprintf(fp, "ASCII\n"); + + // fprintf(fp, "DATASET UNSTRUCTURED_GRID\n"); + // fprintf(fp, "FIELD FieldData 2\n"); + // fprintf(fp, "TIME 1 1 double\n"); + // fprintf(fp, "%d\n", ts); + // fprintf(fp, "CYCLE 1 1 int\n"); + // fprintf(fp, "1\n"); + +#ifdef _MPI + int overallTotalParticles = sum(collectedBuffIndex, particletracer->size); + + // fprintf(fp, "POINTS %d float\n", overallTotalParticles); + + // printf("Total particles : %d\n", overallTotalParticles); + for (int i = 1; i < particletracer->size; ++i) { + Particle recvBuff[collectedBuffIndex[i]]; + MPI_Recv(&recvBuff, + collectedBuffIndex[i], + particletracer->mpi_particle, + i, + 1, + comm->comm, + MPI_STATUS_IGNORE); + + for (int j = 0; j < collectedBuffIndex[i]; ++j) { + double x = recvBuff[j].x; + double y = recvBuff[j].y; + fprintf(fp, "%f %f\n", x, y); + + // printf("Rank : 0 receiving from rank %d X : %.2f, Y : %.2f with totalpt + // : %d\n", i, x, y, particletracer->totalParticles); + } + } +#else + int overallTotalParticles = particletracer->totalParticles; + + // fprintf(fp, "POINTS %d float\n", overallTotalParticles); + + // printf("Total particles : %d\n", overallTotalParticles); +#endif + for (int i = 0; i < particletracer->totalParticles; ++i) { + double x = particletracer->particlePool[i].x; + double y = particletracer->particlePool[i].y; + fprintf(fp, "%f %f\n", x, y); + } + + // fprintf(fp, "CELLS %d %d\n", overallTotalParticles, 2 * overallTotalParticles); + + // for (int i = 0; i < overallTotalParticles; ++i) + // { + // fprintf(fp, "1 %d\n", i); + // } + + // fprintf(fp, "CELL_TYPES %d\n", overallTotalParticles); + + // for (int i = 0; i < overallTotalParticles; ++i) + // { + // fprintf(fp, "1\n"); + // } + + fclose(fp); + } + + ++ts; +} + +void printParticleTracerParameters(ParticleTracer* particletracer) +{ + printf("Particle Tracing data:\n"); + printf("Rank : %d\n", particletracer->rank); + printf("\tNumber of particles : %d being injected for every period of %.2f\n", + particletracer->numberOfParticles, + particletracer->injectTimePeriod); + printf("\tstartTime : %.2f\n", particletracer->startTime); + printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : " + "%.2f, x2 : %.2f, y2 : %.2f\n", + particletracer->x1, + particletracer->y1, + particletracer->x2, + particletracer->y2); + printf("\tPointer : %d, TotalParticles : %d\n", + particletracer->pointer, + particletracer->totalParticles); + printf("\tdt : %.2f, dx : %.2f, dy : %.2f\n", + particletracer->dt, + particletracer->dx, + particletracer->dy); + printf("\txOffset : %.2f, yOffset : %.2f\n", + particletracer->xOffset, + particletracer->yOffset); + printf("\txOffsetEnd : %.2f, yOffsetEnd : %.2f\n", + particletracer->xOffsetEnd, + particletracer->yOffsetEnd); + printf("\txLocal : %.2f, yLocal : %.2f\n", + particletracer->xLocal, + particletracer->yLocal); +} + +void trace(ParticleTracer* particletracer, Discretization* d, double time) +{ + if (time >= particletracer->startTime) { + if ((time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod) { + injectParticles(particletracer, d->grid.s); + particletracer->lastInjectTime = time; + } + if ((time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod) { + writeParticles(particletracer, &d->comm); + particletracer->lastWriteTime = time; + } + advanceParticles(particletracer, d->u, d->v, d->grid.s, &d->comm, time); + + if (particletracer->removedParticles > (particletracer->totalParticles * 0.2)) { + compress(particletracer); + } + + particletracer->lastUpdateTime = time; + } +} + +void compress(ParticleTracer* particletracer) +{ + Particle* memPool = particletracer->particlePool; + Particle tempPool[particletracer->totalParticles]; + int totalParticles = 0; + + // printf("\nPerforming compression ..."); + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (memPool[i].flag == true) { + tempPool[totalParticles].x = memPool[i].x; + tempPool[totalParticles].y = memPool[i].y; + tempPool[totalParticles].flag = memPool[i].flag; + ++totalParticles; + } + } + + // printf(" remove %d particles\n", particletracer->totalParticles - totalParticles); + + particletracer->totalParticles = totalParticles; + particletracer->removedParticles = 0; + particletracer->pointer = totalParticles; + + memcpy(particletracer->particlePool, tempPool, totalParticles * sizeof(Particle)); +} + +void particleRandomizer(ParticleTracer* particletracer) +{ + memset(particletracer->linSpaceLine, + 0, + sizeof(Particle) * particletracer->numberOfParticles); + + for (int i = 0; i < particletracer->numberOfParticles; ++i) { + + particletracer->linSpaceLine[i].x = (((double)rand() / RAND_MAX) * + (particletracer->x2 - + particletracer->x1)) + + particletracer->x1; + particletracer->linSpaceLine[i].y = (((double)rand() / RAND_MAX) * + (particletracer->y2 - + particletracer->y1)) + + particletracer->y1; + particletracer->linSpaceLine[i].flag = true; + } +} \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/particletracing.h b/EnhancedSolver/2D-mpi/src/particletracing.h new file mode 100644 index 0000000..b771014 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/particletracing.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARTICLETRACING_H_ +#define __PARTICLETRACING_H_ +#include "allocate.h" +#include "parameter.h" + +#include "solver.h" +#include +#include + +#define NDIMS 2 + +typedef enum COORD { X = 0, Y, NCOORD } COORD; + +typedef struct { + double x, y; + bool flag; +} Particle; + +typedef struct { + int numberOfParticles, removedParticles, totalParticles; + double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime, + lastWriteTime; + + int estimatedNumParticles; + + double dx, dy, dt; + Particle* linSpaceLine; + Particle* particlePool; + + int pointer; + + double imax, jmax, xlength, ylength, imaxLocal, jmaxLocal; + + double x1, y1, x2, y2; + + int size, rank; + +#ifdef _MPI + MPI_Datatype mpi_particle; +#endif + + double xLocal, yLocal, xOffset, yOffset, xOffsetEnd, yOffsetEnd; + + double* offset; + +} ParticleTracer; + +extern void initParticleTracer(ParticleTracer*, Discretization*, Parameter*); +extern void injectParticles(ParticleTracer*, double*); +extern void advanceParticles(ParticleTracer*, double*, double*, double*, Comm*, double); +extern void freeParticles(ParticleTracer*); +extern void writeParticles(ParticleTracer*, Comm*); +extern void printParticleTracerParameters(ParticleTracer*); +extern void printParticles(ParticleTracer*); +extern void trace(ParticleTracer*, Discretization*, double); +extern void compress(ParticleTracer*); +extern void particleRandomizer(ParticleTracer*); + +#endif \ No newline at end of file diff --git a/EnhancedSolver/2D-mpi/src/progress.c b/EnhancedSolver/2D-mpi/src/progress.c new file mode 100644 index 0000000..78944bd --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/progress.c @@ -0,0 +1,70 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; + +void initProgress(double end) +{ + _end = end; + _current = 0; + + printf("[ ]"); + fflush(stdout); +} + +void printProgress(double current) +{ + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); +} + +void stopProgress() +{ + printf("\n"); + fflush(stdout); +} + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(-1); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} \ No newline at end of file diff --git a/BasicSolver/2D-mpi-v3/src/progress.h b/EnhancedSolver/2D-mpi/src/progress.h similarity index 63% rename from BasicSolver/2D-mpi-v3/src/progress.h rename to EnhancedSolver/2D-mpi/src/progress.h index 9ef2d96..9508797 100644 --- a/BasicSolver/2D-mpi-v3/src/progress.h +++ b/EnhancedSolver/2D-mpi/src/progress.h @@ -1,14 +1,17 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ +#include + #ifndef __PROGRESS_H_ #define __PROGRESS_H_ extern void initProgress(double); extern void printProgress(double); extern void stopProgress(); - +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/EnhancedSolver/2D-mpi/src/solver-mg.c b/EnhancedSolver/2D-mpi/src/solver-mg.c new file mode 100644 index 0000000..920c9e7 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/solver-mg.c @@ -0,0 +1,325 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) + +#define E(i, j) e[(j) * (imaxLocal + 2) + (i)] +#define R(i, j) r[(j) * (imaxLocal + 2) + (i)] +#define OLD(i, j) old[(j) * (imaxLocal + 2) + (i)] + +void printSolver(Solver* s, double* grid, char* gridname) +{ + if (0 == s->comm->rank) printf("Grid name : %s", gridname); + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + + for (int i = 0; i < s->comm->size; i++) { + if (i == s->comm->rank) { + sleep(1 * s->comm->rank); + printf("### RANK %d LVL " + "###################################################### #\n ", + s->comm->rank); + for (int j = 0; j < jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < imaxLocal + 2; i++) { + printf("%2.2f ", grid[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + } +} + +static void restrictMG(Solver* s, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double* r = s->r[level + 1]; + double* old = s->r[level]; + +#ifdef _MPI + commExchange(comm, old); +#endif + + for (int j = 1; j < (jmaxLocal / 2) + 1; j++) { + for (int i = 1; i < (imaxLocal / 2) + 1; i++) { + R(i, j) = (OLD(2 * i - 1, 2 * j - 1) + OLD(2 * i, 2 * j - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1) + OLD(2 * i - 1, 2 * j) * 2 + + OLD(2 * i, 2 * j) * 4 + OLD(2 * i + 1, 2 * j) * 2 + + OLD(2 * i - 1, 2 * j + 1) + OLD(2 * i, 2 * j + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1)) / + 16.0; + } + } +} + +static void prolongate(Solver* s, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double* old = s->r[level + 1]; + double* e = s->r[level]; + + for (int j = 2; j < jmaxLocal + 1; j += 2) { + for (int i = 2; i < imaxLocal + 1; i += 2) { + E(i, j) = OLD(i / 2, j / 2); + } + } +} + +static void correct(Solver* s, double* p, int level, Comm* comm) +{ + double* e = s->e[level]; + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + for (int j = 1; j < jmaxLocal + 1; ++j) { + for (int i = 1; i < imaxLocal + 1; ++i) { + P(i, j) += E(i, j); + } + } +} + +static void setBoundaryCondition(Solver* s, double* p, int imaxLocal, int jmaxLocal) +{ +#ifdef _MPI + if (commIsBoundary(s->comm, B)) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (commIsBoundary(s->comm, T)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (commIsBoundary(s->comm, L)) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (commIsBoundary(s->comm, R)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } +#else + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + P(imaxLocal + 1, j) = P(imaxLocal, j); + } +#endif +} + +static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + +#ifdef _MPI + commExchange(comm, p); +#endif + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + P(i, j) -= factor * + (RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + +#ifdef _MPI + commExchange(comm, p); +#endif + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + R(i, j) = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + res += (R(i, j) * R(i, j)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + +#ifdef _MPI + commReduction(&res, SUM); +#endif + + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + return res; +} + +static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, comm); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, comm); + + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + } + + // calculate residuals + res = calculateResidual(s, p, rhs, level, comm); + + // restrict + restrictMG(s, level, comm); + + Comm newcomm; + commUpdateDatatypes(s->comm, &newcomm, comm->imaxLocal / 2, comm->jmaxLocal / 2); + + // MGSolver on residual and error. + multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm); + + commFreeCommunicator(&newcomm); + + // prolongate + prolongate(s, level, comm); + + // correct p on finer level using residual + correct(s, p, level, comm); + + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, comm->imaxLocal, comm->jmaxLocal); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->comm = &d->comm; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int levels = s->levels; + printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * sizeof(double); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size); + s->e[j] = allocate(64, size); + + for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->comm); + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Residuum: %.6f\n", res); + } +#endif +return res; +} diff --git a/EnhancedSolver/2D-mpi/src/solver-rb.c b/EnhancedSolver/2D-mpi/src/solver-rb.c new file mode 100644 index 0000000..7d67075 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/solver-rb.c @@ -0,0 +1,131 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "discretization.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +void printSolver(Solver* s, double* grid, char* gridname) +{ + printf("Grid name : %s", gridname); + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + + for (int i = 0; i < s->comm->size; i++) { + if (i == s->comm->rank) { + sleep(1 * s->comm->rank); + printf("### RANK %d LVL " + "###################################################### #\n ", + s->comm->rank); + for (int j = 0; j < jmaxLocal + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < imaxLocal + 2; i++) { + printf("%2.2f ", grid[j * (imaxLocal + 2) + i]); + } + printf("\n"); + } + fflush(stdout); + } + } +} + + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->comm = &d->comm; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double epssq = eps * eps; + int pass, jsw, isw; + int it = 0; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + jsw = 1; + for (pass = 0; pass < 2; pass++) { + isw = jsw; + commExchange(s->comm, p); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + + if (commIsBoundary(s->comm, B)) { // set bottom bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0) = P(i, 1); + } + } + + if (commIsBoundary(s->comm, T)) { // set top bc + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + } + + if (commIsBoundary(s->comm, L)) { // set left bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + } + } + + if (commIsBoundary(s->comm, R)) { // set right bc + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j) = P(imaxLocal, j); + } + } + + commReduction(&res, SUM); + res = res / (double)(imax * jmax); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + it++; + } + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + +return res; +} diff --git a/EnhancedSolver/2D-mpi/src/solver.h b/EnhancedSolver/2D-mpi/src/solver.h new file mode 100644 index 0000000..fd47ff1 --- /dev/null +++ b/EnhancedSolver/2D-mpi/src/solver.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "comm.h" +#include "discretization.h" +#include "grid.h" +#include "mpi.h" +#include "parameter.h" + +typedef struct { + /* geometry and grid information */ + Grid* grid; + /* parameters */ + double eps, omega; + int itermax; + int levels, presmooth, postsmooth; + double **r, **e; + /* communication */ + Comm* comm; +} Solver; + +extern void initSolver(Solver*, Discretization*, Parameter*); +extern double solve(Solver*, double*, double*); +extern void printSolver(Solver* , double*, char*); +#endif diff --git a/BasicSolver/2D-mpi-v3/src/timing.c b/EnhancedSolver/2D-mpi/src/timing.c similarity index 72% rename from BasicSolver/2D-mpi-v3/src/timing.c rename to EnhancedSolver/2D-mpi/src/timing.c index c4025a4..78b01c4 100644 --- a/BasicSolver/2D-mpi-v3/src/timing.c +++ b/EnhancedSolver/2D-mpi/src/timing.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,18 +7,16 @@ #include #include -double getTimeStamp() +double getTimeStamp(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } -double getTimeResolution() +double getTimeResolution(void) { struct timespec ts; clock_getres(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } - -double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-mpi-v2/src/timing.h b/EnhancedSolver/2D-mpi/src/timing.h similarity index 55% rename from BasicSolver/2D-mpi-v2/src/timing.h rename to EnhancedSolver/2D-mpi/src/timing.h index db95329..ed05a8c 100644 --- a/BasicSolver/2D-mpi-v2/src/timing.h +++ b/EnhancedSolver/2D-mpi/src/timing.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,8 +7,7 @@ #ifndef __TIMING_H_ #define __TIMING_H_ -extern double getTimeStamp(); -extern double getTimeResolution(); -extern double getTimeStamp_(); +extern double getTimeStamp(void); +extern double getTimeResolution(void); #endif // __TIMING_H_ diff --git a/BasicSolver/2D-mpi-v2/src/util.h b/EnhancedSolver/2D-mpi/src/util.h similarity index 57% rename from BasicSolver/2D-mpi-v2/src/util.h rename to EnhancedSolver/2D-mpi/src/util.h index 657b009..ae396bd 100644 --- a/BasicSolver/2D-mpi-v2/src/util.h +++ b/EnhancedSolver/2D-mpi/src/util.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -19,4 +19,12 @@ #define ABS(a) ((a) >= 0 ? (a) : -(a)) #endif +#define P(i, j) p[(j) * (imaxLocal + 2) + (i)] +#define F(i, j) f[(j) * (imaxLocal + 2) + (i)] +#define G(i, j) g[(j) * (imaxLocal + 2) + (i)] +#define U(i, j) u[(j) * (imaxLocal + 2) + (i)] +#define V(i, j) v[(j) * (imaxLocal + 2) + (i)] +#define S(i, j) s[(j) * (imaxLocal + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imaxLocal + 2) + (i)] + #endif // __UTIL_H_ diff --git a/BasicSolver/2D-mpi-v1/surface.plot b/EnhancedSolver/2D-mpi/surface.plot similarity index 100% rename from BasicSolver/2D-mpi-v1/surface.plot rename to EnhancedSolver/2D-mpi/surface.plot diff --git a/BasicSolver/2D-mpi-v3/vector.plot b/EnhancedSolver/2D-mpi/vector.plot similarity index 68% rename from BasicSolver/2D-mpi-v3/vector.plot rename to EnhancedSolver/2D-mpi/vector.plot index 0934ab2..e24b2b2 100644 --- a/BasicSolver/2D-mpi-v3/vector.plot +++ b/EnhancedSolver/2D-mpi/vector.plot @@ -1,5 +1,6 @@ -set terminal png size 1800,768 enhanced font ,12 +set terminal png size 3600,1400 enhanced font ,12 set output 'velocity.png' set datafile separator whitespace +set size ratio -1 plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/BasicSolver/2D-seq-pt/animate.plot b/EnhancedSolver/2D-mpi/vis_files/animate.plot similarity index 60% rename from BasicSolver/2D-seq-pt/animate.plot rename to EnhancedSolver/2D-mpi/vis_files/animate.plot index 6205e77..d54ba32 100644 --- a/BasicSolver/2D-seq-pt/animate.plot +++ b/EnhancedSolver/2D-mpi/vis_files/animate.plot @@ -1,7 +1,10 @@ unset border; unset tics; unset key; -set term gif animate delay 50 +set term gif animate delay 30 set output "trace.gif" -do for [ts=0:23] { +set xrange [0:1] +set yrange [0:1] + +do for [ts=0:120] { plot "particles_".ts.".dat" with points pointtype 7 } unset output diff --git a/EnhancedSolver/2D-mpi/vis_files/backstep_animate.plot b/EnhancedSolver/2D-mpi/vis_files/backstep_animate.plot new file mode 100644 index 0000000..028a26f --- /dev/null +++ b/EnhancedSolver/2D-mpi/vis_files/backstep_animate.plot @@ -0,0 +1,14 @@ +unset border; unset tics; unset key; +set term gif animate delay 10 +set output "trace.gif" +set xrange [0:7] +set yrange [0:1.5] +set size ratio -1 + +set object 1 rect from 0.0,0.0 to 1.0,0.5 lw 5 + + +do for [ts=0:550] { + plot "particles_".ts.".dat" with points pointtype 7 +} +unset output diff --git a/EnhancedSolver/2D-mpi/vis_files/canal_animate.plot b/EnhancedSolver/2D-mpi/vis_files/canal_animate.plot new file mode 100644 index 0000000..bf9ce60 --- /dev/null +++ b/EnhancedSolver/2D-mpi/vis_files/canal_animate.plot @@ -0,0 +1,10 @@ +unset border; unset tics; unset key; +set term gif animate delay 30 +set output "trace.gif" +set xrange [0:30] +set yrange [0:4] + +do for [ts=0:120] { + plot "particles_".ts.".dat" with points pointtype 7 +} +unset output diff --git a/EnhancedSolver/2D-mpi/vis_files/karman_animate.plot b/EnhancedSolver/2D-mpi/vis_files/karman_animate.plot new file mode 100644 index 0000000..d78f0c3 --- /dev/null +++ b/EnhancedSolver/2D-mpi/vis_files/karman_animate.plot @@ -0,0 +1,13 @@ +unset border; unset tics; unset key; +set term gif animate delay 10 +set output "trace.gif" +set xrange [0:30] +set yrange [0:8] +set size ratio -1 +set object 1 circle front at 5.0,4.0 size 1.0 fillcolor rgb "black" lw 2 + + +do for [ts=0:500] { + plot "particles_".ts.".dat" with points pointtype 7 pointsize 0.3 +} +unset output diff --git a/BasicSolver/2D-mpi-v2/Makefile b/EnhancedSolver/2D-seq/Makefile similarity index 71% rename from BasicSolver/2D-mpi-v2/Makefile rename to EnhancedSolver/2D-seq/Makefile index 57f99f4..33c7ad8 100644 --- a/BasicSolver/2D-mpi-v2/Makefile +++ b/EnhancedSolver/2D-seq/Makefile @@ -1,5 +1,5 @@ #======================================================================================= -# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. # All rights reserved. # Use of this source code is governed by a MIT-style # license that can be found in the LICENSE file. @@ -18,9 +18,10 @@ include $(MAKE_DIR)/include_$(TAG).mk INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) VPATH = $(SRC_DIR) -SRC = $(wildcard $(SRC_DIR)/*.c) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) @@ -37,9 +38,9 @@ $(BUILD_DIR)/%.s: %.c $(info ===> GENERATE ASM $@) $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ -.PHONY: clean distclean tags info asm format +.PHONY: clean distclean vis vis_clean tags info asm format -clean: +clean: vis_clean $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @rm -f tags @@ -47,6 +48,10 @@ clean: distclean: clean $(info ===> DIST CLEAN) @rm -f $(TARGET) + @rm -f *.dat + @rm -f *.png + @rm -f ./vis_files/*.dat + @rm -f ./vis_files/*.gif info: $(info $(CFLAGS)) @@ -65,6 +70,19 @@ format: done @echo "Done" +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='pressure.dat'" ./surface.plot + @gnuplot -e "filename='velocity.dat'" ./vector.plot + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.png + @rm -f ./vis_files/*.dat + @rm -f ./vis_files/*.gif + $(BUILD_DIR): @mkdir $(BUILD_DIR) diff --git a/BasicSolver/2D-seq-pt/README.md b/EnhancedSolver/2D-seq/README.md similarity index 100% rename from BasicSolver/2D-seq-pt/README.md rename to EnhancedSolver/2D-seq/README.md diff --git a/EnhancedSolver/2D-seq/backstep.par b/EnhancedSolver/2D-seq/backstep.par new file mode 100644 index 0000000..fb25403 --- /dev/null +++ b/EnhancedSolver/2D-seq/backstep.par @@ -0,0 +1,79 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name backstep # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 36000.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 1.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 7.0 # domain size in x-direction +ylength 1.5 # domain size in y-direction +imax 210 # number of interior cells in x-direction +jmax 45 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 60.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.8 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 0 +injectTimePeriod 1.0 +writeTimePeriod 0.5 + +x1 0.0 +y1 0.5 +x2 0.0 +y2 1.5 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 1 +xCenter 0.0 +yCenter 0.0 +xRectLength 2.0 +yRectLength 1.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/BasicSolver/2D-seq-pt/canal.par b/EnhancedSolver/2D-seq/canal.par similarity index 59% rename from BasicSolver/2D-seq-pt/canal.par rename to EnhancedSolver/2D-seq/canal.par index 2eb04b6..87a797a 100644 --- a/BasicSolver/2D-seq-pt/canal.par +++ b/EnhancedSolver/2D-seq/canal.par @@ -19,15 +19,15 @@ re 100.0 # Reynolds number u_init 1.0 # initial value for velocity in x-direction v_init 0.0 # initial value for velocity in y-direction -p_init 0.0 # initial value for pressure +p_init 1.0 # initial value for pressure # Geometry Data: # ------------- xlength 30.0 # domain size in x-direction -ylength 4.0 # domain size in y-direction +ylength 4.0 # domain size in y-direction imax 200 # number of interior cells in x-direction -jmax 50 # number of interior cells in y-direction +jmax 40 # number of interior cells in y-direction # Time Data: # --------- @@ -40,20 +40,40 @@ tau 0.5 # safety factor for time stepsize control (<0 constant delt) # ----------------------- itermax 500 # maximal number of pressure iteration in one time step -eps 0.00001 # stopping tolerance for pressure iteration +eps 0.0001 # stopping tolerance for pressure iteration +rho 0.52 omg 1.8 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma -# Visualization Data: -# ------------------ +# Multigrid data: +# --------- -traceStart 10.0 # time for starting visualization -traceWrite 2.0 # time stepsize for saving particle data -traceInject 2.0 # time stepsize for particle injection -lineX1 2.0 # Coordinates of line segment for particle injection -lineY1 0.5 -lineX2 2.0 -lineY2 3.5 -nparticles 30 # number of particles to inject +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 60 +startTime 5.0 +injectTimePeriod 4.0 +writeTimePeriod 1.0 + +x1 1.0 +y1 0.0 +x2 1.0 +y2 4.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 10.0 +yCenter 2 +xRectLength 6.0 +yRectLength 1.0 +circleRadius 1.0 #=============================================================================== diff --git a/BasicSolver/2D-mpi-v1/config.mk b/EnhancedSolver/2D-seq/config.mk similarity index 53% rename from BasicSolver/2D-mpi-v1/config.mk rename to EnhancedSolver/2D-seq/config.mk index 46cef95..af8912f 100644 --- a/BasicSolver/2D-mpi-v1/config.mk +++ b/EnhancedSolver/2D-seq/config.mk @@ -1,10 +1,12 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC ENABLE_OPENMP ?= false +# Supported: sor, rb, mg +SOLVER ?= mg +# Run in debug settings +DEBUG ?= false #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 #OPTIONS += -DVERBOSE -#OPTIONS += -DVERBOSE_AFFINITY -#OPTIONS += -DVERBOSE_DATASIZE -#OPTIONS += -DVERBOSE_TIMER +#OPTIONS += -DDEBUG diff --git a/BasicSolver/2D-mpi-v3/dcavity.par b/EnhancedSolver/2D-seq/dcavity.par similarity index 51% rename from BasicSolver/2D-mpi-v3/dcavity.par rename to EnhancedSolver/2D-seq/dcavity.par index b4013d6..2b7b416 100644 --- a/BasicSolver/2D-mpi-v3/dcavity.par +++ b/EnhancedSolver/2D-seq/dcavity.par @@ -17,7 +17,7 @@ gy 0.0 # re 10.0 # Reynolds number -u_init 0.0 # initial value for velocity in x-direction +u_init 1.0 # initial value for velocity in x-direction v_init 0.0 # initial value for velocity in y-direction p_init 0.0 # initial value for pressure @@ -26,21 +26,54 @@ p_init 0.0 # initial value for pressure xlength 1.0 # domain size in x-direction ylength 1.0 # domain size in y-direction -imax 100 # number of interior cells in x-direction -jmax 100 # number of interior cells in y-direction +imax 128 # number of interior cells in x-direction +jmax 128 # number of interior cells in y-direction # Time Data: # --------- -te 5.0 # final time -dt 0.02 # time stepsize -tau 0.5 # safety factor for time stepsize control (<0 constant delt) +te 10.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) # Pressure Iteration Data: # ----------------------- itermax 1000 # maximal number of pressure iteration in one time step eps 0.001 # stopping tolerance for pressure iteration -omg 1.7 # relaxation parameter for SOR iteration +rho 0.5 +omg 1.8 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 2.0 +injectTimePeriod 0.5 +writeTimePeriod 0.2 + +x1 0.1 +y1 0.9 +x2 0.9 +y2 0.9 + + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 0.5 +yCenter 0.5 +xRectLength 0.5 +yRectLength 0.5 +circleRadius 0.5 #=============================================================================== diff --git a/EnhancedSolver/2D-seq/include_CLANG.mk b/EnhancedSolver/2D-seq/include_CLANG.mk new file mode 100644 index 0000000..a124053 --- /dev/null +++ b/EnhancedSolver/2D-seq/include_CLANG.mk @@ -0,0 +1,19 @@ +CC = clang +GCC = cc +LINKER = $(CC) + +ifeq ($(strip $(ENABLE_OPENMP)),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif +ifeq ($(strip $(DEBUG)),true) +CFLAGS = -O0 -g -std=c17 +else +CFLAGS = -O3 -std=c17 $(OPENMP) +endif + +VERSION = --version +LFLAGS = $(OPENMP) -lm +DEFINES = -D_GNU_SOURCE +INCLUDES = diff --git a/BasicSolver/2D-mpi-v1/include_GCC.mk b/EnhancedSolver/2D-seq/include_GCC.mk similarity index 100% rename from BasicSolver/2D-mpi-v1/include_GCC.mk rename to EnhancedSolver/2D-seq/include_GCC.mk diff --git a/BasicSolver/3D-mpi-io/include_ICC.mk b/EnhancedSolver/2D-seq/include_ICC.mk similarity index 95% rename from BasicSolver/3D-mpi-io/include_ICC.mk rename to EnhancedSolver/2D-seq/include_ICC.mk index 94b8e20..d0c2a1b 100644 --- a/BasicSolver/3D-mpi-io/include_ICC.mk +++ b/EnhancedSolver/2D-seq/include_ICC.mk @@ -1,4 +1,4 @@ -CC = icc +CC = icx GCC = gcc LINKER = $(CC) diff --git a/EnhancedSolver/2D-seq/karman.par b/EnhancedSolver/2D-seq/karman.par new file mode 100644 index 0000000..fb61961 --- /dev/null +++ b/EnhancedSolver/2D-seq/karman.par @@ -0,0 +1,79 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name karman # name of flow setup + +bcTop 1 # flags for boundary conditions +bcBottom 1 # 1 = no-slip 3 = outflow +bcLeft 3 # 2 = free-slip 4 = periodic +bcRight 3 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # + +re 5050.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 8.0 # domain size in y-direction +imax 400 # number of interior cells in x-direction +jmax 200 # number of interior cells in y-direction + +# Time Data: +# --------- + +te 150.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 200 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.75 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 50 +injectTimePeriod 1.0 +writeTimePeriod 0.5 + +x1 0.0 +y1 3.8 +x2 0.0 +y2 4.1 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 2 +xCenter 5.0 +yCenter 4.0 +xRectLength 2.0 +yRectLength 1.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/EnhancedSolver/2D-seq/residual.plot b/EnhancedSolver/2D-seq/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/EnhancedSolver/2D-seq/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/2D-mpi-v2/src/allocate.c b/EnhancedSolver/2D-seq/src/allocate.c similarity index 83% rename from BasicSolver/2D-mpi-v2/src/allocate.c rename to EnhancedSolver/2D-seq/src/allocate.c index 81e1e9d..cf2efd6 100644 --- a/BasicSolver/2D-mpi-v2/src/allocate.c +++ b/EnhancedSolver/2D-seq/src/allocate.c @@ -1,14 +1,17 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ #include +#include #include #include -void* allocate(int alignment, size_t bytesize) +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) { int errorCode; void* ptr; diff --git a/BasicSolver/2D-seq-pt/src/allocate.h b/EnhancedSolver/2D-seq/src/allocate.h similarity index 64% rename from BasicSolver/2D-seq-pt/src/allocate.h rename to EnhancedSolver/2D-seq/src/allocate.h index 54cfe06..77f4ba0 100644 --- a/BasicSolver/2D-seq-pt/src/allocate.h +++ b/EnhancedSolver/2D-seq/src/allocate.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -8,6 +8,6 @@ #define __ALLOCATE_H_ #include -extern void* allocate(int alignment, size_t bytesize); +extern void* allocate(size_t alignment, size_t bytesize); #endif diff --git a/EnhancedSolver/2D-seq/src/discretization.c b/EnhancedSolver/2D-seq/src/discretization.c new file mode 100644 index 0000000..daa346d --- /dev/null +++ b/EnhancedSolver/2D-seq/src/discretization.c @@ -0,0 +1,649 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "discretization.h" +#include "grid.h" +#include "parameter.h" +#include "util.h" + +#define S(i, j) s[(j) * (imax + 2) + (i)] + +static double distance(double i, double j, double iCenter, double jCenter) +{ + return sqrt(pow(iCenter - i, 2) + pow(jCenter - j, 2) * 1.0); +} + +void print(Discretization* d, double* grid) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + + for (int j = 0; j < jmax + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < imax + 2; i++) { + printf("%3.2f ", grid[j * (imax + 2) + i]); + } + printf("\n"); + } + fflush(stdout); +} + +void printGrid(Discretization* d, int* grid) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + + for (int j = 0; j < jmax + 2; j++) { + printf("%02d: ", j); + for (int i = 0; i < imax + 2; i++) { + printf("%2d ", grid[j * (imax + 2) + i]); + } + printf("\n"); + } + fflush(stdout); +} + +static void printConfig(Discretization* d) +{ + printf("Parameters for #%s#\n", d->problem); + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + d->bcLeft, + d->bcRight, + d->bcBottom, + d->bcTop); + printf("\tReynolds number: %.2f\n", d->re); + printf("\tGx Gy: %.2f %.2f\n", d->gx, d->gy); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y): %.2f, %.2f\n", d->grid.xlength, d->grid.ylength); + printf("\tCells (x, y): %d, %d\n", d->grid.imax, d->grid.jmax); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te); + printf("\tdt bound: %.6f\n", d->dtBound); + printf("\tTau factor: %.2f\n", d->tau); + printf("Iterative d parameters:\n"); + printf("\tgamma factor: %f\n", d->gamma); +} + +void initDiscretization(Discretization* d, Parameter* p) +{ + d->problem = p->name; + d->bcLeft = p->bcLeft; + d->bcRight = p->bcRight; + d->bcBottom = p->bcBottom; + d->bcTop = p->bcTop; + d->grid.imax = p->imax; + d->grid.jmax = p->jmax; + d->grid.xlength = p->xlength; + d->grid.ylength = p->ylength; + d->grid.dx = p->xlength / p->imax; + d->grid.dy = p->ylength / p->jmax; + d->re = p->re; + d->gx = p->gx; + d->gy = p->gy; + d->dt = p->dt; + d->te = p->te; + d->tau = p->tau; + d->gamma = p->gamma; + + int imax = d->grid.imax; + int jmax = d->grid.jmax; + size_t size = (imax + 2) * (jmax + 2) * sizeof(double); + d->u = allocate(64, size); + d->v = allocate(64, size); + d->grid.s = allocate(64, size); + d->p = allocate(64, size); + d->rhs = allocate(64, size); + d->f = allocate(64, size); + d->g = allocate(64, size); + + for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { + + d->u[i] = p->u_init; + d->v[i] = p->v_init; + d->p[i] = p->p_init; + d->rhs[i] = 0.0; + d->f[i] = 0.0; + d->g[i] = 0.0; + d->grid.s[i] = FLUID; + } + + double dx = d->grid.dx; + double dy = d->grid.dy; + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy); + d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; + + double xCenter = 0, yCenter = 0, radius = 0; + double x1 = 0, x2 = 0, y1 = 0, y2 = 0; + + int* s = d->grid.s; + + switch (p->shape) { + case NOSHAPE: + break; + case RECT: + x1 = p->xCenter - p->xRectLength / 2; + x2 = p->xCenter + p->xRectLength / 2; + y1 = p->yCenter - p->yRectLength / 2; + y2 = p->yCenter + p->yRectLength / 2; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + if ((x1 <= (i * dx)) && ((i * dx) <= x2) && (y1 <= (j * dy)) && + ((j * dy) <= y2)) { + S(i, j) = OBSTACLE; + } + } + } + break; + case CIRCLE: + xCenter = p->xCenter; + yCenter = p->yCenter; + radius = p->circleRadius; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + if (distance((i * dx), (j * dy), xCenter, yCenter) <= radius) { + S(i, j) = OBSTACLE; + } + } + } + break; + } + + if (p->shape != NOSHAPE) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + + if (S(i, j - 1) == FLUID && S(i, j + 1) == OBSTACLE && + S(i, j) == OBSTACLE) + S(i, j) = BOTTOM; // TOP + if (S(i - 1, j) == FLUID && S(i + 1, j) == OBSTACLE && + S(i, j) == OBSTACLE) + S(i, j) = LEFT; // LEFT + if (S(i + 1, j) == FLUID && S(i - 1, j) == OBSTACLE && + S(i, j) == OBSTACLE) + S(i, j) = RIGHT; // RIGHT + if (S(i, j + 1) == FLUID && S(i, j - 1) == OBSTACLE && + S(i, j) == OBSTACLE) + S(i, j) = TOP; // BOTTOM + if (S(i - 1, j - 1) == FLUID && S(i, j - 1) == FLUID && + S(i - 1, j) == FLUID && S(i + 1, j + 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == LEFT || S(i, j) == BOTTOM)) + S(i, j) = BOTTOMLEFT; // TOPLEFT + if (S(i + 1, j - 1) == FLUID && S(i, j - 1) == FLUID && + S(i + 1, j) == FLUID && S(i - 1, j + 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == RIGHT || S(i, j) == BOTTOM)) + S(i, j) = BOTTOMRIGHT; // TOPRIGHT + if (S(i - 1, j + 1) == FLUID && S(i - 1, j) == FLUID && + S(i, j + 1) == FLUID && S(i + 1, j - 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == LEFT || S(i, j) == TOP)) + S(i, j) = TOPLEFT; // BOTTOMLEFT + if (S(i + 1, j + 1) == FLUID && S(i + 1, j) == FLUID && + S(i, j + 1) == FLUID && S(i - 1, j - 1) == OBSTACLE && + (S(i, j) == OBSTACLE || S(i, j) == RIGHT || S(i, j) == TOP)) + S(i, j) = TOPRIGHT; // BOTTOMRIGHT + } + } + } + +#ifdef VERBOSE + printConfig(solver); +#endif +} + +static double maxElement(Discretization* d, double* m) +{ + int size = (d->grid.imax + 2) * (d->grid.jmax + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + return maxval; +} + +void computeRHS(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double idx = 1.0 / d->grid.dx; + double idy = 1.0 / d->grid.dy; + double idt = 1.0 / d->dt; + double* rhs = d->rhs; + double* f = d->f; + double* g = d->g; + int* s = d->grid.s; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + RHS(i, j) = idt * + ((F(i, j) - F(i - 1, j)) * idx + (G(i, j) - G(i, j - 1)) * idy); + } + } +} + +void normalizePressure(Discretization* d) +{ + int size = (d->grid.imax + 2) * (d->grid.jmax + 2); + double* p = d->p; + double avgP = 0.0; + + for (int i = 0; i < size; i++) { + avgP += p[i]; + } + avgP /= size; + + for (int i = 0; i < size; i++) { + p[i] = p[i] - avgP; + } +} + +void computeTimestep(Discretization* d) +{ + double dt = d->dtBound; + double dx = d->grid.dx; + double dy = d->grid.dy; + double umax = maxElement(d, d->u); + double vmax = maxElement(d, d->v); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + + d->dt = dt * d->tau; +} + +void setBoundaryConditions(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double* u = d->u; + double* v = d->v; + + // Left boundary + switch (d->bcLeft) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + U(0, j) = 0.0; + V(0, j) = -V(1, j); + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + U(0, j) = 0.0; + V(0, j) = V(1, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + U(0, j) = U(1, j); + V(0, j) = V(1, j); + } + break; + case PERIODIC: + break; + } + + // Right boundary + switch (d->bcRight) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + U(imax, j) = 0.0; + V(imax + 1, j) = -V(imax, j); + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + U(imax, j) = 0.0; + V(imax + 1, j) = V(imax, j); + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + U(imax, j) = U(imax - 1, j); + V(imax + 1, j) = V(imax, j); + } + break; + case PERIODIC: + break; + } + + // Bottom boundary + switch (d->bcBottom) { + case NOSLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = -U(i, 1); + } + break; + case SLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, 0) = 0.0; + U(i, 0) = U(i, 1); + } + break; + case OUTFLOW: + for (int i = 1; i < imax + 1; i++) { + U(i, 0) = U(i, 1); + V(i, 0) = V(i, 1); + } + break; + case PERIODIC: + break; + } + + // Top boundary + switch (d->bcTop) { + case NOSLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, jmax) = 0.0; + U(i, jmax + 1) = -U(i, jmax); + } + break; + case SLIP: + for (int i = 1; i < imax + 1; i++) { + V(i, jmax) = 0.0; + U(i, jmax + 1) = U(i, jmax); + } + break; + case OUTFLOW: + for (int i = 1; i < imax + 1; i++) { + U(i, jmax + 1) = U(i, jmax); + V(i, jmax) = V(i, jmax - 1); + } + break; + case PERIODIC: + break; + } +} + +void setSpecialBoundaryCondition(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double mDy = d->grid.dy; + double* u = d->u; + int* s = d->grid.s; + + if (strcmp(d->problem, "dcavity") == 0) { + for (int i = 1; i < imax; i++) { + U(i, jmax + 1) = 2.0 - U(i, jmax); + } + } else if (strcmp(d->problem, "canal") == 0) { + double ylength = d->grid.ylength; + double y; + + for (int j = 1; j < jmax + 1; j++) { + y = mDy * (j - 0.5); + U(0, j) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } else if (strcmp(d->problem, "backstep") == 0) { + for (int j = 1; j < jmax + 1; j++) { + if (S(0, j) == FLUID) U(0, j) = 1.0; + } + } else if (strcmp(d->problem, "karman") == 0) { + for (int j = 1; j < jmax + 1; j++) { + U(0, j) = 1.0; + } + } +} + +void setObjectBoundaryCondition(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double* u = d->u; + double* v = d->v; + int* s = d->grid.s; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + switch (S(i, j)) { + case TOP: + U(i, j) = -U(i, j + 1); + U(i - 1, j) = -U(i - 1, j + 1); + V(i, j) = 0.0; + break; + case BOTTOM: + U(i, j) = -U(i, j - 1); + U(i - 1, j) = -U(i - 1, j - 1); + V(i, j) = 0.0; + break; + case LEFT: + U(i - 1, j) = 0.0; + V(i, j) = -V(i - 1, j); + V(i, j - 1) = -V(i - 1, j - 1); + break; + case RIGHT: + U(i, j) = 0.0; + V(i, j) = -V(i + 1, j); + V(i, j - 1) = -V(i + 1, j - 1); + break; + case TOPLEFT: + U(i, j) = -U(i, j + 1); + U(i - 1, j) = 0.0; + V(i, j) = 0.0; + V(i, j - 1) = -V(i - 1, j - 1); + break; + case TOPRIGHT: + U(i, j) = 0.0; + U(i - 1, j) = -U(i - 1, j + 1); + V(i, j) = 0.0; + V(i, j - 1) = -V(i + 1, j - 1); + break; + case BOTTOMLEFT: + U(i, j) = -U(i, j - 1); + U(i - 1, j) = 0.0; + V(i, j) = -V(i - 1, j); + V(i, j - 1) = 0.0; + break; + case BOTTOMRIGHT: + U(i, j) = 0.0; + U(i - 1, j) = -U(i - 1, j - 1); + V(i, j) = -V(i, j + 1); + V(i, j - 1) = 0.0; + break; + } + } + } +} + +void computeFG(Discretization* d) +{ + double* u = d->u; + double* v = d->v; + double* f = d->f; + double* g = d->g; + int* s = d->grid.s; + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double gx = d->gx; + double gy = d->gy; + double gamma = d->gamma; + double dt = d->dt; + double inverseRe = 1.0 / d->re; + double inverseDx = 1.0 / d->grid.dx; + double inverseDy = 1.0 / d->grid.dy; + double du2dx, dv2dy, duvdx, duvdy; + double du2dx2, du2dy2, dv2dx2, dv2dy2; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + if (S(i, j) == FLUID) { + du2dx = inverseDx * 0.25 * + ((U(i, j) + U(i + 1, j)) * (U(i, j) + U(i + 1, j)) - + (U(i, j) + U(i - 1, j)) * (U(i, j) + U(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i + 1, j)) * (U(i, j) - U(i + 1, j)) + + fabs(U(i, j) + U(i - 1, j)) * (U(i, j) - U(i - 1, j))); + + duvdy = inverseDy * 0.25 * + ((V(i, j) + V(i + 1, j)) * (U(i, j) + U(i, j + 1)) - + (V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) + U(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i + 1, j)) * (U(i, j) - U(i, j + 1)) + + fabs(V(i, j - 1) + V(i + 1, j - 1)) * + (U(i, j) - U(i, j - 1))); + + du2dx2 = inverseDx * inverseDx * + (U(i + 1, j) - 2.0 * U(i, j) + U(i - 1, j)); + du2dy2 = inverseDy * inverseDy * + (U(i, j + 1) - 2.0 * U(i, j) + U(i, j - 1)); + F(i, j) = U(i, j) + + dt * (inverseRe * (du2dx2 + du2dy2) - du2dx - duvdy + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j) + U(i, j + 1)) * (V(i, j) + V(i + 1, j)) - + (U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) + V(i - 1, j))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j) + U(i, j + 1)) * (V(i, j) - V(i + 1, j)) + + fabs(U(i - 1, j) + U(i - 1, j + 1)) * + (V(i, j) - V(i - 1, j))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j) + V(i, j + 1)) * (V(i, j) + V(i, j + 1)) - + (V(i, j) + V(i, j - 1)) * (V(i, j) + V(i, j - 1))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j) + V(i, j + 1)) * (V(i, j) - V(i, j + 1)) + + fabs(V(i, j) + V(i, j - 1)) * (V(i, j) - V(i, j - 1))); + + dv2dx2 = inverseDx * inverseDx * + (V(i + 1, j) - 2.0 * V(i, j) + V(i - 1, j)); + dv2dy2 = inverseDy * inverseDy * + (V(i, j + 1) - 2.0 * V(i, j) + V(i, j - 1)); + G(i, j) = V(i, j) + + dt * (inverseRe * (dv2dx2 + dv2dy2) - duvdx - dv2dy + gy); + } else { + switch (S(i, j)) { + case TOP: + G(i, j) = V(i, j); + break; + case BOTTOM: + G(i, j - 1) = V(i, j - 1); + break; + case LEFT: + F(i - 1, j) = U(i - 1, j); + break; + case RIGHT: + F(i, j) = U(i, j); + break; + case TOPLEFT: + F(i - 1, j) = U(i - 1, j); + G(i, j) = V(i, j); + break; + case TOPRIGHT: + F(i, j) = U(i, j); + G(i, j) = V(i, j); + break; + case BOTTOMLEFT: + F(i - 1, j) = U(i - 1, j); + G(i, j - 1) = V(i, j - 1); + break; + case BOTTOMRIGHT: + F(i, j) = U(i, j); + G(i, j - 1) = V(i, j - 1); + break; + } + } + } + } + + /* ---------------------- boundary of F --------------------------- */ + for (int j = 1; j < jmax + 1; j++) { + F(0, j) = U(0, j); + F(imax, j) = U(imax, j); + } + + /* ---------------------- boundary of G --------------------------- */ + for (int i = 1; i < imax + 1; i++) { + G(i, 0) = V(i, 0); + G(i, jmax) = V(i, jmax); + } +} + +void adaptUV(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double* p = d->p; + double* u = d->u; + double* v = d->v; + double* f = d->f; + double* g = d->g; + double factorX = d->dt / d->grid.dx; + double factorY = d->dt / d->grid.dy; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j) = F(i, j) - (P(i + 1, j) - P(i, j)) * factorX; + V(i, j) = G(i, j) - (P(i, j + 1) - P(i, j)) * factorY; + } + } +} + +void writeResult(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + double dx = d->grid.dx; + double dy = d->grid.dy; + double* p = d->p; + double* u = d->u; + double* v = d->v; + double x = 0.0, y = 0.0; + + FILE* fp; + fp = fopen("pressure.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax + 1; j++) { + y = (double)(j - 0.5) * dy; + for (int i = 1; i < imax + 1; i++) { + x = (double)(i - 0.5) * dx; + fprintf(fp, "%.2f %.2f %f\n", x, y, P(i, j)); + } + fprintf(fp, "\n"); + } + + fclose(fp); + + fp = fopen("velocity.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int j = 1; j < jmax + 1; j++) { + y = dy * (j - 0.5); + for (int i = 1; i < imax + 1; i++) { + x = dx * (i - 0.5); + double velU = (U(i, j) + U(i - 1, j)) / 2.0; + double velV = (V(i, j) + V(i, j - 1)) / 2.0; + double len = sqrt((velU * velU) + (velV * velV)); + fprintf(fp, "%.2f %.2f %f %f %f\n", x, y, velU, velV, len); + } + } + + fclose(fp); +} diff --git a/EnhancedSolver/2D-seq/src/discretization.h b/EnhancedSolver/2D-seq/src/discretization.h new file mode 100644 index 0000000..1b8d92d --- /dev/null +++ b/EnhancedSolver/2D-seq/src/discretization.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ +#include "grid.h" +#include "parameter.h" + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g; + double *u, *v; + /* parameters */ + double rho; + double re, tau, gamma; + double gx, gy; + /* time stepping */ + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop; +} Discretization; + +extern void initDiscretization(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void setObjectBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +extern void writeResult(Discretization*); +extern void print(Discretization*, double*); +extern void printGrid(Discretization*, int*); +#endif diff --git a/EnhancedSolver/2D-seq/src/grid.h b/EnhancedSolver/2D-seq/src/grid.h new file mode 100644 index 0000000..14c4f44 --- /dev/null +++ b/EnhancedSolver/2D-seq/src/grid.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __GRID_H_ +#define __GRID_H_ + +#define S(i, j) s[(j) * (imax + 2) + (i)] + +enum OBJECTBOUNDARY { + FLUID = 0, + TOP, + BOTTOM, + LEFT, + RIGHT, + TOPLEFT, + BOTTOMLEFT, + TOPRIGHT, + BOTTOMRIGHT, + OBSTACLE +}; + +enum SHAPE { NOSHAPE = 0, RECT, CIRCLE }; + +typedef struct { + double dx, dy; + int imax, jmax; + double xlength, ylength; + int* s; +} Grid; + +static inline int gridIsFluid(Grid* g, int i, int j) +{ + return g->s[j * (g->imax + 2) + i] == FLUID; +} +#endif // __GRID_H_ diff --git a/BasicSolver/2D-mpi-v1/src/likwid-marker.h b/EnhancedSolver/2D-seq/src/likwid-marker.h similarity index 100% rename from BasicSolver/2D-mpi-v1/src/likwid-marker.h rename to EnhancedSolver/2D-seq/src/likwid-marker.h diff --git a/EnhancedSolver/2D-seq/src/main.c b/EnhancedSolver/2D-seq/src/main.c new file mode 100644 index 0000000..34deea4 --- /dev/null +++ b/EnhancedSolver/2D-seq/src/main.c @@ -0,0 +1,87 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "discretization.h" +#include "parameter.h" +#include "particletracing.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" + +int main(int argc, char** argv) +{ + double timeStart, timeStop; + Parameter p; + Discretization d; + Solver s; + ParticleTracer particletracer; + + initParameter(&p); + FILE* fp; + fp = initResidualWriter(); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(&p, argv[1]); + printParameter(&p); + initDiscretization(&d, &p); + initSolver(&s, &d, &p); + initParticleTracer(&particletracer, &d.grid, &p); + printParticleTracerParameters(&particletracer); + +#ifndef VERBOSE + initProgress(d.te); +#endif + + double tau = d.tau; + double te = d.te; + double t = 0.0; + int nt = 0; + double res = 0.0; + + timeStart = getTimeStamp(); + + while (t <= te) { + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + setObjectBoundaryCondition(&d); + + computeFG(&d); + computeRHS(&d); + if (nt % 100 == 0) normalizePressure(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + trace(&particletracer, d.u, d.v, d.dt, t); + + writeResidual(fp, t, res); + + t += d.dt; + nt++; + +#ifdef VERBOSE + printf("TIME %f , TIMESTEP %f\n", t, solver.dt); +#else + printProgress(t); +#endif + } + + timeStop = getTimeStamp(); + + fclose(fp); + stopProgress(); + freeParticles(&particletracer); + printf("Solution took %.2fs\n", timeStop - timeStart); + writeResult(&d); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/2D-mpi-v3/src/parameter.c b/EnhancedSolver/2D-seq/src/parameter.c similarity index 63% rename from BasicSolver/2D-mpi-v3/src/parameter.c rename to EnhancedSolver/2D-seq/src/parameter.c index d691627..48c3181 100644 --- a/BasicSolver/2D-mpi-v3/src/parameter.c +++ b/EnhancedSolver/2D-seq/src/parameter.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -14,16 +14,20 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->imax = 100; - param->jmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.7; - param->re = 100.0; - param->gamma = 0.9; - param->tau = 0.5; + param->xlength = 1.0; + param->ylength = 1.0; + param->imax = 100; + param->jmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; + param->rho = 0.99; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) @@ -61,6 +65,9 @@ void readParameter(Parameter* param, const char* filename) PARSE_INT(imax); PARSE_INT(jmax); PARSE_INT(itermax); + PARSE_INT(levels); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); PARSE_REAL(eps); PARSE_REAL(omg); PARSE_REAL(re); @@ -78,6 +85,25 @@ void readParameter(Parameter* param, const char* filename) PARSE_REAL(u_init); PARSE_REAL(v_init); PARSE_REAL(p_init); + PARSE_REAL(rho); + + /* Added new particle tracing parameters */ + PARSE_INT(numberOfParticles); + PARSE_REAL(startTime); + PARSE_REAL(injectTimePeriod); + PARSE_REAL(writeTimePeriod); + PARSE_REAL(x1); + PARSE_REAL(y1); + PARSE_REAL(x2); + PARSE_REAL(y2); + + /* Added obstacle geometry parameters */ + PARSE_INT(shape); + PARSE_REAL(xCenter); + PARSE_REAL(yCenter); + PARSE_REAL(xRectLength); + PARSE_REAL(yRectLength); + PARSE_REAL(circleRadius); } } @@ -108,4 +134,18 @@ void printParameter(Parameter* param) printf("\tepsilon (stopping tolerance) : %f\n", param->eps); printf("\tgamma (stopping tolerance) : %f\n", param->gamma); printf("\tomega (SOR relaxation): %f\n", param->omg); + printf("\trho (SOR relaxation): %f\n", param->rho); + printf("\tMultiGrid levels : %d\n", param->levels); + + printf("Particle Tracing data:\n"); + printf("\tNumber of particles : %d being injected for every period of %.2f\n", + param->numberOfParticles, + param->injectTimePeriod); + printf("\tstartTime : %.2f\n", param->startTime); + printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : " + "%.2f, x2 : %.2f, y2 : %.2f\n", + param->x1, + param->y1, + param->x2, + param->y2); } diff --git a/BasicSolver/2D-seq-pt/src/parameter.h b/EnhancedSolver/2D-seq/src/parameter.h similarity index 63% rename from BasicSolver/2D-seq-pt/src/parameter.h rename to EnhancedSolver/2D-seq/src/parameter.h index d3ee2e9..ffc880e 100644 --- a/BasicSolver/2D-seq-pt/src/parameter.h +++ b/EnhancedSolver/2D-seq/src/parameter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -10,17 +10,23 @@ typedef struct { double xlength, ylength; int imax, jmax; - int itermax; - double eps, omg; + int itermax, levels, presmooth, postsmooth; + double eps, omg, rho; double re, tau, gamma; double te, dt; double gx, gy; char* name; int bcLeft, bcRight, bcBottom, bcTop; double u_init, v_init, p_init; - double traceStart, traceWrite, traceInject; - double lineX1, lineX2, lineY1, lineY2; - int nparticles; + + int numberOfParticles; + double startTime, injectTimePeriod, writeTimePeriod; + + double x1, y1, x2, y2; + + int shape; + double xCenter, yCenter, xRectLength, yRectLength, circleRadius; + } Parameter; void initParameter(Parameter*); diff --git a/EnhancedSolver/2D-seq/src/particletracing.c b/EnhancedSolver/2D-seq/src/particletracing.c new file mode 100644 index 0000000..3208245 --- /dev/null +++ b/EnhancedSolver/2D-seq/src/particletracing.c @@ -0,0 +1,248 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "grid.h" +#include "vtkWriter.h" + +#define U(i, j) u[(j) * (imax + 2) + (i)] +#define V(i, j) v[(j) * (imax + 2) + (i)] +#define S(i, j) s[(j) * (imax + 2) + (i)] + +void printParticles(ParticleTracer* p) +{ + for (int i = 0; i < p->totalParticles; ++i) { + printf("Particle position X : %.2f, Y : %.2f, flag : %d\n", + p->particlePool[i].x, + p->particlePool[i].y, + p->particlePool[i].flag); + } +} + +static void injectParticles(ParticleTracer* p) +{ + if (p->totalParticles + p->numParticlesInLine > p->numAllocatedParticles) { + return; + } + for (int i = 0; i < p->numParticlesInLine; ++i) { + p->particlePool[p->pointer].x = p->linSpaceLine[i].x; + p->particlePool[p->pointer].y = p->linSpaceLine[i].y; + p->particlePool[p->pointer].flag = true; + p->pointer++; + p->totalParticles++; + } +} + +static void advanceParticles( + ParticleTracer* p, double* restrict u, double* restrict v, double dt) +{ + int imax = p->grid->imax; + int jmax = p->grid->jmax; + double dx = p->grid->dx; + double dy = p->grid->dy; + double xlength = p->grid->xlength; + double ylength = p->grid->ylength; + + for (int i = 0; i < p->totalParticles; ++i) { + if (p->particlePool[i].flag == true) { + double x = p->particlePool[i].x; + double y = p->particlePool[i].y; + + int iCoord = (int)(x / dx) + 1; + int jCoord = (int)((y + 0.5 * dy) / dy) + 1; + + double x1 = (double)(iCoord - 1) * dx; + double y1 = ((double)(jCoord - 1) - 0.5) * dy; + double x2 = (double)iCoord * dx; + double y2 = ((double)jCoord - 0.5) * dy; + + double intU = (1.0 / (dx * dy)) * + ((x2 - x) * (y2 - y) * U(iCoord - 1, jCoord - 1) + + (x - x1) * (y2 - y) * U(iCoord, jCoord - 1) + + (x2 - x) * (y - y1) * U(iCoord - 1, jCoord) + + (x - x1) * (y - y1) * U(iCoord, jCoord)); + + double newX = x + dt * intU; + p->particlePool[i].x = newX; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)(y / dy) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = (double)(jCoord - 1) * dy; + x2 = ((double)iCoord - 0.5) * dx; + y2 = (double)jCoord * dy; + + double intV = (1.0 / (dx * dy)) * + ((x2 - x) * (y2 - y) * V(iCoord - 1, jCoord - 1) + + (x - x1) * (y2 - y) * V(iCoord, jCoord - 1) + + (x2 - x) * (y - y1) * V(iCoord - 1, jCoord) + + (x - x1) * (y - y1) * V(iCoord, jCoord)); + + double newY = y + dt * intV; + p->particlePool[i].y = newY; + + if (((newX < 0.0) || (newX > xlength) || (newY < 0.0) || (newY > ylength))) { + p->particlePool[i].flag = false; + p->removedParticles++; + } + + int newI = newX / dx, newJ = newY / dy; + + if (!gridIsFluid(p->grid, newI, newJ)) { + p->particlePool[i].flag = false; + p->removedParticles++; + // printf("Forbidden movement of particle into obstacle!\n"); + } + } + } +} + +static void compress(ParticleTracer* p) +{ + Particle* memPool = p->particlePool; + Particle tempPool[p->totalParticles]; + int totalParticles = 0; + + // printf("Performing compression ..."); + + for (int i = 0; i < p->totalParticles; i++) { + if (memPool[i].flag == 1) { + tempPool[totalParticles].x = memPool[i].x; + tempPool[totalParticles].y = memPool[i].y; + tempPool[totalParticles].flag = memPool[i].flag; + totalParticles++; + } + } + + // printf(" remove %d particles\n", p->totalParticles - totalParticles); + p->totalParticles = totalParticles; + p->removedParticles = 0; + p->pointer = totalParticles + 1; + memcpy(p->particlePool, tempPool, totalParticles * sizeof(Particle)); +} + +void writeParticles(ParticleTracer* p) +{ + static int ts = 0; + compress(p); + + VtkOptions opts = { .particletracer = p }; + + char filename[50]; + // snprintf(filename, 50, "vtk_files/particles%d.vtk", ts); + // vtkOpen(&opts, filename, ts); + // vtkParticle(&opts, "particle"); + // vtkClose(&opts); + + FILE* fp; + Particle* particlePool = p->particlePool; + + snprintf(filename, 50, "vis_files/particles_%d.dat", ts); + fp = fopen(filename, "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + for (int i = 0; i < p->totalParticles; ++i) { + double x = particlePool[i].x; + double y = particlePool[i].y; + fprintf(fp, "%f %f\n", x, y); + } + fclose(fp); + + ++ts; +} + +void initParticleTracer(ParticleTracer* pt, Grid* g, Parameter* p) +{ + pt->numParticlesInLine = p->numberOfParticles; + pt->startTime = p->startTime; + pt->injectTimePeriod = p->injectTimePeriod; + pt->writeTimePeriod = p->writeTimePeriod; + pt->grid = g; + + pt->x1 = p->x1; + pt->y1 = p->y1; + pt->x2 = p->x2; + pt->y2 = p->y2; + + pt->lastInjectTime = p->startTime; + pt->lastWriteTime = p->startTime; + + pt->pointer = 0; + pt->removedParticles = 0; + pt->totalParticles = 0; + + if (p->te > p->startTime) { + pt->numAllocatedParticles = ((p->te - p->startTime) / p->injectTimePeriod) * + p->numberOfParticles; + pt->numAllocatedParticles += (2 * p->numberOfParticles); + + pt->particlePool = malloc(sizeof(Particle) * pt->numAllocatedParticles); + pt->linSpaceLine = malloc(sizeof(Particle) * pt->numParticlesInLine); + + for (int i = 0; i < pt->numParticlesInLine; ++i) { + double spacing = (double)i / (double)(pt->numParticlesInLine - 1); + pt->linSpaceLine[i].x = spacing * pt->x1 + (1.0 - spacing) * pt->x2; + pt->linSpaceLine[i].y = spacing * pt->y1 + (1.0 - spacing) * pt->y2; + pt->linSpaceLine[i].flag = true; + } + } else { + pt->particlePool = NULL; + pt->linSpaceLine = NULL; + } +} + +void printParticleTracerParameters(ParticleTracer* p) +{ + printf("Particle Tracing data:\n"); + printf("\tNumber of particles : %d being injected for every period of %.2f\n", + p->numParticlesInLine, + p->injectTimePeriod); + printf("\tstartTime : %.2f\n", p->startTime); + printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : " + "%.2f, x2 : %.2f, y2 : %.2f\n", + p->x1, + p->y1, + p->x2, + p->y2); + printf("\tPointer : %d, TotalParticles : %d\n", p->pointer, p->totalParticles); +} + +void trace(ParticleTracer* p, double* u, double* v, double dt, double time) +{ + if (time >= p->startTime) { + if ((time - p->lastInjectTime) >= p->injectTimePeriod) { + injectParticles(p); + p->lastInjectTime = time; + } + + if ((time - p->lastWriteTime) >= p->writeTimePeriod) { + writeParticles(p); + p->lastWriteTime = time; + } + + advanceParticles(p, u, v, dt); + + if (p->removedParticles > (p->totalParticles * 0.2)) { + compress(p); + } + } +} + +void freeParticles(ParticleTracer* p) +{ + if (p->particlePool != NULL) { + free(p->particlePool); + free(p->linSpaceLine); + } +} diff --git a/EnhancedSolver/2D-seq/src/particletracing.h b/EnhancedSolver/2D-seq/src/particletracing.h new file mode 100644 index 0000000..09855a5 --- /dev/null +++ b/EnhancedSolver/2D-seq/src/particletracing.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARTICLETRACING_H_ +#define __PARTICLETRACING_H_ +#include + +#include "grid.h" +#include "parameter.h" + +typedef enum COORD { X = 0, Y, NCOORD } COORD; + +typedef struct { + double x, y; + bool flag; +} Particle; + +typedef struct { + int numParticlesInLine, removedParticles, totalParticles; + double startTime, injectTimePeriod, writeTimePeriod; + double lastInjectTime, lastUpdateTime, lastWriteTime; + + int numAllocatedParticles; + + Particle* linSpaceLine; + Particle* particlePool; + + int pointer; + double x1, y1, x2, y2; + Grid* grid; +} ParticleTracer; + +extern void initParticleTracer(ParticleTracer*, Grid*, Parameter*); +extern void freeParticles(ParticleTracer*); +extern void writeParticles(ParticleTracer*); +extern void printParticleTracerParameters(ParticleTracer*); +extern void trace(ParticleTracer*, double*, double*, double, double); +#endif diff --git a/BasicSolver/2D-seq-pt/src/progress.c b/EnhancedSolver/2D-seq/src/progress.c similarity index 74% rename from BasicSolver/2D-seq-pt/src/progress.c rename to EnhancedSolver/2D-seq/src/progress.c index a9b82bd..54a1dbc 100644 --- a/BasicSolver/2D-seq-pt/src/progress.c +++ b/EnhancedSolver/2D-seq/src/progress.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -49,3 +49,21 @@ void stopProgress() printf("\n"); fflush(stdout); } +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} \ No newline at end of file diff --git a/BasicSolver/2D-seq-pt/src/progress.h b/EnhancedSolver/2D-seq/src/progress.h similarity index 58% rename from BasicSolver/2D-seq-pt/src/progress.h rename to EnhancedSolver/2D-seq/src/progress.h index 9ef2d96..240c279 100644 --- a/BasicSolver/2D-seq-pt/src/progress.h +++ b/EnhancedSolver/2D-seq/src/progress.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -9,6 +9,7 @@ extern void initProgress(double); extern void printProgress(double); -extern void stopProgress(); - +extern void stopProgress(void); +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/EnhancedSolver/2D-seq/src/solver-mg.c b/EnhancedSolver/2D-seq/src/solver-mg.c new file mode 100644 index 0000000..395d536 --- /dev/null +++ b/EnhancedSolver/2D-seq/src/solver-mg.c @@ -0,0 +1,221 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +#define S(i, j) s[(j) * (imax + 2) + (i)] +#define E(i, j) e[(j) * (imax + 2) + (i)] +#define R(i, j) r[(j) * (imax + 2) + (i)] +#define OLD(i, j) old[(j) * (imax + 2) + (i)] + +static void restrictMG(Solver* s, int level, int imax, int jmax) +{ + double* r = s->r[level + 1]; + double* old = s->r[level]; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + R(i, j) = (OLD(2 * i - 1, 2 * j - 1) + OLD(2 * i, 2 * j - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1) + OLD(2 * i - 1, 2 * j) * 2 + + OLD(2 * i, 2 * j) * 4 + OLD(2 * i + 1, 2 * j) * 2 + + OLD(2 * i - 1, 2 * j + 1) + OLD(2 * i, 2 * j + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1)) / + 16.0; + } + } +} + +static void prolongate(Solver* s, int level, int imax, int jmax) +{ + double* old = s->r[level + 1]; + double* e = s->r[level]; + + for (int j = 2; j < jmax + 1; j += 2) { + for (int i = 2; i < imax + 1; i += 2) { + E(i, j) = OLD(i / 2, j / 2); + } + } +} + +static void correct(Solver* s, double* p, int level, int imax, int jmax) +{ + double* e = s->e[level]; + + for (int j = 1; j < jmax + 1; ++j) { + for (int i = 1; i < imax + 1; ++i) { + P(i, j) += E(i, j); + } + } +} + +static void setBoundaryCondition(double* p, int imax, int jmax) +{ + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmax + 1) = P(i, jmax); + } + + for (int j = 1; j < jmax + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } +} + +static void smooth(Solver* s, double* p, double* rhs, int level, int imax, int jmax) +{ + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + P(i, j) -= factor * (RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2)); + + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, int imax, int jmax) +{ + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* r = s->r[level]; + double res = 1.0; + int pass, jsw, isw; + + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + R(i, j) = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + res += (R(i, j) * R(i, j)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + + res = res / (double)(imax * jmax); + return res; +} + +static double multiGrid(Solver* s, double* p, double* rhs, int level, int imax, int jmax) +{ + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, imax, jmax); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, imax, jmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); + } + + res = calculateResidual(s, p, rhs, level, imax, jmax); + + // restrict + restrictMG(s, level, imax, jmax); + + // MGSolver on residual and error. + multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, imax / 2, jmax / 2); + + // prolongate + prolongate(s, level, imax, jmax); + + // correct p on finer level using residual + correct(s, p, level, imax, jmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, imax, jmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int levels = s->levels; + printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * sizeof(double); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size); + s->e[j] = allocate(64, size); + + for (int i = 0; i < (imax + 2) * (jmax + 2); i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax); + +#ifdef VERBOSE + printf("Residuum: %.6f\n", res); +#endif + +return res; +} diff --git a/EnhancedSolver/2D-seq/src/solver-rb.c b/EnhancedSolver/2D-seq/src/solver-rb.c new file mode 100644 index 0000000..e2716a0 --- /dev/null +++ b/EnhancedSolver/2D-seq/src/solver-rb.c @@ -0,0 +1,84 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "grid.h" +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->itermax = p->itermax; + s->eps = p->eps; + s->omega = p->omg; + + Grid* g = s->grid; + int imax = s->grid->imax; + int jmax = s->grid->jmax; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double epssq = eps * eps; + int it = 0; + Grid* g = s->grid; + double res = 1.0; + int pass, jsw, isw; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmax + 1) = P(i, jmax); + } + + for (int j = 1; j < jmax + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } + + res = res / (double)(imax * jmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif + + return res; +} diff --git a/EnhancedSolver/2D-seq/src/solver-sor.c b/EnhancedSolver/2D-seq/src/solver-sor.c new file mode 100644 index 0000000..2db5452 --- /dev/null +++ b/EnhancedSolver/2D-seq/src/solver-sor.c @@ -0,0 +1,75 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "grid.h" +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->itermax = p->itermax; + s->eps = p->eps; + s->omega = p->omg; + + Grid* g = s->grid; + int imax = s->grid->imax; + int jmax = s->grid->jmax; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = s->omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double epssq = eps * eps; + int it = 0; + Grid* g = s->grid; + double res = 1.0; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + } + + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmax + 1) = P(i, jmax); + } + + for (int j = 1; j < jmax + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } + + res = res / (double)(imax * jmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif + + return res; +} diff --git a/EnhancedSolver/2D-seq/src/solver.h b/EnhancedSolver/2D-seq/src/solver.h new file mode 100644 index 0000000..a8dc072 --- /dev/null +++ b/EnhancedSolver/2D-seq/src/solver.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "discretization.h" +#include "grid.h" +#include "parameter.h" + +typedef struct { + /* geometry and grid information */ + Grid* grid; + /* parameters */ + double eps, omega, rho; + int itermax; + int levels, presmooth, postsmooth; + int totalFluidCells; + double **r, **e; +} Solver; + +extern void initSolver(Solver*, Discretization*, Parameter*); +extern double solve(Solver*, double*, double*); + +#endif diff --git a/BasicSolver/2D-mpi-v2/src/timing.c b/EnhancedSolver/2D-seq/src/timing.c similarity index 72% rename from BasicSolver/2D-mpi-v2/src/timing.c rename to EnhancedSolver/2D-seq/src/timing.c index c4025a4..78b01c4 100644 --- a/BasicSolver/2D-mpi-v2/src/timing.c +++ b/EnhancedSolver/2D-seq/src/timing.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,18 +7,16 @@ #include #include -double getTimeStamp() +double getTimeStamp(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } -double getTimeResolution() +double getTimeResolution(void) { struct timespec ts; clock_getres(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } - -double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-mpi-v1/src/timing.h b/EnhancedSolver/2D-seq/src/timing.h similarity index 55% rename from BasicSolver/2D-mpi-v1/src/timing.h rename to EnhancedSolver/2D-seq/src/timing.h index db95329..ed05a8c 100644 --- a/BasicSolver/2D-mpi-v1/src/timing.h +++ b/EnhancedSolver/2D-seq/src/timing.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,8 +7,7 @@ #ifndef __TIMING_H_ #define __TIMING_H_ -extern double getTimeStamp(); -extern double getTimeResolution(); -extern double getTimeStamp_(); +extern double getTimeStamp(void); +extern double getTimeResolution(void); #endif // __TIMING_H_ diff --git a/BasicSolver/2D-mpi-v3/src/util.h b/EnhancedSolver/2D-seq/src/util.h similarity index 62% rename from BasicSolver/2D-mpi-v3/src/util.h rename to EnhancedSolver/2D-seq/src/util.h index 657b009..e4a93c2 100644 --- a/BasicSolver/2D-mpi-v3/src/util.h +++ b/EnhancedSolver/2D-seq/src/util.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -19,4 +19,11 @@ #define ABS(a) ((a) >= 0 ? (a) : -(a)) #endif +#define P(i, j) p[(j) * (imax + 2) + (i)] +#define F(i, j) f[(j) * (imax + 2) + (i)] +#define G(i, j) g[(j) * (imax + 2) + (i)] +#define U(i, j) u[(j) * (imax + 2) + (i)] +#define V(i, j) v[(j) * (imax + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imax + 2) + (i)] + #endif // __UTIL_H_ diff --git a/EnhancedSolver/2D-seq/src/vtkWriter.c b/EnhancedSolver/2D-seq/src/vtkWriter.c new file mode 100644 index 0000000..521335d --- /dev/null +++ b/EnhancedSolver/2D-seq/src/vtkWriter.c @@ -0,0 +1,125 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "vtkWriter.h" + +static float floatSwap(float f) +{ + union { + float f; + char b[4]; + } dat1, dat2; + + dat1.f = f; + dat2.b[0] = dat1.b[3]; + dat2.b[1] = dat1.b[2]; + dat2.b[2] = dat1.b[1]; + dat2.b[3] = dat1.b[0]; + return dat2.f; +} + +static void writeHeader(VtkOptions* o, int ts) +{ + fprintf(o->fh, "# vtk DataFile Version 3.0\n"); + fprintf(o->fh, "PAMPI cfd solver particle tracing file\n"); + if (o->fmt == ASCII) { + fprintf(o->fh, "ASCII\n"); + } else if (o->fmt == BINARY) { + fprintf(o->fh, "BINARY\n"); + } + + fprintf(o->fh, "DATASET UNSTRUCTURED_GRID\n"); + fprintf(o->fh, "FIELD FieldData 2\n"); + fprintf(o->fh, "TIME 1 1 double\n"); + fprintf(o->fh, "%d\n", ts); + fprintf(o->fh, "CYCLE 1 1 int\n"); + fprintf(o->fh, "1\n"); +} + +void vtkOpen(VtkOptions* o, char* problem, int ts) +{ + o->fh = fopen(problem, "w"); + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + + writeHeader(o, ts); + + printf("Writing VTK output for %s\n", problem); +} + +void vtkParticle(VtkOptions* o, char* name) +{ + Particle* particlePool = o->particletracer->particlePool; + + int imax = o->particletracer->grid->imax; + int jmax = o->particletracer->grid->jmax; + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + + fprintf(o->fh, "POINTS %d float\n", o->particletracer->totalParticles); + + for (int i = 0; i < o->particletracer->totalParticles; ++i) { + double x = particlePool[i].x; + double y = particlePool[i].y; + fprintf(o->fh, "%.2f %.2f 0.0\n", x, y); + } + + fprintf(o->fh, + "CELLS %d %d\n", + o->particletracer->totalParticles, + 2 * o->particletracer->totalParticles); + + for (int i = 0; i < o->particletracer->totalParticles; ++i) { + fprintf(o->fh, "1 %d\n", i); + } + + fprintf(o->fh, "CELL_TYPES %d\n", o->particletracer->totalParticles); + + for (int i = 0; i < o->particletracer->totalParticles; ++i) { + fprintf(o->fh, "1\n"); + } + + /* + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, + "%f %f %f\n", + G(vec.u, i, j, k), + G(vec.v, i, j, k), + G(vec.w, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((float[3]) { floatSwap(G(vec.u, i, j, k)), + floatSwap(G(vec.v, i, j, k)), + floatSwap(G(vec.w, i, j, k)) }, + sizeof(float), + 3, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); + + */ +} + +void vtkClose(VtkOptions* o) +{ + fclose(o->fh); + o->fh = NULL; +} diff --git a/EnhancedSolver/2D-seq/src/vtkWriter.h b/EnhancedSolver/2D-seq/src/vtkWriter.h new file mode 100644 index 0000000..1e582fc --- /dev/null +++ b/EnhancedSolver/2D-seq/src/vtkWriter.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __VTKWRITER_H_ +#define __VTKWRITER_H_ +#include + +#include "particletracing.h" +#include "solver.h" + +typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat; + +typedef struct VtkOptions { + VtkFormat fmt; + FILE* fh; + ParticleTracer* particletracer; +} VtkOptions; + +extern void vtkOpen(VtkOptions* opts, char* filename, int ts); +extern void vtkParticle(VtkOptions* opts, char* name); +extern void vtkClose(VtkOptions* opts); +#endif // __VTKWRITER_H_ diff --git a/BasicSolver/2D-mpi-v2/surface.plot b/EnhancedSolver/2D-seq/surface.plot similarity index 100% rename from BasicSolver/2D-mpi-v2/surface.plot rename to EnhancedSolver/2D-seq/surface.plot diff --git a/BasicSolver/2D-seq-pt/vector.plot b/EnhancedSolver/2D-seq/vector.plot similarity index 68% rename from BasicSolver/2D-seq-pt/vector.plot rename to EnhancedSolver/2D-seq/vector.plot index 0934ab2..232da08 100644 --- a/BasicSolver/2D-seq-pt/vector.plot +++ b/EnhancedSolver/2D-seq/vector.plot @@ -1,5 +1,7 @@ -set terminal png size 1800,768 enhanced font ,12 +set terminal png size 3600,768 enhanced font ,28 set output 'velocity.png' + +set size ratio -1 set datafile separator whitespace plot 'velocity.dat' using 1:2:3:4:5 with vectors filled head size 0.01,20,60 lc palette diff --git a/EnhancedSolver/2D-seq/vis_files/animate.plot b/EnhancedSolver/2D-seq/vis_files/animate.plot new file mode 100644 index 0000000..d54ba32 --- /dev/null +++ b/EnhancedSolver/2D-seq/vis_files/animate.plot @@ -0,0 +1,10 @@ +unset border; unset tics; unset key; +set term gif animate delay 30 +set output "trace.gif" +set xrange [0:1] +set yrange [0:1] + +do for [ts=0:120] { + plot "particles_".ts.".dat" with points pointtype 7 +} +unset output diff --git a/EnhancedSolver/2D-seq/vis_files/backstep_animate.plot b/EnhancedSolver/2D-seq/vis_files/backstep_animate.plot new file mode 100644 index 0000000..2888e5d --- /dev/null +++ b/EnhancedSolver/2D-seq/vis_files/backstep_animate.plot @@ -0,0 +1,14 @@ +unset border; unset tics; unset key; +set term gif animate delay 10 +set output "trace.gif" +set xrange [0:7] +set yrange [0:1.5] +set size ratio -1 + +set object 1 rect from 0.0,0.0 to 1.0,0.5 lw 5 + + +do for [ts=0:300] { + plot "particles_".ts.".dat" with points pointtype 7 +} +unset output diff --git a/EnhancedSolver/2D-seq/vis_files/canal_animate.plot b/EnhancedSolver/2D-seq/vis_files/canal_animate.plot new file mode 100644 index 0000000..bf9ce60 --- /dev/null +++ b/EnhancedSolver/2D-seq/vis_files/canal_animate.plot @@ -0,0 +1,10 @@ +unset border; unset tics; unset key; +set term gif animate delay 30 +set output "trace.gif" +set xrange [0:30] +set yrange [0:4] + +do for [ts=0:120] { + plot "particles_".ts.".dat" with points pointtype 7 +} +unset output diff --git a/EnhancedSolver/2D-seq/vis_files/karman_animate.plot b/EnhancedSolver/2D-seq/vis_files/karman_animate.plot new file mode 100644 index 0000000..d78f0c3 --- /dev/null +++ b/EnhancedSolver/2D-seq/vis_files/karman_animate.plot @@ -0,0 +1,13 @@ +unset border; unset tics; unset key; +set term gif animate delay 10 +set output "trace.gif" +set xrange [0:30] +set yrange [0:8] +set size ratio -1 +set object 1 circle front at 5.0,4.0 size 1.0 fillcolor rgb "black" lw 2 + + +do for [ts=0:500] { + plot "particles_".ts.".dat" with points pointtype 7 pointsize 0.3 +} +unset output diff --git a/BasicSolver/2D-seq-pt/Makefile b/EnhancedSolver/3D-mpi/Makefile similarity index 67% rename from BasicSolver/2D-seq-pt/Makefile rename to EnhancedSolver/3D-mpi/Makefile index 57f99f4..b7fce27 100644 --- a/BasicSolver/2D-seq-pt/Makefile +++ b/EnhancedSolver/3D-mpi/Makefile @@ -1,5 +1,5 @@ #======================================================================================= -# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. # All rights reserved. # Use of this source code is governed by a MIT-style # license that can be found in the LICENSE file. @@ -18,13 +18,19 @@ include $(MAKE_DIR)/include_$(TAG).mk INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) VPATH = $(SRC_DIR) -SRC = $(wildcard $(SRC_DIR)/*.c) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/vtkWriter-$(VTK_OUTPUT_FMT).o +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) +ifeq ($(VTK_OUTPUT_FMT),mpi) +DEFINES += -D_VTK_WRITER_MPI +endif + CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) -${TARGET}: $(BUILD_DIR) $(OBJ) +${TARGET}: sanity-checks $(BUILD_DIR) $(OBJ) $(info ===> LINKING $(TARGET)) $(Q)${LINKER} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS) @@ -37,9 +43,20 @@ $(BUILD_DIR)/%.s: %.c $(info ===> GENERATE ASM $@) $(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@ -.PHONY: clean distclean tags info asm format +.PHONY: clean distclean vis vis_clean tags info asm format -clean: +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f *.vtk + @rm -f vtk_files/*.vtk + @rm -f *.png + +clean: vis_clean $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @rm -f tags @@ -65,6 +82,14 @@ format: done @echo "Done" +sanity-checks: +ifeq ($(VTK_OUTPUT_FMT),mpi) +ifeq ($(ENABLE_MPI),false) + $(error VTK_OUTPUT_FMT mpi only supported for ENABLE_MPI true!) +endif +endif + + $(BUILD_DIR): @mkdir $(BUILD_DIR) diff --git a/BasicSolver/3D-mpi-io/README.md b/EnhancedSolver/3D-mpi/README.md similarity index 100% rename from BasicSolver/3D-mpi-io/README.md rename to EnhancedSolver/3D-mpi/README.md diff --git a/EnhancedSolver/3D-mpi/backstep.par b/EnhancedSolver/3D-mpi/backstep.par new file mode 100644 index 0000000..8075448 --- /dev/null +++ b/EnhancedSolver/3D-mpi/backstep.par @@ -0,0 +1,89 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name backstep # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 36000.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 1.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 7.0 # domain size in x-direction +ylength 1.5 # domain size in y-direction +zlength 1.0 # domain size in z-direction +imax 210 # number of interior cells in x-direction +jmax 45 # number of interior cells in y-direction +kmax 30 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 40.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 2000 +startTime 0 +injectTimePeriod 1.0 +writeTimePeriod 0.2 + +x1 0.0 +y1 0.5 +z1 0.0 +x2 0.0 +y2 1.5 +z2 1.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 1 +xCenter 0.0 +yCenter 0.0 +zCenter 0.0 +xRectLength 2.0 +yRectLength 1.0 +zRectLength 2.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/BasicSolver/3D-mpi-io/canal.par b/EnhancedSolver/3D-mpi/canal.par similarity index 61% rename from BasicSolver/3D-mpi-io/canal.par rename to EnhancedSolver/3D-mpi/canal.par index 3ff4a5f..59fbe85 100644 --- a/BasicSolver/3D-mpi-io/canal.par +++ b/EnhancedSolver/3D-mpi/canal.par @@ -32,13 +32,13 @@ xlength 30.0 # domain size in x-direction ylength 4.0 # domain size in y-direction zlength 4.0 # domain size in z-direction imax 200 # number of interior cells in x-direction -jmax 50 # number of interior cells in y-direction -kmax 50 # number of interior cells in z-direction +jmax 80 # number of interior cells in y-direction +kmax 80 # number of interior cells in z-direction # Time Data: # --------- -te 100.0 # final time +te 60.0 # final time dt 0.02 # time stepsize tau 0.5 # safety factor for time stepsize control (<0 constant delt) @@ -47,6 +47,42 @@ tau 0.5 # safety factor for time stepsize control (<0 constant delt) itermax 500 # maximal number of pressure iteration in one time step eps 0.0001 # stopping tolerance for pressure iteration -omg 1.3 # relaxation parameter for SOR iteration +omg 0.52 +omg 1.6 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 500 +startTime 30 +injectTimePeriod 1.0 +writeTimePeriod 0.5 + +x1 0.0 +y1 0.0 +z1 0.0 +x2 4.0 +y2 4.0 +z2 4.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 10.0 +yCenter 2.0 +zCenter 2.0 +xRectLength 8.0 +yRectLength 2.0 +zRectLength 2.0 +circleRadius 1.0 #=============================================================================== diff --git a/BasicSolver/3D-mpi-io/config.mk b/EnhancedSolver/3D-mpi/config.mk similarity index 69% rename from BasicSolver/3D-mpi-io/config.mk rename to EnhancedSolver/3D-mpi/config.mk index 88fec3f..0dbe440 100644 --- a/BasicSolver/3D-mpi-io/config.mk +++ b/EnhancedSolver/3D-mpi/config.mk @@ -1,6 +1,12 @@ # Supported: GCC, CLANG, ICC TAG ?= ICC +# Supported: true, false +ENABLE_MPI ?= true ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg +# Supported: seq, mpi +VTK_OUTPUT_FMT ?= seq #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 diff --git a/BasicSolver/3D-mpi-io/dcavity.par b/EnhancedSolver/3D-mpi/dcavity.par similarity index 59% rename from BasicSolver/3D-mpi-io/dcavity.par rename to EnhancedSolver/3D-mpi/dcavity.par index c5cb201..204e007 100644 --- a/BasicSolver/3D-mpi-io/dcavity.par +++ b/EnhancedSolver/3D-mpi/dcavity.par @@ -31,9 +31,9 @@ p_init 0.0 # initial value for pressure xlength 1.0 # domain size in x-direction ylength 1.0 # domain size in y-direction zlength 1.0 # domain size in z-direction -imax 128 # number of interior cells in x-direction -jmax 128 # number of interior cells in y-direction -kmax 128 # number of interior cells in z-direction +imax 80 # number of interior cells in x-direction +jmax 80 # number of interior cells in y-direction +kmax 80 # number of interior cells in z-direction # Time Data: # --------- @@ -47,6 +47,42 @@ tau 0.5 # safety factor for time stepsize control (<0 constant delt) itermax 1000 # maximal number of pressure iteration in one time step eps 0.001 # stopping tolerance for pressure iteration -omg 1.8 # relaxation parameter for SOR iteration +rho 0.5 +omg 1.7 # relaxation parameter for SOR iteration gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 0 +injectTimePeriod 1.0 +writeTimePeriod 0.1 + +x1 0.0 +y1 0.1 +z1 0.0 +x2 1.0 +y2 0.1 +z2 1.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 0.5 +yCenter 0.5 +zCenter 0.5 +xRectLength 0.25 +yRectLength 0.25 +zRectLength 0.25 +circleRadius 0.25 #=============================================================================== diff --git a/BasicSolver/2D-seq-pt/include_CLANG.mk b/EnhancedSolver/3D-mpi/include_CLANG.mk similarity index 54% rename from BasicSolver/2D-seq-pt/include_CLANG.mk rename to EnhancedSolver/3D-mpi/include_CLANG.mk index 0caff0c..3641cad 100644 --- a/BasicSolver/2D-seq-pt/include_CLANG.mk +++ b/EnhancedSolver/3D-mpi/include_CLANG.mk @@ -1,4 +1,10 @@ -CC = clang +ifeq ($(ENABLE_MPI),true) +CC = mpicc +DEFINES = -D_MPI +else +CC = cc +endif + GCC = cc LINKER = $(CC) @@ -9,9 +15,7 @@ LIBS = # -lomp endif VERSION = --version -# CFLAGS = -O3 -std=c17 $(OPENMP) -CFLAGS = -O3 std=c17 -#CFLAGS = -Ofast -fnt-store=aggressive -std=c99 $(OPENMP) #AMD CLANG +CFLAGS = -Ofast -std=c17 LFLAGS = $(OPENMP) -lm -DEFINES = -D_GNU_SOURCE# -DDEBUG -INCLUDES = +DEFINES += -D_GNU_SOURCE# -DDEBUG +INCLUDES = -I/opt/homebrew/include diff --git a/BasicSolver/2D-seq-pt/include_GCC.mk b/EnhancedSolver/3D-mpi/include_GCC.mk similarity index 68% rename from BasicSolver/2D-seq-pt/include_GCC.mk rename to EnhancedSolver/3D-mpi/include_GCC.mk index 427e798..cf1e2aa 100644 --- a/BasicSolver/2D-seq-pt/include_GCC.mk +++ b/EnhancedSolver/3D-mpi/include_GCC.mk @@ -1,4 +1,10 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpicc +DEFINES = -D_MPI +else CC = gcc +endif + GCC = gcc LINKER = $(CC) @@ -9,6 +15,6 @@ endif VERSION = --version CFLAGS = -Ofast -ffreestanding -std=c99 $(OPENMP) LFLAGS = $(OPENMP) -DEFINES = -D_GNU_SOURCE +DEFINES += -D_GNU_SOURCE INCLUDES = LIBS = diff --git a/EnhancedSolver/3D-mpi/include_ICC.mk b/EnhancedSolver/3D-mpi/include_ICC.mk new file mode 100644 index 0000000..1ffac3a --- /dev/null +++ b/EnhancedSolver/3D-mpi/include_ICC.mk @@ -0,0 +1,20 @@ +ifeq ($(ENABLE_MPI),true) +CC = mpiicc +DEFINES = -D_MPI +else +CC = icc +endif + +GCC = gcc +LINKER = $(CC) + +ifeq ($(ENABLE_OPENMP),true) +OPENMP = -qopenmp +endif + +VERSION = --version +CFLAGS = -O3 -qopt-zmm-usage=high -std=c99 $(OPENMP) -xHost +LFLAGS = $(OPENMP) +DEFINES += -D_GNU_SOURCE# -DDEBUG +INCLUDES = +LIBS = diff --git a/EnhancedSolver/3D-mpi/karman.par b/EnhancedSolver/3D-mpi/karman.par new file mode 100644 index 0000000..a74bf2b --- /dev/null +++ b/EnhancedSolver/3D-mpi/karman.par @@ -0,0 +1,89 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name karman # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 5050.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 8.0 # domain size in y-direction +zlength 8.0 # domain size in z-direction +imax 200 # number of interior cells in x-direction +jmax 80 # number of interior cells in y-direction +kmax 80 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 200.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 200 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.75 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 1000 +startTime 50 +injectTimePeriod 1.0 +writeTimePeriod 1.0 + +x1 0.0 +y1 3.6 +z1 3.6 +x2 0.0 +y2 4.7 +z2 4.7 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 2 +xCenter 5.0 +yCenter 4.0 +zCenter 4.0 +xRectLength 8.0 +yRectLength 2.0 +zRectLength 2.0 +circleRadius 2.0 + +#=============================================================================== diff --git a/EnhancedSolver/3D-mpi/residual.plot b/EnhancedSolver/3D-mpi/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/EnhancedSolver/3D-mpi/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/2D-mpi-v3/src/allocate.c b/EnhancedSolver/3D-mpi/src/allocate.c similarity index 83% rename from BasicSolver/2D-mpi-v3/src/allocate.c rename to EnhancedSolver/3D-mpi/src/allocate.c index 81e1e9d..cf2efd6 100644 --- a/BasicSolver/2D-mpi-v3/src/allocate.c +++ b/EnhancedSolver/3D-mpi/src/allocate.c @@ -1,14 +1,17 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ #include +#include #include #include -void* allocate(int alignment, size_t bytesize) +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) { int errorCode; void* ptr; diff --git a/BasicSolver/2D-mpi-v1/src/allocate.h b/EnhancedSolver/3D-mpi/src/allocate.h similarity index 64% rename from BasicSolver/2D-mpi-v1/src/allocate.h rename to EnhancedSolver/3D-mpi/src/allocate.h index 54cfe06..77f4ba0 100644 --- a/BasicSolver/2D-mpi-v1/src/allocate.h +++ b/EnhancedSolver/3D-mpi/src/allocate.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -8,6 +8,6 @@ #define __ALLOCATE_H_ #include -extern void* allocate(int alignment, size_t bytesize); +extern void* allocate(size_t alignment, size_t bytesize); #endif diff --git a/EnhancedSolver/3D-mpi/src/comm.c b/EnhancedSolver/3D-mpi/src/comm.c new file mode 100644 index 0000000..df9f795 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/comm.c @@ -0,0 +1,647 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "allocate.h" +#include "comm.h" + +#if defined(_MPI) +// subroutines local to this module +static int sizeOfRank(int rank, int size, int N) +{ + return N / size + ((N % size > rank) ? 1 : 0); +} + +static void setupCommunication(Comm* c, Direction direction, int layer) +{ + int imaxLocal = c->imaxLocal; + int jmaxLocal = c->jmaxLocal; + int kmaxLocal = c->kmaxLocal; + + size_t dblsize = sizeof(double); + int sizes[NDIMS]; + int subSizes[NDIMS]; + int starts[NDIMS]; + int offset = 0; + + sizes[IDIM] = imaxLocal + 2; + sizes[JDIM] = jmaxLocal + 2; + sizes[KDIM] = kmaxLocal + 2; + + if (layer == HALO) { + offset = 1; + } + + switch (direction) { + case LEFT: + subSizes[IDIM] = 1; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = 1 - offset; + starts[JDIM] = 1; + starts[KDIM] = 1; + break; + case RIGHT: + subSizes[IDIM] = 1; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = imaxLocal + offset; + starts[JDIM] = 1; + starts[KDIM] = 1; + break; + case BOTTOM: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = 1; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = 1; + starts[JDIM] = 1 - offset; + starts[KDIM] = 1; + break; + case TOP: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = 1; + subSizes[KDIM] = kmaxLocal; + starts[IDIM] = 1; + starts[JDIM] = jmaxLocal + offset; + starts[KDIM] = 1; + break; + case FRONT: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = 1; + starts[IDIM] = 1; + starts[JDIM] = 1; + starts[KDIM] = 1 - offset; + break; + case BACK: + subSizes[IDIM] = imaxLocal; + subSizes[JDIM] = jmaxLocal; + subSizes[KDIM] = 1; + starts[IDIM] = 1; + starts[JDIM] = 1; + starts[KDIM] = kmaxLocal + offset; + break; + case NDIRS: + printf("ERROR!\n"); + break; + } + + if (layer == HALO) { + MPI_Type_create_subarray(NDIMS, + sizes, + subSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &c->rbufferTypes[direction]); + MPI_Type_commit(&c->rbufferTypes[direction]); + } else if (layer == BULK) { + MPI_Type_create_subarray(NDIMS, + sizes, + subSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &c->sbufferTypes[direction]); + MPI_Type_commit(&c->sbufferTypes[direction]); + } +} + +static void assembleResult(Comm* c, + double* src, + double* dst, + int imaxLocal[], + int jmaxLocal[], + int kmaxLocal[], + int offset[], + int kmax, + int jmax, + int imax) +{ + int numRequests = 1; + + if (c->rank == 0) { + numRequests = c->size + 1; + } + + MPI_Request requests[numRequests]; + + /* all ranks send their interpolated bulk array */ + MPI_Isend(src, + c->imaxLocal * c->jmaxLocal * c->kmaxLocal, + MPI_DOUBLE, + 0, + 0, + c->comm, + &requests[0]); + + /* rank 0 assembles the subdomains */ + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + MPI_Datatype domainType; + int oldSizes[NDIMS] = { kmax, jmax, imax }; + int newSizes[NDIMS] = { kmaxLocal[i], jmaxLocal[i], imaxLocal[i] }; + int starts[NDIMS] = { offset[i * NDIMS + KDIM], + offset[i * NDIMS + JDIM], + offset[i * NDIMS + IDIM] }; + MPI_Type_create_subarray(NDIMS, + oldSizes, + newSizes, + starts, + MPI_ORDER_C, + MPI_DOUBLE, + &domainType); + MPI_Type_commit(&domainType); + + MPI_Irecv(dst, 1, domainType, i, 0, c->comm, &requests[i + 1]); + MPI_Type_free(&domainType); + } + } + + MPI_Waitall(numRequests, requests, MPI_STATUSES_IGNORE); +} + +// subroutines local to this module +static int sum(int* sizes, int init, int offset, int coord) +{ + int sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} +#endif // defined _MPI + +// exported subroutines +void commReduction(double* v, int op) +{ +#if defined(_MPI) + if (op == MAX) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + } else if (op == SUM) { + MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + } +#endif +} + +int commIsBoundary(Comm* c, Direction direction) +{ +#if defined(_MPI) + switch (direction) { + case LEFT: + return c->coords[ICORD] == 0; + break; + case RIGHT: + return c->coords[ICORD] == (c->dims[ICORD] - 1); + break; + case BOTTOM: + return c->coords[JCORD] == 0; + break; + case TOP: + return c->coords[JCORD] == (c->dims[JCORD] - 1); + break; + case FRONT: + return c->coords[KCORD] == 0; + break; + case BACK: + return c->coords[KCORD] == (c->dims[KCORD] - 1); + break; + case NDIRS: + printf("ERROR!\n"); + break; + } +#endif + + return 1; +} + +void commExchange(Comm* c, double* grid) +{ +#if defined(_MPI) + int counts[6] = { 1, 1, 1, 1, 1, 1 }; + MPI_Aint displs[6] = { 0, 0, 0, 0, 0, 0 }; + + MPI_Neighbor_alltoallw(grid, + counts, + displs, + c->sbufferTypes, + grid, + counts, + displs, + c->rbufferTypes, + c->comm); +#endif +} + +void commShift(Comm* c, double* f, double* g, double* h) +{ +#if defined(_MPI) + MPI_Request requests[6] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; + + /* shift G */ + /* receive ghost cells from bottom neighbor */ + MPI_Irecv(g, + 1, + c->rbufferTypes[BOTTOM], + c->neighbours[BOTTOM], + 0, + c->comm, + &requests[0]); + + /* send ghost cells to top neighbor */ + MPI_Isend(g, 1, c->sbufferTypes[TOP], c->neighbours[TOP], 0, c->comm, &requests[1]); + + /* shift F */ + /* receive ghost cells from left neighbor */ + MPI_Irecv(f, 1, c->rbufferTypes[LEFT], c->neighbours[LEFT], 1, c->comm, &requests[2]); + + /* send ghost cells to right neighbor */ + MPI_Isend(f, + 1, + c->sbufferTypes[RIGHT], + c->neighbours[RIGHT], + 1, + c->comm, + &requests[3]); + + /* shift H */ + /* receive ghost cells from front neighbor */ + MPI_Irecv(h, + 1, + c->rbufferTypes[FRONT], + c->neighbours[FRONT], + 2, + c->comm, + &requests[4]); + + /* send ghost cells to back neighbor */ + MPI_Isend(h, 1, c->sbufferTypes[BACK], c->neighbours[BACK], 2, c->comm, &requests[5]); + + MPI_Waitall(6, requests, MPI_STATUSES_IGNORE); +#endif +} + +void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax) +{ +#if defined(_MPI) + int sum = 0; + + for (int i = 0; i < c->coords[ICORD]; i++) { + sum += sizeOfRank(i, c->dims[ICORD], imax); + } + offsets[IDIM] = sum; + sum = 0; + + for (int i = 0; i < c->coords[JCORD]; i++) { + sum += sizeOfRank(i, c->dims[JCORD], jmax); + } + offsets[JDIM] = sum; + sum = 0; + + for (int i = 0; i < c->coords[KCORD]; i++) { + sum += sizeOfRank(i, c->dims[KCORD], kmax); + } + offsets[KDIM] = sum; +#endif +} + +#define G(v, i, j, k) \ + v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +void commCollectResult(Comm* c, + double* ug, + double* vg, + double* wg, + double* pg, + double* u, + double* v, + double* w, + double* p, + int kmax, + int jmax, + int imax) +{ + int imaxLocal = c->imaxLocal; + int jmaxLocal = c->jmaxLocal; + int kmaxLocal = c->kmaxLocal; +#if defined(_MPI) + int offset[c->size * NDIMS]; + int imaxLocalAll[c->size]; + int jmaxLocalAll[c->size]; + int kmaxLocalAll[c->size]; + + MPI_Gather(&imaxLocal, 1, MPI_INT, imaxLocalAll, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&jmaxLocal, 1, MPI_INT, jmaxLocalAll, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&kmaxLocal, 1, MPI_INT, kmaxLocalAll, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (c->rank == 0) { + for (int i = 0; i < c->size; i++) { + int coords[NCORDS]; + MPI_Cart_coords(c->comm, i, NDIMS, coords); + offset[i * NDIMS + IDIM] = sum(imaxLocalAll, + i, + c->dims[IDIM] * c->dims[JDIM], + coords[ICORD]); + offset[i * NDIMS + JDIM] = sum(jmaxLocalAll, i, c->dims[IDIM], coords[JCORD]); + offset[i * NDIMS + KDIM] = sum(kmaxLocalAll, i, 1, coords[KCORD]); + + printf("Rank: %d, Coords(k,j,i): %d %d %d, Size(k,j,i): %d %d %d, " + "Offset(k,j,i): %d %d %d\n", + i, + coords[KCORD], + coords[JCORD], + coords[ICORD], + kmaxLocalAll[i], + jmaxLocalAll[i], + imaxLocalAll[i], + offset[i * NDIMS + KDIM], + offset[i * NDIMS + JDIM], + offset[i * NDIMS + IDIM]); + } + } + + size_t bytesize = imaxLocal * jmaxLocal * kmaxLocal * sizeof(double); + double* tmp = allocate(64, bytesize); + int idx = 0; + + /* collect P */ + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = G(p, i, j, k); + } + } + } + + assembleResult(c, + tmp, + pg, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + /* collect U */ + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(u, i, j, k) + G(u, i - 1, j, k)) / 2.0; + } + } + } + + assembleResult(c, + tmp, + ug, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + /* collect V */ + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(v, i, j, k) + G(v, i, j - 1, k)) / 2.0; + } + } + } + + assembleResult(c, + tmp, + vg, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + /* collect W */ + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(w, i, j, k) + G(w, i, j, k - 1)) / 2.0; + } + } + } + + assembleResult(c, + tmp, + wg, + imaxLocalAll, + jmaxLocalAll, + kmaxLocalAll, + offset, + kmax, + jmax, + imax); + + free(tmp); +#else + int idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + pg[idx++] = G(p, i, j, k); + } + } + } + + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + ug[idx++] = (G(u, i, j, k) + G(u, i - 1, j, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + vg[idx++] = (G(v, i, j, k) + G(v, i, j - 1, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + wg[idx++] = (G(w, i, j, k) + G(w, i, j, k - 1)) / 2.0; + } + } + } +#endif +} + +void commPrintConfig(Comm* c) +{ +#if defined(_MPI) + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + if (commIsMaster(c)) { + printf("Communication setup:\n"); + } + + for (int i = 0; i < c->size; i++) { + if (i == c->rank) { + printf("\tRank %d of %d\n", c->rank, c->size); + printf("\tNeighbours (front, back, bottom, top, left, right): %d, %d, %d, " + "%d, %d, %d\n", + c->neighbours[FRONT], + c->neighbours[BACK], + c->neighbours[BOTTOM], + c->neighbours[TOP], + c->neighbours[LEFT], + c->neighbours[RIGHT]); + printf("\tCoordinates (k,j,i) %d %d %d\n", + c->coords[KCORD], + c->coords[JCORD], + c->coords[ICORD]); + printf("\tLocal domain size (k,j,i) %dx%dx%d\n", + c->kmaxLocal, + c->jmaxLocal, + c->imaxLocal); + fflush(stdout); + } + } + MPI_Barrier(MPI_COMM_WORLD); +#endif +} + +void commInit(Comm* c, int argc, char** argv) +{ +#if defined(_MPI) + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank)); + MPI_Comm_size(MPI_COMM_WORLD, &(c->size)); +#else + c->rank = 0; + c->size = 1; +#endif +} + +void commPartition(Comm* c, int kmax, int jmax, int imax) +{ +#if defined(_MPI) + int dims[NDIMS] = { 0, 0, 0 }; + int periods[NDIMS] = { 0, 0, 0 }; + MPI_Dims_create(c->size, NDIMS, dims); + MPI_Cart_create(MPI_COMM_WORLD, NCORDS, dims, periods, 0, &c->comm); + MPI_Cart_shift(c->comm, ICORD, 1, &c->neighbours[LEFT], &c->neighbours[RIGHT]); + MPI_Cart_shift(c->comm, JCORD, 1, &c->neighbours[BOTTOM], &c->neighbours[TOP]); + MPI_Cart_shift(c->comm, KCORD, 1, &c->neighbours[FRONT], &c->neighbours[BACK]); + MPI_Cart_get(c->comm, NCORDS, c->dims, periods, c->coords); + + c->imaxLocal = sizeOfRank(c->coords[KDIM], dims[ICORD], imax); + c->jmaxLocal = sizeOfRank(c->coords[JDIM], dims[JCORD], jmax); + c->kmaxLocal = sizeOfRank(c->coords[IDIM], dims[KCORD], kmax); + + // setup buffer types for communication + setupCommunication(c, LEFT, BULK); + setupCommunication(c, LEFT, HALO); + setupCommunication(c, RIGHT, BULK); + setupCommunication(c, RIGHT, HALO); + setupCommunication(c, BOTTOM, BULK); + setupCommunication(c, BOTTOM, HALO); + setupCommunication(c, TOP, BULK); + setupCommunication(c, TOP, HALO); + setupCommunication(c, FRONT, BULK); + setupCommunication(c, FRONT, HALO); + setupCommunication(c, BACK, BULK); + setupCommunication(c, BACK, HALO); +#else + c->imaxLocal = imax; + c->jmaxLocal = jmax; + c->kmaxLocal = kmax; +#endif +} + +void commFinalize(Comm* c) +{ +#if defined(_MPI) + for (int i = 0; i < NDIRS; i++) { + MPI_Type_free(&c->sbufferTypes[i]); + MPI_Type_free(&c->rbufferTypes[i]); + } + + MPI_Finalize(); +#endif +} + +void commUpdateDatatypes( + Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal) +{ +#if defined _MPI + + int result = MPI_Comm_dup(oldcomm->comm, &newcomm->comm); + + if (result == MPI_ERR_COMM) { + printf("\nNull communicator. Duplication failed !!\n"); + } + + newcomm->rank = oldcomm->rank; + newcomm->size = oldcomm->size; + + newcomm->imaxLocal = imaxLocal / 2; + newcomm->jmaxLocal = jmaxLocal / 2; + newcomm->kmaxLocal = kmaxLocal / 2; + + setupCommunication(newcomm, LEFT, BULK); + setupCommunication(newcomm, LEFT, HALO); + setupCommunication(newcomm, RIGHT, BULK); + setupCommunication(newcomm, RIGHT, HALO); + setupCommunication(newcomm, BOTTOM, BULK); + setupCommunication(newcomm, BOTTOM, HALO); + setupCommunication(newcomm, TOP, BULK); + setupCommunication(newcomm, TOP, HALO); + setupCommunication(newcomm, FRONT, BULK); + setupCommunication(newcomm, FRONT, HALO); + setupCommunication(newcomm, BACK, BULK); + setupCommunication(newcomm, BACK, HALO); +#else + newcomm->imaxLocal = imaxLocal; + newcomm->jmaxLocal = jmaxLocal; + newcomm->kmaxLocal = kmaxLocal; +#endif +} + +void commFreeCommunicator(Comm* comm) +{ +#ifdef _MPI + MPI_Comm_free(&comm->comm); +#endif +} \ No newline at end of file diff --git a/BasicSolver/3D-mpi-io/src/comm.h b/EnhancedSolver/3D-mpi/src/comm.h similarity index 69% rename from BasicSolver/3D-mpi-io/src/comm.h rename to EnhancedSolver/3D-mpi/src/comm.h index fead94f..58ed4d7 100644 --- a/BasicSolver/3D-mpi-io/src/comm.h +++ b/EnhancedSolver/3D-mpi/src/comm.h @@ -1,12 +1,14 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ #ifndef __COMM_H_ #define __COMM_H_ +#if defined(_MPI) #include +#endif /* * Spatial directions: * ICORD (0) from 0 (LEFT) to imax (RIGHT) @@ -21,26 +23,44 @@ typedef enum dimension { KDIM = 0, JDIM, IDIM, NDIMS } Dimension; enum layer { HALO = 0, BULK }; enum op { MAX = 0, SUM }; + typedef struct { int rank; int size; +#if defined(_MPI) MPI_Comm comm; MPI_Datatype sbufferTypes[NDIRS]; MPI_Datatype rbufferTypes[NDIRS]; +#endif int neighbours[NDIRS]; int coords[NDIMS], dims[NDIMS]; int imaxLocal, jmaxLocal, kmaxLocal; - MPI_File fh; } Comm; -extern void commInit(Comm* comm, int kmax, int jmax, int imax); -extern void commFree(Comm* comm); +extern void commInit(Comm* c, int argc, char** argv); +extern void commPartition(Comm* c, int kmax, int jmax, int imax); +extern void commFinalize(Comm* comm); extern void commPrintConfig(Comm*); extern void commExchange(Comm*, double*); extern void commShift(Comm* c, double* f, double* g, double* h); extern void commReduction(double* v, int op); extern int commIsBoundary(Comm* c, Direction direction); extern void commGetOffsets(Comm* c, int offsets[], int kmax, int jmax, int imax); +extern void commFreeCommunicator(Comm* comm); +extern void commUpdateDatatypes( + Comm* oldcomm, Comm* newcomm, int imaxLocal, int jmaxLocal, int kmaxLocal); +extern void commCollectResult(Comm* c, + double* ug, + double* vg, + double* wg, + double* pg, + double* u, + double* v, + double* w, + double* p, + int kmax, + int jmax, + int imax); static inline int commIsMaster(Comm* c) { return c->rank == 0; } #endif // __COMM_H_ diff --git a/EnhancedSolver/3D-mpi/src/discretization.c b/EnhancedSolver/3D-mpi/src/discretization.c new file mode 100644 index 0000000..c77408a --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/discretization.c @@ -0,0 +1,1356 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "allocate.h" +#include "discretization.h" +#include "parameter.h" +#include "util.h" + +static double distance( + double i, double j, double k, double iCenter, double jCenter, double kCenter) +{ + return sqrt(pow(iCenter - i, 2) + pow(jCenter - j, 2) + pow(kCenter - k, 2) * 1.0); +} + +static double sumOffset(double* sizes, int init, int offset, int coord) +{ + double sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +static void printConfig(Discretization* d) +{ + if (commIsMaster(&d->comm)) { + printf("Parameters for #%s#\n", d->problem); + printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n", + d->bcLeft, + d->bcRight, + d->bcBottom, + d->bcTop, + d->bcFront, + d->bcBack); + printf("\tReynolds number: %.2f\n", d->re); + printf("\tGx Gy: %.2f %.2f %.2f\n", d->gx, d->gy, d->gz); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + d->grid.xlength, + d->grid.ylength, + d->grid.zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", + d->grid.imax, + d->grid.jmax, + d->grid.kmax); + printf("\tCell size (dx, dy, dz): %f, %f, %f\n", + d->grid.dx, + d->grid.dy, + d->grid.dz); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te); + printf("\tdt bound: %.6f\n", d->dtBound); + printf("\tTau factor: %.2f\n", d->tau); + printf("Iterative parameters:\n"); + printf("\tMax iterations: %d\n", d->itermax); + printf("\tepsilon (stopping tolerance) : %f\n", d->eps); + printf("\tgamma factor: %f\n", d->gamma); + printf("\tomega (SOR relaxation): %f\n", d->omega); + } + commPrintConfig(&d->comm); +} + +void printGrid(Discretization* d, double* s) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + for (int k = 0; k < kmaxLocal + 2; k++) { + printf("K : %02d:\n", k); + for (int j = 0; j < jmaxLocal + 2; j++) { + printf("J : %02d: ", j); + for (int i = 0; i problem = params->name; + d->bcLeft = params->bcLeft; + d->bcRight = params->bcRight; + d->bcBottom = params->bcBottom; + d->bcTop = params->bcTop; + d->bcFront = params->bcFront; + d->bcBack = params->bcBack; + + d->grid.imax = params->imax; + d->grid.jmax = params->jmax; + d->grid.kmax = params->kmax; + d->grid.xlength = params->xlength; + d->grid.ylength = params->ylength; + d->grid.zlength = params->zlength; + d->grid.dx = params->xlength / params->imax; + d->grid.dy = params->ylength / params->jmax; + d->grid.dz = params->zlength / params->kmax; + + d->eps = params->eps; + d->omega = params->omg; + d->itermax = params->itermax; + d->re = params->re; + d->gx = params->gx; + d->gy = params->gy; + d->gz = params->gz; + d->dt = params->dt; + d->te = params->te; + d->tau = params->tau; + d->gamma = params->gamma; + + /* allocate arrays */ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + size_t size = (imaxLocal + 2) * (jmaxLocal + 2) * (kmaxLocal + 2); + + d->u = allocate(64, size * sizeof(double)); + d->v = allocate(64, size * sizeof(double)); + d->w = allocate(64, size * sizeof(double)); + d->p = allocate(64, size * sizeof(double)); + d->rhs = allocate(64, size * sizeof(double)); + d->f = allocate(64, size * sizeof(double)); + d->g = allocate(64, size * sizeof(double)); + d->h = allocate(64, size * sizeof(double)); + d->s = allocate(64, size * sizeof(double)); + + for (int i = 0; i < size; i++) { + d->u[i] = params->u_init; + d->v[i] = params->v_init; + d->w[i] = params->w_init; + d->p[i] = params->p_init; + d->rhs[i] = 0.0; + d->f[i] = 0.0; + d->g[i] = 0.0; + d->h[i] = 0.0; + d->s[i] = FLUID; + } + + double dx = d->grid.dx; + double dy = d->grid.dy; + double dz = d->grid.dz; + + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz); + d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; + + double xCenter = 0, yCenter = 0, zCenter = 0, radius = 0; + double x1 = 0, x2 = 0, y1 = 0, y2 = 0, z1 = 0, z2 = 0; + int iOffset = 0, jOffset = 0, kOffset = 0; + + d->xLocal = d->comm.imaxLocal * d->grid.dx; + d->yLocal = d->comm.jmaxLocal * d->grid.dy; + d->zLocal = d->comm.kmaxLocal * d->grid.dz; + +#ifdef _MPI + double xLocal[d->comm.size]; + double yLocal[d->comm.size]; + double zLocal[d->comm.size]; + + MPI_Allgather(&d->xLocal, 1, MPI_DOUBLE, xLocal, 1, MPI_DOUBLE, d->comm.comm); + MPI_Allgather(&d->yLocal, 1, MPI_DOUBLE, yLocal, 1, MPI_DOUBLE, d->comm.comm); + MPI_Allgather(&d->zLocal, 1, MPI_DOUBLE, zLocal, 1, MPI_DOUBLE, d->comm.comm); + + d->xOffset = sumOffset(xLocal, + d->comm.rank, + (d->comm.dims[1] * d->comm.dims[2]), + d->comm.coords[0]); + d->yOffset = sumOffset(yLocal, d->comm.rank, d->comm.dims[2], d->comm.coords[1]); + d->zOffset = sumOffset(zLocal, d->comm.rank, 1, d->comm.coords[2]); + d->xOffsetEnd = d->xOffset + d->xLocal; + d->yOffsetEnd = d->yOffset + d->yLocal; + d->zOffsetEnd = d->zOffset + d->zLocal; + +#else + d->xOffset = 0; + d->yOffset = 0; + d->zOffset = 0; + d->xOffsetEnd = d->xOffset + d->xLocal; + d->yOffsetEnd = d->yOffset + d->yLocal; + d->zOffsetEnd = d->zOffset + d->zLocal; +#endif + + iOffset = d->xOffset / dx; + jOffset = d->yOffset / dy; + kOffset = d->zOffset / dz; + + double* s = d->s; + + switch (params->shape) { + case NOSHAPE: + break; + case RECT: + x1 = params->xCenter - params->xRectLength / 2; + x2 = params->xCenter + params->xRectLength / 2; + y1 = params->yCenter - params->yRectLength / 2; + y2 = params->yCenter + params->yRectLength / 2; + z1 = params->zCenter - params->zRectLength / 2; + z2 = params->zCenter + params->zRectLength / 2; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if ((x1 <= ((i + iOffset) * dx)) && (((i + iOffset) * dx) <= x2) && + (y1 <= ((j + jOffset) * dy)) && (((j + jOffset) * dy) <= y2) && + ((z1 <= ((k + kOffset) * dz)) && (((k + kOffset) * dz) <= z2))) { + S(i, j, k) = OBSTACLE; + } + } + } + } + + break; + case CIRCLE: + + xCenter = params->xCenter; + yCenter = params->yCenter; + zCenter = params->zCenter; + radius = params->circleRadius; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if (distance(((i + iOffset) * dx), + ((j + jOffset) * dy), + ((k + kOffset) * dz), + xCenter, + yCenter, + zCenter) <= radius) { + S(i, j, k) = OBSTACLE; + } + } + } + } + break; + default: + break; + } + +#ifdef _MPI + commExchange(&d->comm, s); +#endif + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + + /* Assigning enum values to Corners */ + if (S(i - 1, j + 1, k - 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i + 1, j - 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPLEFTCORNER; // + } + if (S(i + 1, j + 1, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j - 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPRIGHTCORNER; // + } + if (S(i - 1, j - 1, k - 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i + 1, j + 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMLEFTCORNER; // + } + if (S(i + 1, j - 1, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j + 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMRIGHTCORNER; // + } + if (S(i - 1, j + 1, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i + 1, j - 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPLEFTCORNER; // + } + if (S(i + 1, j + 1, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i - 1, j - 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPRIGHTCORNER; + } + if (S(i - 1, j - 1, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i + 1, j + 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMLEFTCORNER; + } + if (S(i + 1, j - 1, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i - 1, j + 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMRIGHTCORNER; + } + /* Assigning enum values to Lines */ + if (S(i - 1, j, k - 1) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j, k) == FLUID && S(i + 1, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTLEFTLINE; + } + if (S(i + 1, j, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j, k - 1) == FLUID && S(i - 1, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTRIGHTLINE; + } + if (S(i, j + 1, k - 1) == FLUID && S(i, j + 1, k) == FLUID && + S(i, j, k - 1) == FLUID && S(i, j - 1, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPLINE; + } + if (S(i, j - 1, k - 1) == FLUID && S(i, j, k - 1) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j + 1, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMLINE; + } + if (S(i - 1, j + 1, k) == FLUID && S(i, j + 1, k) == FLUID && + S(i - 1, j, k) == FLUID && S(i + 1, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDTOPLEFTLINE; + } + if (S(i + 1, j + 1, k) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i - 1, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDTOPRIGHTLINE; + } + if (S(i - 1, j - 1, k) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i + 1, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDBOTTOMLEFTLINE; + } + if (S(i + 1, j - 1, k) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i - 1, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDBOTTOMRIGHTLINE; + } + if (S(i - 1, j, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i + 1, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKLEFTLINE; + } + if (S(i + 1, j, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i - 1, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKRIGHTLINE; + } + if (S(i, j + 1, k + 1) == FLUID && S(i, j + 1, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i, j - 1, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPLINE; + } + if (S(i, j - 1, k + 1) == FLUID && S(i, j - 1, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i, j + 1, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMLINE; + } + /* Assigning enum values to Faces */ + if (S(i, j, k - 1) == FLUID && S(i, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTFACE; // + } + if (S(i, j, k + 1) == FLUID && S(i, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKFACE; // + } + if (S(i, j - 1, k) == FLUID && S(i, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BOTTOMFACE; // + } + if (S(i, j + 1, k) == FLUID && S(i, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = TOPFACE; // + } + if (S(i - 1, j, k) == FLUID && S(i + 1, j, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = LEFTFACE; // + } + if (S(i + 1, j, k) == FLUID && S(i - 1, j, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = RIGHTFACE; // + } + } + } + } + +#ifdef VERBOSE + printConfig(d); +#endif /* VERBOSE */ +} + +void setBoundaryConditions(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double* u = d->u; + double* v = d->v; + double* w = d->w; + + if (commIsBoundary(&d->comm, TOP)) { + switch (d->bcTop) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = -U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = 0.0; + W(i, jmaxLocal + 1, k) = -W(i, jmaxLocal, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = 0.0; + W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, jmaxLocal + 1, k) = U(i, jmaxLocal, k); + V(i, jmaxLocal, k) = V(i, jmaxLocal - 1, k); + W(i, jmaxLocal + 1, k) = W(i, jmaxLocal, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, BOTTOM)) { + switch (d->bcBottom) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = -U(i, 1, k); + V(i, 0, k) = 0.0; + W(i, 0, k) = -W(i, 1, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = 0.0; + W(i, 0, k) = W(i, 1, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = V(i, 1, k); + W(i, 0, k) = W(i, 1, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, LEFT)) { + switch (d->bcLeft) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = -V(1, j, k); + W(0, j, k) = -W(1, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = U(1, j, k); + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, RIGHT)) { + switch (d->bcRight) { + case NOSLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = 0.0; + V(imaxLocal + 1, j, k) = -V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = -W(imaxLocal, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = 0.0; + V(imaxLocal + 1, j, k) = V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = W(imaxLocal, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(imaxLocal, j, k) = U(imaxLocal - 1, j, k); + V(imaxLocal + 1, j, k) = V(imaxLocal, j, k); + W(imaxLocal + 1, j, k) = W(imaxLocal, j, k); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, FRONT)) { + switch (d->bcFront) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = -U(i, j, 1); + V(i, j, 0) = -V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = W(i, j, 1); + } + } + break; + case PERIODIC: + break; + } + } + + if (commIsBoundary(&d->comm, BACK)) { + switch (d->bcBack) { + case NOSLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = -U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = -V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, kmaxLocal + 1) = U(i, j, kmaxLocal); + V(i, j, kmaxLocal + 1) = V(i, j, kmaxLocal); + W(i, j, kmaxLocal) = W(i, j, kmaxLocal - 1); + } + } + break; + case PERIODIC: + break; + } + } +} + +void computeRHS(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double idx = 1.0 / d->grid.dx; + double idy = 1.0 / d->grid.dy; + double idz = 1.0 / d->grid.dz; + double idt = 1.0 / d->dt; + + double* rhs = d->rhs; + double* f = d->f; + double* g = d->g; + double* h = d->h; + + commShift(&d->comm, f, g, h); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx + + (G(i, j, k) - G(i, j - 1, k)) * idy + + (H(i, j, k) - H(i, j, k - 1)) * idz) * + idt; + } + } + } +} + +void setSpecialBoundaryCondition(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double* u = d->u; + + if (strcmp(d->problem, "dcavity") == 0) { + if (commIsBoundary(&d->comm, TOP)) { + for (int k = 1; k < kmaxLocal; k++) { + for (int i = 1; i < imaxLocal; i++) { + U(i, jmaxLocal + 1, k) = 2.0 - U(i, jmaxLocal, k); + } + } + } + } else if (strcmp(d->problem, "canal") == 0) { + if (commIsBoundary(&d->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(0, j, k) = 2.0; + } + } + } + } else if (strcmp(d->problem, "backstep") == 0) { + if (commIsBoundary(&d->comm, LEFT)) { + double* s = d->s; + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + if (S(1, j, k) == FLUID) U(0, j, k) = 1.0; + else { + U(0, j, k) = 0.0; + U(1, j, k) = 0.0; + } + } + } + } + } else if (strcmp(d->problem, "karman") == 0) { + if (commIsBoundary(&d->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + U(1, j, k) = 1.0; + } + } + } + } +} + +static double maxElement(Discretization* d, double* m) +{ + int size = (d->comm.imaxLocal + 2) * (d->comm.jmaxLocal + 2) * + (d->comm.kmaxLocal + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + commReduction(&maxval, MAX); + return maxval; +} + +void normalizePressure(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double* p = d->p; + double avgP = 0.0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + avgP += P(i, j, k); + } + } + } + commReduction(&avgP, SUM); + avgP /= (d->grid.imax * d->grid.jmax * d->grid.kmax); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, k) = P(i, j, k) - avgP; + } + } + } +} + +void computeTimestep(Discretization* d) +{ + double dt = d->dtBound; + double dx = d->grid.dx; + double dy = d->grid.dy; + double dz = d->grid.dz; + + double umax = maxElement(d, d->u); + double vmax = maxElement(d, d->v); + double wmax = maxElement(d, d->w); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + if (wmax > 0) { + dt = (dt > dz / wmax) ? dz / wmax : dt; + } + + d->dt = dt * d->tau; +} + +void setObjectBoundaryCondition(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* s = d->s; + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + switch ((int)S(i, j, k)) { + case TOPFACE: + U(i, j, k) = -U(i, j + 1, k); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + ; + break; + case BOTTOMFACE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case LEFTFACE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i - 1, j, k); + break; + case RIGHTFACE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i + 1, j, k); + break; + case FRONTFACE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = 0.0; + break; + case BACKFACE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = 0.0; + break; + case FRONTLEFTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = -W(i - 1, j, k); + break; + case FRONTRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = -W(i + 1, j, k); + break; + case FRONTTOPLINE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTBOTTOMLINE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case MIDTOPLEFTLINE: + U(i, j, k) = -U(i, j + 1, k); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = 0.0; + break; + case MIDTOPRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = 0.0; + U(i - 1, j, k) = -U(i - 1, j + 1, k); + V(i, j - 1, k) = -V(i + 1, j - 1, k); + W(i, j, k) = 0.0; + break; + case MIDBOTTOMLEFTLINE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = 0.0; + break; + case MIDBOTTOMRIGHTLINE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = 0.0; + break; + case BACKLEFTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = -W(i - 1, j, k); + break; + case BACKRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = -W(i + 1, j, k); + break; + case BACKTOPLINE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKBOTTOMLINE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case FRONTTOPLEFTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTTOPRIGHTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTBOTTOMLEFTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case FRONTBOTTOMRIGHTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case BACKTOPLEFTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKTOPRIGHTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKBOTTOMLEFTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case BACKBOTTOMRIGHTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + } + } + } + } +} + +void computeFG(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* f = d->f; + double* g = d->g; + double* h = d->h; + double* s = d->s; + + double gx = d->gx; + double gy = d->gy; + double gz = d->gz; + double dt = d->dt; + + double gamma = d->gamma; + double inverseRe = 1.0 / d->re; + double inverseDx = 1.0 / d->grid.dx; + double inverseDy = 1.0 / d->grid.dy; + double inverseDz = 1.0 / d->grid.dz; + double du2dx, dv2dy, dw2dz; + double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz; + double du2dx2, du2dy2, du2dz2; + double dv2dx2, dv2dy2, dv2dz2; + double dw2dx2, dw2dy2, dw2dz2; + + commExchange(&d->comm, u); + commExchange(&d->comm, v); + commExchange(&d->comm, w); + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + if (S(i, j, k) == FLUID) { + du2dx = inverseDx * 0.25 * + ((U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) + U(i + 1, j, k)) - + (U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) + U(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) - U(i + 1, j, k)) + + fabs(U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) - U(i - 1, j, k))); + + duvdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) + U(i, j + 1, k)) - + (V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) + U(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) - U(i, j + 1, k)) + + fabs(V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) - U(i, j - 1, k))); + + duwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) + U(i, j, k + 1)) - + (W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) + U(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) - U(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) - U(i, j, k - 1))); + + du2dx2 = inverseDx * inverseDx * + (U(i + 1, j, k) - 2.0 * U(i, j, k) + U(i - 1, j, k)); + du2dy2 = inverseDy * inverseDy * + (U(i, j + 1, k) - 2.0 * U(i, j, k) + U(i, j - 1, k)); + du2dz2 = inverseDz * inverseDz * + (U(i, j, k + 1) - 2.0 * U(i, j, k) + U(i, j, k - 1)); + F(i, j, k) = U(i, j, k) + + dt * (inverseRe * (du2dx2 + du2dy2 + du2dz2) - du2dx - + duvdy - duwdz + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) + V(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) + V(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) - V(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) - V(i - 1, j, k))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) + V(i, j + 1, k)) - + (V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) + V(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) - V(i, j + 1, k)) + + fabs(V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) - V(i, j - 1, k))); + + dvwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) + V(i, j, k + 1)) - + (W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) + V(i, j, k + 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) - V(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) - V(i, j, k + 1))); + + dv2dx2 = inverseDx * inverseDx * + (V(i + 1, j, k) - 2.0 * V(i, j, k) + V(i - 1, j, k)); + dv2dy2 = inverseDy * inverseDy * + (V(i, j + 1, k) - 2.0 * V(i, j, k) + V(i, j - 1, k)); + dv2dz2 = inverseDz * inverseDz * + (V(i, j, k + 1) - 2.0 * V(i, j, k) + V(i, j, k - 1)); + G(i, j, k) = V(i, j, k) + + dt * (inverseRe * (dv2dx2 + dv2dy2 + dv2dz2) - duvdx - + dv2dy - dvwdz + gy); + + duwdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) + W(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) + W(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) - W(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) - W(i - 1, j, k))); + + dvwdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) + W(i, j + 1, k)) - + (V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) + W(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) - W(i, j + 1, k)) + + fabs(V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) - W(i, j - 1, k))); + + dw2dz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) + W(i, j, k + 1)) - + (W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) + W(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) - W(i, j, k + 1)) + + fabs(W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) - W(i, j, k - 1))); + + dw2dx2 = inverseDx * inverseDx * + (W(i + 1, j, k) - 2.0 * W(i, j, k) + W(i - 1, j, k)); + dw2dy2 = inverseDy * inverseDy * + (W(i, j + 1, k) - 2.0 * W(i, j, k) + W(i, j - 1, k)); + dw2dz2 = inverseDz * inverseDz * + (W(i, j, k + 1) - 2.0 * W(i, j, k) + W(i, j, k - 1)); + H(i, j, k) = W(i, j, k) + + dt * (inverseRe * (dw2dx2 + dw2dy2 + dw2dz2) - duwdx - + dvwdy - dw2dz + gz); + } else { + switch ((int)S(i, j, k)) { + case TOPFACE: + G(i, j, k) = V(i, j, k); + break; + case BOTTOMFACE: + G(i, j, k) = V(i, j, k); + break; + case LEFTFACE: + F(i, j, k) = U(i, j, k); + break; + case RIGHTFACE: + F(i, j, k) = U(i, j, k); + break; + case FRONTFACE: + H(i, j, k) = W(i, j, k); + break; + case BACKFACE: + H(i, j, k) = W(i, j, k); + break; + case FRONTLEFTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTRIGHTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case MIDTOPLEFTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case MIDTOPRIGHTLINE: + F(i, j, k) = U(i, j, k); + G(i, j, k) = V(i, j, k); + break; + case MIDBOTTOMLEFTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case MIDBOTTOMRIGHTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case BACKLEFTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKRIGHTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + } + } + } + } + } + + /* ----------------------------- boundary of F --------------------------- + */ + if (commIsBoundary(&d->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(0, j, k) = U(0, j, k); + } + } + } + + if (commIsBoundary(&d->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + F(imaxLocal, j, k) = U(imaxLocal, j, k); + } + } + } + + /* ----------------------------- boundary of G --------------------------- + */ + if (commIsBoundary(&d->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, 0, k) = V(i, 0, k); + } + } + } + + if (commIsBoundary(&d->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + G(i, jmaxLocal, k) = V(i, jmaxLocal, k); + } + } + } + + /* ----------------------------- boundary of H --------------------------- + */ + if (commIsBoundary(&d->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + H(i, j, 0) = W(i, j, 0); + } + } + } + + if (commIsBoundary(&d->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + H(i, j, kmaxLocal) = W(i, j, kmaxLocal); + } + } + } +} + +void adaptUV(Discretization* d) +{ + int imaxLocal = d->comm.imaxLocal; + int jmaxLocal = d->comm.jmaxLocal; + int kmaxLocal = d->comm.kmaxLocal; + + double* p = d->p; + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* f = d->f; + double* g = d->g; + double* h = d->h; + + double factorX = d->dt / d->grid.dx; + double factorY = d->dt / d->grid.dy; + double factorZ = d->dt / d->grid.dz; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + U(i, j, k) = F(i, j, k) - (P(i + 1, j, k) - P(i, j, k)) * factorX; + V(i, j, k) = G(i, j, k) - (P(i, j + 1, k) - P(i, j, k)) * factorY; + W(i, j, k) = H(i, j, k) - (P(i, j, k + 1) - P(i, j, k)) * factorZ; + } + } + } +} \ No newline at end of file diff --git a/EnhancedSolver/3D-mpi/src/discretization.h b/EnhancedSolver/3D-mpi/src/discretization.h new file mode 100644 index 0000000..c09780c --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/discretization.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ + +#include "comm.h" +#include "grid.h" +#include "parameter.h" +#include "util.h" + +enum OBJECTBOUNDARY { + FLUID = 0, + /* Front Corners */ + FRONTTOPLEFTCORNER, + FRONTTOPRIGHTCORNER, + FRONTBOTTOMLEFTCORNER, + FRONTBOTTOMRIGHTCORNER, + /* Back Corners */ + BACKTOPLEFTCORNER, + BACKTOPRIGHTCORNER, + BACKBOTTOMLEFTCORNER, + BACKBOTTOMRIGHTCORNER, + /* Faces */ + FRONTFACE, + BACKFACE, + LEFTFACE, + RIGHTFACE, + TOPFACE, + BOTTOMFACE, + /* Front Lines remaining after Corners and Faces */ + FRONTLEFTLINE, + FRONTRIGHTLINE, + FRONTTOPLINE, + FRONTBOTTOMLINE, + /* Bottom Lines remaining after Corners and Faces */ + BACKLEFTLINE, + BACKRIGHTLINE, + BACKTOPLINE, + BACKBOTTOMLINE, + /* Mid Lines remaining after Corners and Faces */ + MIDTOPLEFTLINE, + MIDTOPRIGHTLINE, + MIDBOTTOMLEFTLINE, + MIDBOTTOMRIGHTLINE, + /* Local where its an object but not a boundary */ + OBSTACLE, + /*Ghost cells boundary */ + OUTSIDEBOUNDARY +}; + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + +enum SHAPE { NOSHAPE = 0, RECT, CIRCLE }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g, *h; + double *u, *v, *w; + double* s; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy, gz; + /* time stepping */ + int itermax; + double dt, te; + double dtBound; + char* problem; + double xLocal, yLocal, zLocal, xOffset, yOffset, zOffset, xOffsetEnd, yOffsetEnd, + zOffsetEnd; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + Comm comm; +} Discretization; + +extern void initDiscretization(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setObjectBoundaryCondition(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +extern void printGrid(Discretization*, double*); +#endif diff --git a/BasicSolver/3D-mpi-io/src/grid.h b/EnhancedSolver/3D-mpi/src/grid.h similarity index 84% rename from BasicSolver/3D-mpi-io/src/grid.h rename to EnhancedSolver/3D-mpi/src/grid.h index c963429..0689ebc 100644 --- a/BasicSolver/3D-mpi-io/src/grid.h +++ b/EnhancedSolver/3D-mpi/src/grid.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. diff --git a/BasicSolver/2D-mpi-v3/src/likwid-marker.h b/EnhancedSolver/3D-mpi/src/likwid-marker.h similarity index 100% rename from BasicSolver/2D-mpi-v3/src/likwid-marker.h rename to EnhancedSolver/3D-mpi/src/likwid-marker.h diff --git a/EnhancedSolver/3D-mpi/src/main.c b/EnhancedSolver/3D-mpi/src/main.c new file mode 100644 index 0000000..cfd2bf6 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/main.c @@ -0,0 +1,151 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "allocate.h" +#include "discretization.h" +#include "parameter.h" +#include "particletracing.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" +#include "vtkWriter.h" + +int main(int argc, char** argv) +{ + double timeStart, timeStop; + Parameter p; + Solver s; + Discretization d; + ParticleTracer particletracer; + + commInit(&d.comm, argc, argv); + initParameter(&p); + FILE* fp; + if (commIsMaster(&d.comm)) fp = initResidualWriter(); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(&p, argv[1]); + commPartition(&d.comm, p.kmax, p.jmax, p.imax); + + if (commIsMaster(&d.comm)) { + printParameter(&p); + } + + initDiscretization(&d, &p); + initSolver(&s, &d, &p); + initParticleTracer(&particletracer, &p, &d); + + +#ifndef VERBOSE + initProgress(d.te); +#endif + + double tau = d.tau; + double te = d.te; + double t = 0.0; + int nt = 0; + double res = 0.0; + + timeStart = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + setObjectBoundaryCondition(&d); + computeFG(&d); + computeRHS(&d); + + if (nt % 100 == 0) normalizePressure(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + trace(&particletracer, d.u, d.v, d.w, d.s, t); + + if (commIsMaster(&d.comm)) writeResidual(fp, t, res); + + t += d.dt; + nt++; + +#ifdef VERBOSE + if (commIsMaster(s.comm)) { + printf("TIME %f , TIMESTEP %f\n", t, d.dt); + } +#else + printProgress(t); +#endif + } + timeStop = getTimeStamp(); + +#ifndef VERBOSE + stopProgress(); +#endif + + if (commIsMaster(s.comm)) { + printf("Solution took %.2fs\n", timeStop - timeStart); + } + + timeStart = getTimeStamp(); +#ifdef _VTK_WRITER_MPI + VtkOptions opts = { .grid = s.grid, .comm = s.comm }; + vtkOpen(&opts, s.problem); + vtkScalar(&opts, "pressure", d.p); + vtkVector(&opts, "velocity", (VtkVector) { d.u, d.v, d.w }); + vtkClose(&opts); +#else + + if (commIsMaster(&d.comm)) fclose(fp); + freeParticles(&particletracer); + + double *pg, *ug, *vg, *wg; + + if (commIsMaster(s.comm)) { + size_t bytesize = s.grid->imax * s.grid->jmax * s.grid->kmax * sizeof(double); + + pg = allocate(64, bytesize); + ug = allocate(64, bytesize); + vg = allocate(64, bytesize); + wg = allocate(64, bytesize); + } + + commCollectResult(s.comm, + ug, + vg, + wg, + pg, + d.u, + d.v, + d.w, + d.p, + s.grid->kmax, + s.grid->jmax, + s.grid->imax); + + if (commIsMaster(s.comm)) { + VtkOptions opts = { .grid = s.grid }; + vtkOpen(&opts, s.problem); + vtkScalar(&opts, "pressure", pg); + vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); + vtkClose(&opts); + } + +#endif + + timeStop = getTimeStamp(); + + if (commIsMaster(s.comm)) { + printf("Result output took %.2fs\n", timeStop - timeStart); + } + + commFinalize(s.comm); + return EXIT_SUCCESS; +} diff --git a/BasicSolver/2D-seq-pt/src/parameter.c b/EnhancedSolver/3D-mpi/src/parameter.c similarity index 59% rename from BasicSolver/2D-seq-pt/src/parameter.c rename to EnhancedSolver/3D-mpi/src/parameter.c index f76e032..aa8e485 100644 --- a/BasicSolver/2D-seq-pt/src/parameter.c +++ b/EnhancedSolver/3D-mpi/src/parameter.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -14,16 +14,21 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->imax = 100; - param->jmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.7; - param->re = 100.0; - param->gamma = 0.9; - param->tau = 0.5; + param->xlength = 1.0; + param->ylength = 1.0; + param->zlength = 1.0; + param->imax = 100; + param->jmax = 100; + param->kmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) @@ -58,9 +63,14 @@ void readParameter(Parameter* param, const char* filename) if (tok != NULL && val != NULL) { PARSE_REAL(xlength); PARSE_REAL(ylength); + PARSE_REAL(zlength); PARSE_INT(imax); PARSE_INT(jmax); + PARSE_INT(kmax); PARSE_INT(itermax); + PARSE_INT(levels); + PARSE_INT(presmooth); + PARSE_INT(postsmooth); PARSE_REAL(eps); PARSE_REAL(omg); PARSE_REAL(re); @@ -70,22 +80,40 @@ void readParameter(Parameter* param, const char* filename) PARSE_REAL(te); PARSE_REAL(gx); PARSE_REAL(gy); + PARSE_REAL(gz); PARSE_STRING(name); PARSE_INT(bcLeft); PARSE_INT(bcRight); PARSE_INT(bcBottom); PARSE_INT(bcTop); + PARSE_INT(bcFront); + PARSE_INT(bcBack); PARSE_REAL(u_init); PARSE_REAL(v_init); + PARSE_REAL(w_init); PARSE_REAL(p_init); - PARSE_REAL(traceStart); - PARSE_REAL(traceWrite); - PARSE_REAL(traceInject); - PARSE_REAL(lineX1); - PARSE_REAL(lineX2); - PARSE_REAL(lineY1); - PARSE_REAL(lineY2); - PARSE_INT(nparticles); + + /* Added new particle tracing parameters */ + PARSE_INT(numberOfParticles); + PARSE_REAL(startTime); + PARSE_REAL(injectTimePeriod); + PARSE_REAL(writeTimePeriod); + PARSE_REAL(x1); + PARSE_REAL(y1); + PARSE_REAL(z1); + PARSE_REAL(x2); + PARSE_REAL(y2); + PARSE_REAL(z2); + + /* Added obstacle geometry parameters */ + PARSE_INT(shape); + PARSE_REAL(xCenter); + PARSE_REAL(yCenter); + PARSE_REAL(zCenter); + PARSE_REAL(xRectLength); + PARSE_REAL(yRectLength); + PARSE_REAL(zRectLength); + PARSE_REAL(circleRadius); } } @@ -95,19 +123,26 @@ void readParameter(Parameter* param, const char* filename) void printParameter(Parameter* param) { printf("Parameters for %s\n", param->name); - printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d\n", + printf("Boundary conditions Left:%d Right:%d Bottom:%d Top:%d Front:%d " + "Back:%d\n", param->bcLeft, param->bcRight, param->bcBottom, - param->bcTop); + param->bcTop, + param->bcFront, + param->bcBack); printf("\tReynolds number: %.2f\n", param->re); - printf("\tInit arrays: U:%.2f V:%.2f P:%.2f\n", + printf("\tInit arrays: U:%.2f V:%.2f W:%.2f P:%.2f\n", param->u_init, param->v_init, + param->w_init, param->p_init); printf("Geometry data:\n"); - printf("\tDomain box size (x, y): %.2f, %.2f\n", param->xlength, param->ylength); - printf("\tCells (x, y): %d, %d\n", param->imax, param->jmax); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + param->xlength, + param->ylength, + param->zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", param->imax, param->jmax, param->kmax); printf("Timestep parameters:\n"); printf("\tDefault stepsize: %.2f, Final time %.2f\n", param->dt, param->te); printf("\tTau factor: %.2f\n", param->tau); @@ -116,14 +151,4 @@ void printParameter(Parameter* param) printf("\tepsilon (stopping tolerance) : %f\n", param->eps); printf("\tgamma (stopping tolerance) : %f\n", param->gamma); printf("\tomega (SOR relaxation): %f\n", param->omg); - printf("Particle tracing:\n"); - printf("\tTrace start: %f\n", param->traceStart); - printf("\tTrace write: %f\n", param->traceWrite); - printf("\tTrace inject: %f\n", param->traceInject); - printf("\tNumber of particles: %d\n", param->nparticles); - printf("\tLine (x1, y1, x2, y2): %f,%f / %f,%f\n", - param->lineX1, - param->lineY1, - param->lineX2, - param->lineY2); } diff --git a/BasicSolver/3D-mpi-io/src/parameter.h b/EnhancedSolver/3D-mpi/src/parameter.h similarity index 66% rename from BasicSolver/3D-mpi-io/src/parameter.h rename to EnhancedSolver/3D-mpi/src/parameter.h index 6dddf5f..553e82a 100644 --- a/BasicSolver/3D-mpi-io/src/parameter.h +++ b/EnhancedSolver/3D-mpi/src/parameter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -18,6 +18,15 @@ typedef struct { char* name; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; double u_init, v_init, w_init, p_init; + int levels, presmooth, postsmooth; + + int numberOfParticles; + double startTime, injectTimePeriod, writeTimePeriod; + + double x1, y1, z1, x2, y2, z2; + + int shape; + double xCenter, yCenter, zCenter, xRectLength, yRectLength, zRectLength, circleRadius; } Parameter; void initParameter(Parameter*); diff --git a/EnhancedSolver/3D-mpi/src/particletracing.c b/EnhancedSolver/3D-mpi/src/particletracing.c new file mode 100644 index 0000000..86386ef --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/particletracing.c @@ -0,0 +1,749 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "particletracing.h" +#include "solver.h" + +static int ts = 0; +unsigned int seed = 32767; + +#define XOFFSET 0 +#define YOFFSET 1 +#define ZOFFSET 2 +#define XOFFSETEND 3 +#define YOFFSETEND 4 +#define ZOFFSETEND 5 + +static double sum(int* sizes, int size) +{ + double sum = 0; + + for (int i = 0; i < size; ++i) { + sum += sizes[i]; + } + + return sum; +} + +static int maxElement(int* sizes, int size) +{ + + int maxValue = sizes[0]; + + for (size_t i = 1; i < size; ++i) { + if (sizes[i] > maxValue) { + maxValue = sizes[i]; + } + } + return maxValue; +} + +static double sumOffset(double* sizes, int init, int offset, int coord) +{ + double sum = 0; + + for (int i = init - offset; coord > 0; i -= offset, --coord) { + sum += sizes[i]; + } + + return sum; +} + +void printParticles(ParticleTracer* particletracer) +{ + for (int i = 0; i < particletracer->totalParticles; ++i) { + printf("Rank : %d Particle position X : %.2f, Y : %.2f, flag : %d, total pt : " + "%d, pointer : %d, xOffset : %.2f, yOffset : %.2f, xOffsetEnd : %.2f, " + "yOffsetEnd : %.2f\n", + particletracer->rank, + particletracer->particlePool[i].x, + particletracer->particlePool[i].y, + particletracer->particlePool[i].flag, + particletracer->totalParticles, + particletracer->pointer, + particletracer->xOffset, + particletracer->yOffset, + particletracer->xOffsetEnd, + particletracer->yOffsetEnd); + } +} +void injectParticles(ParticleTracer* particletracer, double* restrict s) +{ + compress(particletracer); + particleRandomizer(particletracer); + + double x, y, z; + int imaxLocal = particletracer->imaxLocal; + int jmaxLocal = particletracer->jmaxLocal; + int kmaxLocal = particletracer->kmaxLocal; + for (int i = 0; i < particletracer->numberOfParticles; ++i) { + x = particletracer->linSpaceLine[i].x; + y = particletracer->linSpaceLine[i].y; + z = particletracer->linSpaceLine[i].z; + if (x >= particletracer->xOffset && y >= particletracer->yOffset && + z >= particletracer->zOffset && x <= particletracer->xOffsetEnd && + y <= particletracer->yOffsetEnd && y <= particletracer->zOffsetEnd) { + + particletracer->particlePool[particletracer->pointer].x = x; + particletracer->particlePool[particletracer->pointer].y = y; + particletracer->particlePool[particletracer->pointer].z = z; + + int i = particletracer->particlePool[particletracer->pointer].x / + particletracer->dx; + int j = particletracer->particlePool[particletracer->pointer].y / + particletracer->dy; + int k = particletracer->particlePool[particletracer->pointer].z / + particletracer->dz; + + int iOffset = particletracer->xOffset / particletracer->dx, + jOffset = particletracer->yOffset / particletracer->dy, + kOffset = particletracer->zOffset / particletracer->dz; + + if (S(i - iOffset, j - jOffset, k - kOffset) == FLUID) { + particletracer->particlePool[particletracer->pointer].flag = true; + ++(particletracer->pointer); + ++(particletracer->totalParticles); + } + } + } +} + +void advanceParticles(ParticleTracer* particletracer, + double* restrict u, + double* restrict v, + double* restrict w, + double* restrict s, + double time) +{ + int imax = particletracer->imax; + int jmax = particletracer->jmax; + int kmax = particletracer->kmax; + int imaxLocal = particletracer->imaxLocal; + int jmaxLocal = particletracer->jmaxLocal; + int kmaxLocal = particletracer->kmaxLocal; + + double dx = particletracer->dx; + double dy = particletracer->dy; + double dz = particletracer->dz; + + double xlength = particletracer->xlength; + double ylength = particletracer->ylength; + double zlength = particletracer->zlength; + + Particle buff[particletracer->size][(particletracer->estimatedNumParticles / particletracer->size)]; + memset(buff, 0, sizeof(buff)); + Particle recvbuff[particletracer->size][(particletracer->estimatedNumParticles / particletracer->size)]; + memset(buff, 0, sizeof(recvbuff)); + + int particleBufIndex[particletracer->size], + recvparticleBufIndex[particletracer->size]; + + memset(particleBufIndex, 0, sizeof(particleBufIndex)); + memset(recvparticleBufIndex, 0, sizeof(recvparticleBufIndex)); + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (particletracer->particlePool[i].flag == true) { + double xTemp = particletracer->particlePool[i].x; + double yTemp = particletracer->particlePool[i].y; + double zTemp = particletracer->particlePool[i].z; + + double x = xTemp - particletracer->xOffset; + double y = yTemp - particletracer->yOffset; + double z = zTemp - particletracer->zOffset; + + int iCoord = (int)(x / dx) + 1; + int jCoord = (int)((y + 0.5 * dy) / dy) + 1; + int kCoord = (int)((z + 0.5 * dz) / dz) + 1; + + double x1 = (double)(iCoord - 1) * dx; + double y1 = ((double)(jCoord - 1) - 0.5) * dy; + double z1 = ((double)(kCoord - 1) - 0.5) * dz; + + double x2 = (double)iCoord * dx; + double y2 = ((double)jCoord - 0.5) * dy; + double z2 = ((double)kCoord - 0.5) * dz; + + double u_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * U(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * U(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * U(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * U(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * U(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * U(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * U(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * U(iCoord, jCoord, kCoord)); + + double new_x = (x + particletracer->xOffset) + particletracer->dt * u_n; + particletracer->particlePool[i].x = new_x; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)(y / dy) + 1; + kCoord = (int)((z + 0.5 * dz) / dz) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = (double)(jCoord - 1) * dy; + z1 = ((double)(kCoord - 1) - 0.5) * dz; + + x2 = ((double)iCoord - 0.5) * dx; + y2 = (double)jCoord * dy; + z2 = ((double)kCoord - 0.5) * dz; + + double v_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * V(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * V(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * V(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * V(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * V(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * V(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * V(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * V(iCoord, jCoord, kCoord)); + + double new_y = (y + particletracer->yOffset) + particletracer->dt * v_n; + particletracer->particlePool[i].y = new_y; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)((y + 0.5 * dy) / dy) + 1; + kCoord = (int)(z / dz) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = ((double)(jCoord - 1) - 0.5) * dy; + z1 = (double)(kCoord - 1) * dz; + + x2 = ((double)iCoord - 0.5) * dx; + y2 = ((double)jCoord - 0.5) * dy; + z2 = (double)kCoord * dz; + + double w_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * W(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * W(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * W(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * W(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * W(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * W(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * W(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * W(iCoord, jCoord, kCoord)); + + double new_z = (z + particletracer->zOffset) + particletracer->dt * w_n; + particletracer->particlePool[i].z = new_z; + + if (((new_x < particletracer->xOffset) || + (new_x >= particletracer->xOffsetEnd) || + (new_y < particletracer->yOffset) || + (new_y >= particletracer->yOffsetEnd) || + (new_z < particletracer->zOffset) || + (new_z >= particletracer->zOffsetEnd))) { + // New logic to transfer particles to neighbouring ranks or discard the + // particle. + +#ifdef _MPI + for (int i = 0; i < particletracer->size; ++i) { + if ((new_x >= + particletracer->offset[i + particletracer->size * XOFFSET]) && + (new_x <= particletracer + ->offset[i + particletracer->size * XOFFSETEND]) && + (new_y >= + particletracer->offset[i + particletracer->size * YOFFSET]) && + (new_y <= particletracer + ->offset[i + particletracer->size * YOFFSETEND]) && + (new_z >= + particletracer->offset[i + particletracer->size * ZOFFSET]) && + (new_z <= particletracer + ->offset[i + particletracer->size * ZOFFSETEND]) && + i != particletracer->rank) { + buff[i][particleBufIndex[i]].x = new_x; + buff[i][particleBufIndex[i]].y = new_y; + buff[i][particleBufIndex[i]].z = new_z; + buff[i][particleBufIndex[i]].flag = true; + ++particleBufIndex[i]; + } + } +#endif + + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + + int i_new = new_x / dx, j_new = new_y / dy, k_new = new_z / dz; + + int iOffset = particletracer->xOffset / dx, + jOffset = particletracer->yOffset / dy, + kOffset = particletracer->zOffset / dz; + + if (S(i_new - iOffset, j_new - jOffset, k_new - kOffset) != FLUID) { + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + } + } + +#ifdef _MPI + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Send(&particleBufIndex[i], 1, MPI_INT, i, 0, particletracer->comm); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Recv(&recvparticleBufIndex[i], + 1, + MPI_INT, + i, + 0, + particletracer->comm, + MPI_STATUS_IGNORE); + + // if (0 !=recvparticleBufIndex[i]) { + // printf("Rank %d will receive %d particles from rank %d\n", + // particletracer->rank, + // recvparticleBufIndex[i], + // i); + // } + } + } + + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Send(buff[i], + particleBufIndex[i], + particletracer->mpi_particle, + i, + 0, + particletracer->comm); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + MPI_Recv(recvbuff[i], + recvparticleBufIndex[i], + particletracer->mpi_particle, + i, + 0, + particletracer->comm, + MPI_STATUS_IGNORE); + } + } + for (int i = 0; i < particletracer->size; ++i) { + if (i != particletracer->rank) { + for (int j = 0; j < recvparticleBufIndex[i]; ++j) { + particletracer->particlePool[particletracer->pointer].x = recvbuff[i][j] + .x; + particletracer->particlePool[particletracer->pointer].y = recvbuff[i][j] + .y; + particletracer->particlePool[particletracer->pointer].z = recvbuff[i][j] + .z; + particletracer->particlePool[particletracer->pointer].flag = true; + ++(particletracer->pointer); + ++(particletracer->totalParticles); + } + } + } +#endif +} + +void freeParticles(ParticleTracer* particletracer) +{ + free(particletracer->particlePool); + free(particletracer->linSpaceLine); + free(particletracer->offset); +} + +void writeParticles(ParticleTracer* particletracer) +{ + compress(particletracer); + +#ifdef _MPI + int collectedBuffIndex[particletracer->size]; + + MPI_Gather(&particletracer->totalParticles, + 1, + MPI_INT, + collectedBuffIndex, + 1, + MPI_INT, + 0, + particletracer->comm); + + if (particletracer->rank != 0) { + Particle buff[particletracer->totalParticles]; + for (int i = 0; i < particletracer->totalParticles; ++i) { + buff[i].x = particletracer->particlePool[i].x; + buff[i].y = particletracer->particlePool[i].y; + buff[i].z = particletracer->particlePool[i].z; + buff[i].flag = particletracer->particlePool[i].flag; + // printf("Rank : %d sending to rank 0 X : %.2f, Y : %.2f with totalpt : + // %d\n", particletracer->rank, buff[i].x, buff[i].y, + // particletracer->totalParticles); + } + MPI_Send(buff, + particletracer->totalParticles, + particletracer->mpi_particle, + 0, + 1, + particletracer->comm); + } +#endif + + if (particletracer->rank == 0) { + char filename[50]; + FILE* fp; + + snprintf(filename, 50, "vtk_files/particles_%d.vtk", ts); + fp = fopen(filename, "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + fprintf(fp, "# vtk DataFile Version 3.0\n"); + fprintf(fp, "PAMPI cfd solver particle tracing file\n"); + fprintf(fp, "ASCII\n"); + + fprintf(fp, "DATASET UNSTRUCTURED_GRID\n"); + fprintf(fp, "FIELD FieldData 2\n"); + fprintf(fp, "TIME 1 1 double\n"); + fprintf(fp, "%d\n", ts); + fprintf(fp, "CYCLE 1 1 int\n"); + fprintf(fp, "1\n"); + +#ifdef _MPI + int overallTotalParticles = sum(collectedBuffIndex, particletracer->size); + + fprintf(fp, "POINTS %d float\n", overallTotalParticles); + + // printf("Total particles : %d\n", overallTotalParticles); + + for (int i = 1; i < particletracer->size; ++i) { + Particle recvBuff[collectedBuffIndex[i]]; + MPI_Recv(&recvBuff, + collectedBuffIndex[i], + particletracer->mpi_particle, + i, + 1, + particletracer->comm, + MPI_STATUS_IGNORE); + + for (int j = 0; j < collectedBuffIndex[i]; ++j) { + double x = recvBuff[j].x; + double y = recvBuff[j].y; + double z = recvBuff[j].z; + fprintf(fp, "%f %f %f\n", x, y, z); + } + } +#else + int overallTotalParticles = particletracer->totalParticles; + fprintf(fp, "POINTS %d float\n", overallTotalParticles); + + // printf("Total particles : %d\n", overallTotalParticles); +#endif + for (int i = 0; i < particletracer->totalParticles; ++i) { + double x = particletracer->particlePool[i].x; + double y = particletracer->particlePool[i].y; + double z = particletracer->particlePool[i].z; + fprintf(fp, "%f %f %f\n", x, y, z); + } + + fprintf(fp, "CELLS %d %d\n", overallTotalParticles, 2 * overallTotalParticles); + + for (int i = 0; i < overallTotalParticles; ++i) { + fprintf(fp, "1 %d\n", i); + } + + fprintf(fp, "CELL_TYPES %d\n", overallTotalParticles); + + for (int i = 0; i < overallTotalParticles; ++i) { + fprintf(fp, "1\n"); + } + + fclose(fp); + } + + ++ts; +} + +void initParticleTracer( + ParticleTracer* particletracer, Parameter* params, Discretization* d) +{ + /* initializing local properties from params */ + + particletracer->numberOfParticles = params->numberOfParticles; + particletracer->startTime = params->startTime; + particletracer->injectTimePeriod = params->injectTimePeriod; + particletracer->writeTimePeriod = params->writeTimePeriod; + + particletracer->rank = d->comm.rank; + particletracer->size = d->comm.size; + + particletracer->dt = params->dt; + particletracer->dx = params->xlength / params->imax; + particletracer->dy = params->ylength / params->jmax; + particletracer->dz = params->zlength / params->kmax; + + particletracer->xlength = params->xlength; + particletracer->ylength = params->ylength; + particletracer->zlength = params->zlength; + + particletracer->x1 = params->x1; + particletracer->y1 = params->y1; + particletracer->z1 = params->z1; + particletracer->x2 = params->x2; + particletracer->y2 = params->y2; + particletracer->z2 = params->z2; + + particletracer->lastInjectTime = params->startTime; + particletracer->lastUpdateTime = params->startTime; + particletracer->lastWriteTime = params->startTime; + + particletracer->pointer = 0; + particletracer->totalParticles = 0; + particletracer->removedParticles = 0; + + particletracer->imax = params->imax; + particletracer->jmax = params->jmax; + particletracer->kmax = params->kmax; + + particletracer->imaxLocal = d->comm.imaxLocal; + particletracer->jmaxLocal = d->comm.jmaxLocal; + particletracer->kmaxLocal = d->comm.kmaxLocal; + + particletracer->estimatedNumParticles = (particletracer->imaxLocal * + particletracer->jmaxLocal * + particletracer->kmaxLocal); + + particletracer->particlePool = malloc( + sizeof(Particle) * particletracer->estimatedNumParticles); + + memset(particletracer->particlePool, + 0, + sizeof(Particle) * particletracer->estimatedNumParticles); + + particletracer->linSpaceLine = malloc( + sizeof(Particle) * particletracer->numberOfParticles); + + memset(particletracer->linSpaceLine, + 0, + sizeof(Particle) * particletracer->numberOfParticles); + + particletracer->offset = (double*)malloc(sizeof(double) * 6 * particletracer->size); + double offset[6][particletracer->size]; + +#ifdef _MPI + /* duplicating communication from solver */ + MPI_Comm_dup(d->comm.comm, &particletracer->comm); + + memcpy(particletracer->dims, d->comm.dims, sizeof(d->comm.dims)); + + memcpy(particletracer->coords, d->comm.coords, sizeof(d->comm.coords)); + +#endif + + particletracer->xOffset = d->xOffset; + particletracer->yOffset = d->yOffset; + particletracer->zOffset = d->zOffset; + particletracer->xOffsetEnd = d->xOffsetEnd; + particletracer->yOffsetEnd = d->yOffsetEnd; + particletracer->zOffsetEnd = d->zOffsetEnd; + +#ifdef _MPI + MPI_Allgather(&particletracer->xOffset, + 1, + MPI_DOUBLE, + offset[0], + 1, + MPI_DOUBLE, + particletracer->comm); + MPI_Allgather(&particletracer->yOffset, + 1, + MPI_DOUBLE, + offset[1], + 1, + MPI_DOUBLE, + particletracer->comm); + MPI_Allgather(&particletracer->zOffset, + 1, + MPI_DOUBLE, + offset[2], + 1, + MPI_DOUBLE, + particletracer->comm); + MPI_Allgather(&particletracer->xOffsetEnd, + 1, + MPI_DOUBLE, + offset[3], + 1, + MPI_DOUBLE, + particletracer->comm); + MPI_Allgather(&particletracer->yOffsetEnd, + 1, + MPI_DOUBLE, + offset[4], + 1, + MPI_DOUBLE, + particletracer->comm); + MPI_Allgather(&particletracer->zOffsetEnd, + 1, + MPI_DOUBLE, + offset[5], + 1, + MPI_DOUBLE, + particletracer->comm); + +#endif + memcpy(particletracer->offset, offset, sizeof(offset)); + + particleRandomizer(particletracer); + +#ifdef _MPI + // Create the mpi_particle datatype + MPI_Datatype mpi_particle; + int lengths[4] = { 1, 1, 1, 1 }; + + MPI_Aint displacements[4]; + Particle dummy_particle; + MPI_Aint base_address; + MPI_Get_address(&dummy_particle, &base_address); + MPI_Get_address(&dummy_particle.x, &displacements[0]); + MPI_Get_address(&dummy_particle.y, &displacements[1]); + MPI_Get_address(&dummy_particle.z, &displacements[2]); + MPI_Get_address(&dummy_particle.flag, &displacements[3]); + displacements[0] = MPI_Aint_diff(displacements[0], base_address); + displacements[1] = MPI_Aint_diff(displacements[1], base_address); + displacements[2] = MPI_Aint_diff(displacements[2], base_address); + displacements[3] = MPI_Aint_diff(displacements[3], base_address); + MPI_Datatype types[4] = { MPI_DOUBLE, MPI_DOUBLE, MPI_DOUBLE, MPI_C_BOOL }; + MPI_Type_create_struct(4, + lengths, + displacements, + types, + &particletracer->mpi_particle); + MPI_Type_commit(&particletracer->mpi_particle); +#endif +} + +void printParticleTracerParameters(ParticleTracer* particletracer) +{ + printf("Particle Tracing data:\n"); + printf("Rank : %d\n", particletracer->rank); + printf("\tNumber of particles : %d being injected for every period of %.2f\n", + particletracer->numberOfParticles, + particletracer->injectTimePeriod); + printf("\tstartTime : %.2f\n", particletracer->startTime); + printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : " + "%.2f, z1 : %.2f, x2 : %.2f, y2 : %.2f, z2 : %.2f\n", + particletracer->x1, + particletracer->y1, + particletracer->z1, + particletracer->x2, + particletracer->y2, + particletracer->z2); + printf("\tPointer : %d, TotalParticles : %d\n", + particletracer->pointer, + particletracer->totalParticles); + printf("\tdt : %.2f, dx : %.2f, dy : %.2f, dz : %.2f\n", + particletracer->dt, + particletracer->dx, + particletracer->dy, + particletracer->dz); + printf("\tcoord[0] : %d, coord[1] : %d, coord[2] : %d\n", + particletracer->coords[IDIM], + particletracer->coords[JDIM], + particletracer->coords[KDIM]); + printf("\txOffset : %.2f, yOffset : %.2f, zOffset : %.2f\n", + particletracer->xOffset, + particletracer->yOffset, + particletracer->zOffset); + printf("\txOffsetEnd : %.2f, yOffsetEnd : %.2f, zOffsetEnd : %.2f\n", + particletracer->xOffsetEnd, + particletracer->yOffsetEnd, + particletracer->zOffsetEnd); + printf("\txLocal : %.2f, yLocal : %.2f, zLocal : %.2f\n", + particletracer->xLocal, + particletracer->yLocal, + particletracer->zLocal); +} + +void trace(ParticleTracer* particletracer, + double* restrict u, + double* restrict v, + double* restrict w, + double* restrict s, + double time) +{ + if (time >= particletracer->startTime) { + if ((time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod) { + injectParticles(particletracer, s); + particletracer->lastInjectTime = time; + } + if ((time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod) { + writeParticles(particletracer); + particletracer->lastWriteTime = time; + } + advanceParticles(particletracer, u, v, w, s, time); + + if (particletracer->removedParticles > (particletracer->totalParticles * 0.2)) { + compress(particletracer); + } + particletracer->lastUpdateTime = time; + } +} + +void compress(ParticleTracer* particletracer) +{ + Particle* memPool = particletracer->particlePool; + Particle tempPool[particletracer->totalParticles]; + + memset(tempPool, 0, sizeof(Particle) * particletracer->totalParticles); + + int totalParticles = 0; + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (memPool[i].flag == true) { + tempPool[totalParticles].x = memPool[i].x; + tempPool[totalParticles].y = memPool[i].y; + tempPool[totalParticles].z = memPool[i].z; + tempPool[totalParticles].flag = true; + ++totalParticles; + } + } + + particletracer->totalParticles = totalParticles; + particletracer->removedParticles = 0; + particletracer->pointer = totalParticles; + + memcpy(particletracer->particlePool, tempPool, totalParticles * sizeof(Particle)); +} + +void particleRandomizer(ParticleTracer* particletracer) +{ + memset(particletracer->linSpaceLine, + 0, + sizeof(Particle) * particletracer->numberOfParticles); + + for (int i = 0; i < particletracer->numberOfParticles; ++i) { + + particletracer->linSpaceLine[i].x = (((double)rand() / RAND_MAX) * + (particletracer->x2 - + particletracer->x1)) + + particletracer->x1; + particletracer->linSpaceLine[i].y = (((double)rand() / RAND_MAX) * + (particletracer->y2 - + particletracer->y1)) + + particletracer->y1; + particletracer->linSpaceLine[i].z = (((double)rand() / RAND_MAX) * + (particletracer->z2 - + particletracer->z1)) + + particletracer->z1; + particletracer->linSpaceLine[i].flag = true; + } +} \ No newline at end of file diff --git a/EnhancedSolver/3D-mpi/src/particletracing.h b/EnhancedSolver/3D-mpi/src/particletracing.h new file mode 100644 index 0000000..61da8a5 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/particletracing.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARTICLETRACING_H_ +#define __PARTICLETRACING_H_ +#include "allocate.h" +#include "parameter.h" +#include "discretization.h" +#include +#include + +#define NDIMS 3 + +typedef enum COORD { X = 0, Y, NCOORD } COORD; +typedef struct { + double x, y, z; + bool flag; +} Particle; + +typedef struct { + int numberOfParticles, totalParticles; + double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime, + lastWriteTime; + + int estimatedNumParticles, removedParticles; + + double dx, dy, dz, dt; + Particle* linSpaceLine; + Particle* particlePool; + + int pointer; + + double imax, jmax, kmax, xlength, ylength, zlength, imaxLocal, jmaxLocal, kmaxLocal; + + double x1, y1, z1, x2, y2, z2; + +#ifdef _MPI + MPI_Comm comm; + MPI_Datatype mpi_particle; +#endif + + int rank, size; + int coords[NDIMS], dims[NDIMS]; + + double xLocal, yLocal, zLocal, xOffset, yOffset, zOffset, xOffsetEnd, yOffsetEnd, + zOffsetEnd; + + double* offset; + +} ParticleTracer; + +void initParticleTracer(ParticleTracer*, Parameter*, Discretization*); +void injectParticles(ParticleTracer*, double*); +void advanceParticles(ParticleTracer*, double*, double*, double*, double*, double); +void freeParticles(ParticleTracer*); +void writeParticles(ParticleTracer*); +void printParticleTracerParameters(ParticleTracer*); +void printParticles(ParticleTracer*); +void trace(ParticleTracer*, double*, double*, double*, double*, double); +void compress(ParticleTracer*); +extern void particleRandomizer(ParticleTracer*); +#endif \ No newline at end of file diff --git a/BasicSolver/3D-mpi-io/src/progress.c b/EnhancedSolver/3D-mpi/src/progress.c similarity index 74% rename from BasicSolver/3D-mpi-io/src/progress.c rename to EnhancedSolver/3D-mpi/src/progress.c index a9b82bd..8249ed1 100644 --- a/BasicSolver/3D-mpi-io/src/progress.c +++ b/EnhancedSolver/3D-mpi/src/progress.c @@ -1,16 +1,15 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ +#include "progress.h" #include #include #include #include -#include "progress.h" - static double _end; static int _current; @@ -49,3 +48,18 @@ void stopProgress() printf("\n"); fflush(stdout); } + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; +} + +void writeResidual(FILE* fp, double ts, double res) { fprintf(fp, "%f, %f\n", ts, res); } \ No newline at end of file diff --git a/BasicSolver/2D-mpi-v2/src/progress.h b/EnhancedSolver/3D-mpi/src/progress.h similarity index 56% rename from BasicSolver/2D-mpi-v2/src/progress.h rename to EnhancedSolver/3D-mpi/src/progress.h index 9ef2d96..d612ebe 100644 --- a/BasicSolver/2D-mpi-v2/src/progress.h +++ b/EnhancedSolver/3D-mpi/src/progress.h @@ -1,14 +1,17 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ + +#include + #ifndef __PROGRESS_H_ #define __PROGRESS_H_ - extern void initProgress(double); extern void printProgress(double); -extern void stopProgress(); - +extern void stopProgress(void); +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/EnhancedSolver/3D-mpi/src/solver-mg.c b/EnhancedSolver/3D-mpi/src/solver-mg.c new file mode 100644 index 0000000..7324735 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/solver-mg.c @@ -0,0 +1,389 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +static void restrictMG(Solver* s, int level, Comm* comm) +{ + double* r = s->r[level + 1]; + double* old = s->r[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + commExchange(comm, old); + + for (int k = 1; k < comm->kmaxLocal + 1; k++) { + for (int j = 1; j < comm->jmaxLocal + 1; j++) { + for (int i = 1; i < comm->imaxLocal + 1; ++i) { + R(i, j, k) = (OLD(2 * i - 1, 2 * j - 1, 2 * k) + + OLD(2 * i, 2 * j - 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k) + + OLD(2 * i - 1, 2 * j, 2 * k) * 2 + + OLD(2 * i, 2 * j, 2 * k) * 8 + + OLD(2 * i + 1, 2 * j, 2 * k) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k) + + OLD(2 * i, 2 * j + 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i, 2 * j - 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i - 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i, 2 * j, 2 * k - 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k - 1) + + OLD(2 * i, 2 * j + 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k - 1) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i, 2 * j - 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i - 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i, 2 * j, 2 * k + 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k + 1) + + OLD(2 * i, 2 * j + 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k + 1)) / + 64.0; + } + } + } +} + +static void prolongate(Solver* s, int level, Comm* comm) +{ + double* old = s->r[level + 1]; + double* e = s->r[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + for (int k = 2; k < kmaxLocal + 1; k += 2) { + for (int j = 2; j < jmaxLocal + 1; j += 2) { + for (int i = 2; i < imaxLocal + 1; i += 2) { + E(i, j, k) = OLD(i / 2, j / 2, k / 2); + } + } + } +} + +static void correct(Solver* s, double* p, int level, Comm* comm) +{ + double* e = s->e[level]; + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + for (int k = 1; k < kmaxLocal + 1; ++k) { + for (int j = 1; j < jmaxLocal + 1; ++j) { + for (int i = 1; i < imaxLocal + 1; ++i) { + P(i, j, k) += E(i, j, k); + } + } + } +} + +static void setBoundaryCondition( + Solver* s, double* p, int imaxLocal, int jmaxLocal, int kmaxLocal) +{ +#ifdef _MPI + if (commIsBoundary(s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + } + } + } + + if (commIsBoundary(s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + } + + if (commIsBoundary(s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + } + } + } + + if (commIsBoundary(s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + } + + if (commIsBoundary(s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + } + } + } + + if (commIsBoundary(s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } + } +#else + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } +#endif +} + +static double smooth(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + commExchange(comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + P(i, j, k) -= + factor * + (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +} + +static double calculateResidual(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + commExchange(comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + R(i, + j, + k) = (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * + idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + + res += (R(i, j, k) * R(i, j, k)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + commReduction(&res, SUM); + + res = res / (double)(imaxLocal * jmaxLocal * kmaxLocal); +#ifdef DEBUG + if (commIsMaster(s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + return res; +} + +static double multiGrid(Solver* s, double* p, double* rhs, int level, Comm* comm) +{ + int imaxLocal = comm->imaxLocal; + int jmaxLocal = comm->jmaxLocal; + int kmaxLocal = comm->kmaxLocal; + + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, comm); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + } + + res = calculateResidual(s, p, rhs, level, comm); + + // restrict + restrictMG(s, level, comm); + + Comm newcomm; + commUpdateDatatypes(s->comm, &newcomm, imaxLocal, jmaxLocal, kmaxLocal); + + // MGSolver on residual and error. + multiGrid(s, s->e[level + 1], s->r[level + 1], level + 1, &newcomm); + + commFreeCommunicator(&newcomm); + + // prolongate + prolongate(s, level, comm); + + // correct p on finer level using residual + correct(s, p, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, comm); + if (level == FINEST_LEVEL) + setBoundaryCondition(s, p, imaxLocal, jmaxLocal, kmaxLocal); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + s->comm = &d->comm; + s->problem = p->name; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + int levels = s->levels; + if (commIsMaster(s->comm)) printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * (kmax + 2); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size * sizeof(double)); + s->e[j] = allocate(64, size * sizeof(double)); + + for (size_t i = 0; i < size; i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->comm); +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Residuum: %.6f\n", res); + } +#endif + + return res; +} diff --git a/EnhancedSolver/3D-mpi/src/solver-rb.c b/EnhancedSolver/3D-mpi/src/solver-rb.c new file mode 100644 index 0000000..d64fecd --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/solver-rb.c @@ -0,0 +1,175 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "parameter.h" +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + s->comm = &d->comm; + s->problem = p->name; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double epssq = eps * eps; + int it = 0; + double res = 1.0; + int pass, ksw, jsw, isw; + + while ((res >= epssq) && (it < itermax)) { + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + commExchange(s->comm, p); + + for (int k = 1; k < kmaxLocal + 1; k++) { + isw = jsw; + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imaxLocal + 1; i += 2) { + + double r = + RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2); + + P(i, j, k) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +#ifdef _MPI + if (commIsBoundary(s->comm, FRONT)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, 0) = P(i, j, 1); + } + } + } + + if (commIsBoundary(s->comm, BACK)) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, j, kmaxLocal + 1) = P(i, j, kmaxLocal); + } + } + } + + if (commIsBoundary(s->comm, BOTTOM)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, 0, k) = P(i, 1, k); + } + } + } + + if (commIsBoundary(s->comm, TOP)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int i = 1; i < imaxLocal + 1; i++) { + P(i, jmaxLocal + 1, k) = P(i, jmaxLocal, k); + } + } + } + + if (commIsBoundary(s->comm, LEFT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j, k) = P(1, j, k); + } + } + } + + if (commIsBoundary(s->comm, RIGHT)) { + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + P(imaxLocal + 1, j, k) = P(imaxLocal, j, k); + } + } + } +#else + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } +#endif + commReduction(&res, SUM); + res = res / (double)(imax * jmax * kmax); +#ifdef DEBUG + if (commIsMaster(&s->comm)) { + printf("%d Residuum: %e\n", it, res); + } +#endif + commExchange(s->comm, p); + it++; + } + +#ifdef VERBOSE + if (commIsMaster(s->comm)) { + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + } +#endif + +return res; +} \ No newline at end of file diff --git a/BasicSolver/3D-mpi-io/src/solver.h b/EnhancedSolver/3D-mpi/src/solver.h similarity index 62% rename from BasicSolver/3D-mpi-io/src/solver.h rename to EnhancedSolver/3D-mpi/src/solver.h index ac9f450..d9a0c39 100644 --- a/BasicSolver/3D-mpi-io/src/solver.h +++ b/EnhancedSolver/3D-mpi/src/solver.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -9,12 +9,11 @@ #include "comm.h" #include "grid.h" #include "parameter.h" - -enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; +#include "discretization.h" typedef struct { /* geometry and grid information */ - Grid grid; + Grid* grid; /* arrays */ double *p, *rhs; double *f, *g, *h; @@ -30,16 +29,11 @@ typedef struct { char* problem; int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; /* communication */ - Comm comm; + double **r, **e; + int levels, presmooth, postsmooth; + Comm* comm; } Solver; -void initSolver(Solver*, Parameter*); -void computeRHS(Solver*); -void solve(Solver*); -void normalizePressure(Solver*); -void computeTimestep(Solver*); -void setBoundaryConditions(Solver*); -void setSpecialBoundaryCondition(Solver*); -void computeFG(Solver*); -void adaptUV(Solver*); +extern double solve(Solver* , double* , double* ); +extern void initSolver(Solver*, Discretization*, Parameter*); #endif diff --git a/BasicSolver/3D-mpi-io/src/test.c b/EnhancedSolver/3D-mpi/src/test.c similarity index 65% rename from BasicSolver/3D-mpi-io/src/test.c rename to EnhancedSolver/3D-mpi/src/test.c index 5f61b9f..bf2ad16 100644 --- a/BasicSolver/3D-mpi-io/src/test.c +++ b/EnhancedSolver/3D-mpi/src/test.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -9,15 +9,15 @@ #include "test.h" -#define G(v, i, j, k) \ +#define X(v, i, j, k) \ v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] void testInit(Solver* s) { - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; - int myrank = s->comm.rank; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; + int myrank = s->comm->rank; double* p = s->p; double* f = s->f; double* g = s->g; @@ -26,10 +26,10 @@ void testInit(Solver* s) for (int k = 0; k < kmaxLocal + 2; k++) { for (int j = 0; j < jmaxLocal + 2; j++) { for (int i = 0; i < imaxLocal + 2; i++) { - G(p, i, j, k) = 10.0; - G(f, i, j, k) = myrank + 1.0; - G(g, i, j, k) = myrank + 1.0; - G(h, i, j, k) = myrank + 1.0; + X(p, i, j, k) = 10.0; + X(f, i, j, k) = myrank + 1.0; + X(g, i, j, k) = myrank + 1.0; + X(h, i, j, k) = myrank + 1.0; } } } @@ -37,10 +37,10 @@ void testInit(Solver* s) for (int k = 1; k < kmaxLocal + 1; k++) { for (int j = 1; j < jmaxLocal + 1; j++) { for (int i = 1; i < imaxLocal + 1; i++) { - G(p, i, j, k) = myrank + 1.0; - G(f, i, j, k) = myrank + 1.0; - G(g, i, j, k) = myrank + 1.0; - G(h, i, j, k) = myrank + 1.0; + X(p, i, j, k) = myrank + 1.0; + X(f, i, j, k) = myrank + 1.0; + X(g, i, j, k) = myrank + 1.0; + X(h, i, j, k) = myrank + 1.0; } } } @@ -76,33 +76,33 @@ static char* direction2String(Direction dir) static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir) { - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; char filename[50]; - snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm.rank); + snprintf(filename, 50, "halo-%s-r%d.txt", direction2String(dir), s->comm->rank); FILE* fh = fopen(filename, "w"); for (int y = 0; y < ymax; y++) { for (int x = 0; x < xmax; x++) { switch (dir) { case LEFT: - fprintf(fh, "%12.8f ", G(a, 0, x, y)); + fprintf(fh, "%12.8f ", X(a, 0, x, y)); break; case RIGHT: - fprintf(fh, "%12.8f ", G(a, imaxLocal + 1, x, y)); + fprintf(fh, "%12.8f ", X(a, imaxLocal + 1, x, y)); break; case BOTTOM: - fprintf(fh, "%12.8f ", G(a, x, 0, y)); + fprintf(fh, "%12.8f ", X(a, x, 0, y)); break; case TOP: - fprintf(fh, "%12.8f ", G(a, x, jmaxLocal + 1, y)); + fprintf(fh, "%12.8f ", X(a, x, jmaxLocal + 1, y)); break; case FRONT: - fprintf(fh, "%12.8f ", G(a, x, y, 0)); + fprintf(fh, "%12.8f ", X(a, x, y, 0)); break; case BACK: - fprintf(fh, "%12.8f ", G(a, x, y, kmaxLocal + 1)); + fprintf(fh, "%12.8f ", X(a, x, y, kmaxLocal + 1)); break; case NDIRS: printf("ERROR\n"); @@ -116,9 +116,9 @@ static void printPlane(Solver* s, double* a, int ymax, int xmax, Direction dir) void testPrintHalo(Solver* s, double* a) { - int imaxLocal = s->comm.imaxLocal; - int jmaxLocal = s->comm.jmaxLocal; - int kmaxLocal = s->comm.kmaxLocal; + int imaxLocal = s->comm->imaxLocal; + int jmaxLocal = s->comm->jmaxLocal; + int kmaxLocal = s->comm->kmaxLocal; printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, BOTTOM); printPlane(s, a, kmaxLocal + 2, imaxLocal + 2, TOP); diff --git a/BasicSolver/3D-mpi-io/src/test.h b/EnhancedSolver/3D-mpi/src/test.h similarity index 84% rename from BasicSolver/3D-mpi-io/src/test.h rename to EnhancedSolver/3D-mpi/src/test.h index ecefe4e..779b509 100644 --- a/BasicSolver/3D-mpi-io/src/test.h +++ b/EnhancedSolver/3D-mpi/src/test.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. diff --git a/BasicSolver/2D-mpi-v1/src/timing.c b/EnhancedSolver/3D-mpi/src/timing.c similarity index 72% rename from BasicSolver/2D-mpi-v1/src/timing.c rename to EnhancedSolver/3D-mpi/src/timing.c index c4025a4..78b01c4 100644 --- a/BasicSolver/2D-mpi-v1/src/timing.c +++ b/EnhancedSolver/3D-mpi/src/timing.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,18 +7,16 @@ #include #include -double getTimeStamp() +double getTimeStamp(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } -double getTimeResolution() +double getTimeResolution(void) { struct timespec ts; clock_getres(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } - -double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-mpi-v3/src/timing.h b/EnhancedSolver/3D-mpi/src/timing.h similarity index 55% rename from BasicSolver/2D-mpi-v3/src/timing.h rename to EnhancedSolver/3D-mpi/src/timing.h index db95329..ed05a8c 100644 --- a/BasicSolver/2D-mpi-v3/src/timing.h +++ b/EnhancedSolver/3D-mpi/src/timing.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,8 +7,7 @@ #ifndef __TIMING_H_ #define __TIMING_H_ -extern double getTimeStamp(); -extern double getTimeResolution(); -extern double getTimeStamp_(); +extern double getTimeStamp(void); +extern double getTimeResolution(void); #endif // __TIMING_H_ diff --git a/EnhancedSolver/3D-mpi/src/util.h b/EnhancedSolver/3D-mpi/src/util.h new file mode 100644 index 0000000..735209f --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/util.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#define P(i, j, k) p[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define F(i, j, k) f[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define G(i, j, k) g[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define H(i, j, k) h[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define U(i, j, k) u[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define V(i, j, k) v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define W(i, j, k) w[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define S(i, j, k) s[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define E(i, j, k) e[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define R(i, j, k) r[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define OLD(i, j, k) old[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] +#define RHS(i, j, k) rhs[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +#endif // __UTIL_H_ diff --git a/EnhancedSolver/3D-mpi/src/vtkWriter-mpi.c b/EnhancedSolver/3D-mpi/src/vtkWriter-mpi.c new file mode 100644 index 0000000..d9670d5 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/vtkWriter-mpi.c @@ -0,0 +1,227 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "allocate.h" +#include "comm.h" +#include "vtkWriter.h" + +// reset fileview for output of string headers +static void resetFileview(VtkOptions* o) +{ + MPI_Offset disp; + MPI_File_sync(o->fh); + MPI_Barrier(o->comm.comm); + MPI_File_get_size(o->fh, &disp); + MPI_File_set_view(o->fh, disp, MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL); +} + +static void writeVersion(VtkOptions* o) +{ + char header[50] = "# vtk DataFile Version 3.0\n"; + // always overwrite exiting files + MPI_File_set_view(o->fh, 0, MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL); + + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + } +} + +static void writeHeader(VtkOptions* o) +{ + char header[400]; + char* cursor = header; + + cursor += sprintf(cursor, "PAMPI cfd solver output\n"); + cursor += sprintf(cursor, "BINARY\n"); + + cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n"); + cursor += sprintf(cursor, + "DIMENSIONS %d %d %d\n", + o->grid->imax, + o->grid->jmax, + o->grid->kmax); + cursor += sprintf(cursor, + "ORIGIN %f %f %f\n", + o->grid->dx * 0.5, + o->grid->dy * 0.5, + o->grid->dz * 0.5); + cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz); + cursor += sprintf(cursor, + "POINT_DATA %d\n", + o->grid->imax * o->grid->jmax * o->grid->kmax); + + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + } +} + +void vtkOpen(VtkOptions* o, char* problem) +{ + char filename[50]; + snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size); + MPI_File_open(o->comm.comm, + filename, + MPI_MODE_WRONLY | MPI_MODE_CREATE, + MPI_INFO_NULL, + &o->fh); + + if (commIsMaster(&o->comm)) { + printf("Writing VTK output for %s\n", problem); + } + + writeVersion(o); + writeHeader(o); +} + +void vtkScalar(VtkOptions* o, char* name, double* s) +{ + resetFileview(o); + if (commIsMaster(&o->comm)) printf("Register scalar %s\n", name); + + char header[100]; + char* cursor = header; + + cursor += sprintf(cursor, "SCALARS %s double\n", name); + + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + } + + int offsets[NDIMS]; + commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax); + + // set global view in file + MPI_Offset disp; + MPI_Datatype fileViewType; + MPI_File_sync(o->fh); + MPI_Barrier(o->comm.comm); + MPI_File_get_size(o->fh, &disp); + + MPI_Type_create_subarray(NDIMS, + (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax }, + (int[NDIMS]) { o->comm.kmaxLocal, o->comm.jmaxLocal, o->comm.imaxLocal }, + offsets, + MPI_ORDER_C, + MPI_DOUBLE, + &fileViewType); + MPI_Type_commit(&fileViewType); + MPI_File_set_view(o->fh, disp, MPI_DOUBLE, fileViewType, "external32", MPI_INFO_NULL); + +#ifdef VERBOSE + printf("Rank: %d, Disp: %lld, Size(k,j,i): %d %d %d, Offset(k,j,i): %d %d %d\n", + o->comm.rank, + disp, + o->comm.kmaxLocal, + o->comm.jmaxLocal, + o->comm.imaxLocal, + offsets[KDIM], + offsets[JDIM], + offsets[IDIM]); +#endif + + // create local bulk type + MPI_Datatype bulkType; + + MPI_Type_create_subarray(NDIMS, + (int[NDIMS]) { o->comm.kmaxLocal + 2, + o->comm.jmaxLocal + 2, + o->comm.imaxLocal + 2 }, // oldsizes + (int[NDIMS]) { o->comm.kmaxLocal, + o->comm.jmaxLocal, + o->comm.imaxLocal }, // newsizes + (int[NDIMS]) { 1, 1, 1 }, // offsets + MPI_ORDER_C, + MPI_DOUBLE, + &bulkType); + MPI_Type_commit(&bulkType); + + MPI_File_write(o->fh, s, 1, bulkType, MPI_STATUS_IGNORE); + MPI_Type_free(&bulkType); + MPI_Type_free(&fileViewType); + + // Binary segment must be terminated with newline character + resetFileview(o); + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, "\n", 1, MPI_CHAR, MPI_STATUS_IGNORE); + } +} + +#define G(v, i, j, k) \ + v[(k) * (imaxLocal + 2) * (jmaxLocal + 2) + (j) * (imaxLocal + 2) + (i)] + +void vtkVector(VtkOptions* o, char* name, VtkVector vec) +{ + int imaxLocal = o->comm.imaxLocal; + int jmaxLocal = o->comm.jmaxLocal; + int kmaxLocal = o->comm.kmaxLocal; + + if (commIsMaster(&o->comm)) printf("Register vector %s\n", name); + const size_t MAX_HEADER = 100; + + char* header = (char*)malloc(MAX_HEADER); + sprintf(header, "VECTORS %s double\n", name); + + resetFileview(o); + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, header, (int)strlen(header), MPI_CHAR, MPI_STATUS_IGNORE); + } + + int offsets[NDIMS]; + commGetOffsets(&o->comm, offsets, o->grid->kmax, o->grid->jmax, o->grid->imax); + + // set global view in file + MPI_Offset disp; + MPI_Datatype fileViewType, vectorType; + MPI_File_sync(o->fh); + MPI_Barrier(o->comm.comm); + MPI_File_get_size(o->fh, &disp); + + MPI_Type_contiguous(NDIMS, MPI_DOUBLE, &vectorType); + MPI_Type_commit(&vectorType); + + MPI_Type_create_subarray(NDIMS, + (int[NDIMS]) { o->grid->kmax, o->grid->jmax, o->grid->imax }, + (int[NDIMS]) { kmaxLocal, jmaxLocal, imaxLocal }, + offsets, + MPI_ORDER_C, + vectorType, + &fileViewType); + MPI_Type_commit(&fileViewType); + MPI_File_set_view(o->fh, disp, MPI_DOUBLE, fileViewType, "external32", MPI_INFO_NULL); + + size_t cnt = imaxLocal * jmaxLocal * kmaxLocal; + double* tmp = allocate(64, cnt * NDIMS * sizeof(double)); + int idx = 0; + + for (int k = 1; k < kmaxLocal + 1; k++) { + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = 1; i < imaxLocal + 1; i++) { + tmp[idx++] = (G(vec.u, i, j, k) + G(vec.u, i - 1, j, k)) / 2.0; + tmp[idx++] = (G(vec.v, i, j, k) + G(vec.v, i, j - 1, k)) / 2.0; + tmp[idx++] = (G(vec.w, i, j, k) + G(vec.w, i, j, k - 1)) / 2.0; + } + } + } + + if (commIsMaster(&o->comm)) printf("Write %d vectors\n", (int)cnt); + MPI_File_write(o->fh, tmp, cnt, vectorType, MPI_STATUS_IGNORE); + MPI_Type_free(&fileViewType); + MPI_Type_free(&vectorType); + + // Binary segment must be terminated with newline character + resetFileview(o); + if (commIsMaster(&o->comm)) { + MPI_File_write(o->fh, "\n", 1, MPI_CHAR, MPI_STATUS_IGNORE); + } +} + +void vtkClose(VtkOptions* o) { MPI_File_close(&o->fh); } diff --git a/EnhancedSolver/3D-mpi/src/vtkWriter-seq.c b/EnhancedSolver/3D-mpi/src/vtkWriter-seq.c new file mode 100644 index 0000000..346e367 --- /dev/null +++ b/EnhancedSolver/3D-mpi/src/vtkWriter-seq.c @@ -0,0 +1,147 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "vtkWriter.h" +#define Z(v, i, j, k) v[(k)*imax * jmax + (j)*imax + (i)] + +static double floatSwap(double f) +{ + union { + double f; + char b[8]; + } dat1, dat2; + + dat1.f = f; + dat2.b[0] = dat1.b[7]; + dat2.b[1] = dat1.b[6]; + dat2.b[2] = dat1.b[5]; + dat2.b[3] = dat1.b[4]; + dat2.b[4] = dat1.b[3]; + dat2.b[5] = dat1.b[2]; + dat2.b[6] = dat1.b[1]; + dat2.b[7] = dat1.b[0]; + return dat2.f; +} + +static void writeHeader(VtkOptions* o) +{ + fprintf(o->fh, "# vtk DataFile Version 3.0\n"); + fprintf(o->fh, "PAMPI cfd solver output\n"); + if (o->fmt == ASCII) { + fprintf(o->fh, "ASCII\n"); + } else if (o->fmt == BINARY) { + fprintf(o->fh, "BINARY\n"); + } + + fprintf(o->fh, "DATASET STRUCTURED_POINTS\n"); + fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid->imax, o->grid->jmax, o->grid->kmax); + fprintf(o->fh, + "ORIGIN %f %f %f\n", + o->grid->dx * 0.5, + o->grid->dy * 0.5, + o->grid->dz * 0.5); + fprintf(o->fh, "SPACING %f %f %f\n", o->grid->dx, o->grid->dy, o->grid->dz); + fprintf(o->fh, "POINT_DATA %d\n", o->grid->imax * o->grid->jmax * o->grid->kmax); +} + +void vtkOpen(VtkOptions* o, char* problem) +{ + char filename[50]; + + snprintf(filename, 50, "%s.vtk", problem); + o->fh = fopen(filename, "w"); + writeHeader(o); + + printf("Writing VTK output for %s\n", problem); +} + +static void writeScalar(VtkOptions* o, double* s) +{ + int imax = o->grid->imax; + int jmax = o->grid->jmax; + int kmax = o->grid->kmax; + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, "%f\n", Z(s, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((double[1]) { floatSwap(Z(s, i, j, k)) }, + sizeof(double), + 1, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +static bool isInitialized(FILE* ptr) +{ + if (ptr == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + return false; + } + return true; +} + +void vtkScalar(VtkOptions* o, char* name, double* s) +{ + printf("Register scalar %s\n", name); + if (!isInitialized(o->fh)) return; + fprintf(o->fh, "SCALARS %s double 1\n", name); + fprintf(o->fh, "LOOKUP_TABLE default\n"); + writeScalar(o, s); +} + +static void writeVector(VtkOptions* o, VtkVector vec) +{ + int imax = o->grid->imax; + int jmax = o->grid->jmax; + int kmax = o->grid->kmax; + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, + "%f %f %f\n", + Z(vec.u, i, j, k), + Z(vec.v, i, j, k), + Z(vec.w, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((double[3]) { floatSwap(Z(vec.u, i, j, k)), + floatSwap(Z(vec.v, i, j, k)), + floatSwap(Z(vec.w, i, j, k)) }, + sizeof(double), + 3, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +void vtkVector(VtkOptions* o, char* name, VtkVector vec) +{ + printf("Register vector %s\n", name); + if (!isInitialized(o->fh)) return; + fprintf(o->fh, "VECTORS %s double\n", name); + writeVector(o, vec); +} + +void vtkClose(VtkOptions* o) +{ + fclose(o->fh); + o->fh = NULL; +} diff --git a/BasicSolver/3D-mpi-io/src/vtkWriter.h b/EnhancedSolver/3D-mpi/src/vtkWriter.h similarity index 75% rename from BasicSolver/3D-mpi-io/src/vtkWriter.h rename to EnhancedSolver/3D-mpi/src/vtkWriter.h index 7756826..e93835c 100644 --- a/BasicSolver/3D-mpi-io/src/vtkWriter.h +++ b/EnhancedSolver/3D-mpi/src/vtkWriter.h @@ -1,20 +1,26 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ #ifndef __VTKWRITER_H_ #define __VTKWRITER_H_ -#include #include #include "comm.h" #include "grid.h" +typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat; + typedef struct VtkOptions { - Grid grid; + Grid* grid; +#ifdef _VTK_WRITER_MPI MPI_File fh; +#else + FILE* fh; + VtkFormat fmt; +#endif // _VTK_WRITER_MPI Comm comm; } VtkOptions; diff --git a/BasicSolver/2D-mpi-v1/Makefile b/EnhancedSolver/3D-seq/Makefile similarity index 79% rename from BasicSolver/2D-mpi-v1/Makefile rename to EnhancedSolver/3D-seq/Makefile index 57f99f4..6d1c49d 100644 --- a/BasicSolver/2D-mpi-v1/Makefile +++ b/EnhancedSolver/3D-seq/Makefile @@ -1,5 +1,5 @@ #======================================================================================= -# Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +# Copyright (C) NHR@FAU, University Erlangen-Nuremberg. # All rights reserved. # Use of this source code is governed by a MIT-style # license that can be found in the LICENSE file. @@ -18,9 +18,10 @@ include $(MAKE_DIR)/include_$(TAG).mk INCLUDES += -I$(SRC_DIR) -I$(BUILD_DIR) VPATH = $(SRC_DIR) -SRC = $(wildcard $(SRC_DIR)/*.c) +SRC = $(filter-out $(wildcard $(SRC_DIR)/*-*.c),$(wildcard $(SRC_DIR)/*.c)) ASM = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s, $(SRC)) OBJ = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) +OBJ += $(BUILD_DIR)/solver-$(SOLVER).o SOURCES = $(SRC) $(wildcard $(SRC_DIR)/*.h) CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(OPTIONS) $(INCLUDES) @@ -39,6 +40,20 @@ $(BUILD_DIR)/%.s: %.c .PHONY: clean distclean tags info asm format +vis: + $(info ===> GENERATE VISUALIZATION) + @gnuplot -e "filename='residual.dat'" ./residual.plot + +vis_clean: + $(info ===> CLEAN VISUALIZATION) + @rm -f *.dat + @rm -f vtk_files/*.vtk + @rm -f *.png + @rm -f *.vtk + + +clean: vis_clean + clean: $(info ===> CLEAN) @rm -rf $(BUILD_DIR) @@ -47,6 +62,8 @@ clean: distclean: clean $(info ===> DIST CLEAN) @rm -f $(TARGET) + @rm -f *.dat + @rm -f *.png info: $(info $(CFLAGS)) diff --git a/BasicSolver/2D-mpi-v3/README.md b/EnhancedSolver/3D-seq/README.md similarity index 51% rename from BasicSolver/2D-mpi-v3/README.md rename to EnhancedSolver/3D-seq/README.md index b0a80a6..d980b54 100644 --- a/BasicSolver/2D-mpi-v3/README.md +++ b/EnhancedSolver/3D-seq/README.md @@ -9,13 +9,25 @@ TAG ?= GCC ENABLE_OPENMP ?= false OPTIONS += -DARRAY_ALIGNMENT=64 +#OPTIONS += -DVERBOSE #OPTIONS += -DVERBOSE_AFFINITY #OPTIONS += -DVERBOSE_DATASIZE #OPTIONS += -DVERBOSE_TIMER ``` -The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution. +The verbosity options enable detailed output about solver, affinity settings, allocation sizes and timer resolution. +For debugging you may want to set the VERBOSE option: +``` +# Supported: GCC, CLANG, ICC +TAG ?= GCC +ENABLE_OPENMP ?= false +OPTIONS += -DARRAY_ALIGNMENT=64 +OPTIONS += -DVERBOSE +#OPTIONS += -DVERBOSE_AFFINITY +#OPTIONS += -DVERBOSE_DATASIZE +#OPTIONS += -DVERBOSE_TIMER +` 2. Build with: ``` @@ -46,3 +58,21 @@ to clean intermediate build results and binary. make asm ``` The assembler files will also be located in the `` directory. + +## Usage + +You have to provide a parameter file describing the problem you want to solve: +``` +./exe-CLANG dcavity.par +``` + +Examples are given in in dcavity (a lid driven cavity test case) and canal (simulating a empty canal). + +You can plot the resulting velocity and pressure fields using gnuplot: +``` +gnuplot vector.plot +``` +and for the pressure: +``` +gnuplot surface.plot +``` diff --git a/EnhancedSolver/3D-seq/backstep.par b/EnhancedSolver/3D-seq/backstep.par new file mode 100644 index 0000000..8075448 --- /dev/null +++ b/EnhancedSolver/3D-seq/backstep.par @@ -0,0 +1,89 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name backstep # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 36000.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 1.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 7.0 # domain size in x-direction +ylength 1.5 # domain size in y-direction +zlength 1.0 # domain size in z-direction +imax 210 # number of interior cells in x-direction +jmax 45 # number of interior cells in y-direction +kmax 30 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 40.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 2000 +startTime 0 +injectTimePeriod 1.0 +writeTimePeriod 0.2 + +x1 0.0 +y1 0.5 +z1 0.0 +x2 0.0 +y2 1.5 +z2 1.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 1 +xCenter 0.0 +yCenter 0.0 +zCenter 0.0 +xRectLength 2.0 +yRectLength 1.0 +zRectLength 2.0 +circleRadius 1.0 + +#=============================================================================== diff --git a/EnhancedSolver/3D-seq/canal.par b/EnhancedSolver/3D-seq/canal.par new file mode 100644 index 0000000..3ca4a9e --- /dev/null +++ b/EnhancedSolver/3D-seq/canal.par @@ -0,0 +1,88 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name canal # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 100.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 4.0 # domain size in y-direction +zlength 4.0 # domain size in z-direction +imax 200 # number of interior cells in x-direction +jmax 40 # number of interior cells in y-direction +kmax 40 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 60.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 500 # maximal number of pressure iteration in one time step +eps 0.0001 # stopping tolerance for pressure iteration +omg 0.52 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 5 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 10 +injectTimePeriod 2.0 +writeTimePeriod 1.0 + +x1 0.0 +y1 0.0 +z1 0.0 +x2 4.0 +y2 4.0 +z2 4.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 10.0 +yCenter 2.0 +zCenter 2.0 +xRectLength 8.0 +yRectLength 2.0 +zRectLength 2.0 +circleRadius 1.0 +#=============================================================================== diff --git a/BasicSolver/2D-seq-pt/config.mk b/EnhancedSolver/3D-seq/config.mk similarity index 52% rename from BasicSolver/2D-seq-pt/config.mk rename to EnhancedSolver/3D-seq/config.mk index af5f1a0..2cf33f9 100644 --- a/BasicSolver/2D-seq-pt/config.mk +++ b/EnhancedSolver/3D-seq/config.mk @@ -1,12 +1,12 @@ # Supported: GCC, CLANG, ICC -TAG ?= CLANG +TAG ?= ICC ENABLE_OPENMP ?= false +# Supported: rb, mg +SOLVER ?= mg +# Run in debug settings +DEBUG ?= false #Feature options OPTIONS += -DARRAY_ALIGNMENT=64 OPTIONS += -DVERBOSE #OPTIONS += -DDEBUG -#OPTIONS += -DBOUNDCHECK -#OPTIONS += -DVERBOSE_AFFINITY -#OPTIONS += -DVERBOSE_DATASIZE -#OPTIONS += -DVERBOSE_TIMER diff --git a/EnhancedSolver/3D-seq/dcavity.par b/EnhancedSolver/3D-seq/dcavity.par new file mode 100644 index 0000000..204e007 --- /dev/null +++ b/EnhancedSolver/3D-seq/dcavity.par @@ -0,0 +1,88 @@ +#============================================================================== +# Driven Cavity +#============================================================================== + +# Problem specific Data: +# --------------------- + +name dcavity # name of flow setup + +bcLeft 1 # flags for boundary conditions +bcRight 1 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 1000.0 # Reynolds number + +u_init 0.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 1.0 # domain size in x-direction +ylength 1.0 # domain size in y-direction +zlength 1.0 # domain size in z-direction +imax 80 # number of interior cells in x-direction +jmax 80 # number of interior cells in y-direction +kmax 80 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 10.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 1000 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.5 +omg 1.7 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 2 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 200 +startTime 0 +injectTimePeriod 1.0 +writeTimePeriod 0.1 + +x1 0.0 +y1 0.1 +z1 0.0 +x2 1.0 +y2 0.1 +z2 1.0 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 0 +xCenter 0.5 +yCenter 0.5 +zCenter 0.5 +xRectLength 0.25 +yRectLength 0.25 +zRectLength 0.25 +circleRadius 0.25 +#=============================================================================== diff --git a/EnhancedSolver/3D-seq/include_CLANG.mk b/EnhancedSolver/3D-seq/include_CLANG.mk new file mode 100644 index 0000000..a124053 --- /dev/null +++ b/EnhancedSolver/3D-seq/include_CLANG.mk @@ -0,0 +1,19 @@ +CC = clang +GCC = cc +LINKER = $(CC) + +ifeq ($(strip $(ENABLE_OPENMP)),true) +OPENMP = -fopenmp +#OPENMP = -Xpreprocessor -fopenmp #required on Macos with homebrew libomp +LIBS = # -lomp +endif +ifeq ($(strip $(DEBUG)),true) +CFLAGS = -O0 -g -std=c17 +else +CFLAGS = -O3 -std=c17 $(OPENMP) +endif + +VERSION = --version +LFLAGS = $(OPENMP) -lm +DEFINES = -D_GNU_SOURCE +INCLUDES = diff --git a/BasicSolver/2D-mpi-v2/include_GCC.mk b/EnhancedSolver/3D-seq/include_GCC.mk similarity index 100% rename from BasicSolver/2D-mpi-v2/include_GCC.mk rename to EnhancedSolver/3D-seq/include_GCC.mk diff --git a/BasicSolver/2D-seq-pt/include_ICC.mk b/EnhancedSolver/3D-seq/include_ICC.mk similarity index 100% rename from BasicSolver/2D-seq-pt/include_ICC.mk rename to EnhancedSolver/3D-seq/include_ICC.mk diff --git a/EnhancedSolver/3D-seq/karman.par b/EnhancedSolver/3D-seq/karman.par new file mode 100644 index 0000000..a3eda7f --- /dev/null +++ b/EnhancedSolver/3D-seq/karman.par @@ -0,0 +1,89 @@ +#============================================================================== +# Laminar Canal Flow +#============================================================================== + +# Problem specific Data: +# --------------------- + +name karman # name of flow setup + +bcLeft 3 # flags for boundary conditions +bcRight 3 # 1 = no-slip 3 = outflow +bcBottom 1 # 2 = free-slip 4 = periodic +bcTop 1 # +bcFront 1 # +bcBack 1 # + +gx 0.0 # Body forces (e.g. gravity) +gy 0.0 # +gz 0.0 # + +re 5050.0 # Reynolds number + +u_init 1.0 # initial value for velocity in x-direction +v_init 0.0 # initial value for velocity in y-direction +w_init 0.0 # initial value for velocity in z-direction +p_init 0.0 # initial value for pressure + +# Geometry Data: +# ------------- + +xlength 30.0 # domain size in x-direction +ylength 8.0 # domain size in y-direction +zlength 8.0 # domain size in z-direction +imax 200 # number of interior cells in x-direction +jmax 80 # number of interior cells in y-direction +kmax 80 # number of interior cells in z-direction + +# Time Data: +# --------- + +te 250.0 # final time +dt 0.02 # time stepsize +tau 0.5 # safety factor for time stepsize control (<0 constant delt) + +# Pressure Iteration Data: +# ----------------------- + +itermax 200 # maximal number of pressure iteration in one time step +eps 0.001 # stopping tolerance for pressure iteration +rho 0.52 +omg 1.75 # relaxation parameter for SOR iteration +gamma 0.9 # upwind differencing factor gamma + +# Multigrid data: +# --------- + +levels 3 # Multigrid levels +presmooth 20 # Pre-smoothning iterations +postsmooth 5 # Post-smoothning iterations + +# Particle Tracing Data: +# ----------------------- + +numberOfParticles 500 +startTime 50 +injectTimePeriod 1.0 +writeTimePeriod 1.0 + +x1 0.0 +y1 3.6 +z1 3.6 +x2 0.0 +y2 4.7 +z2 4.7 + +# Obstacle Geometry Data: +# ----------------------- +# Shape 0 disable, 1 Rectangle/Square, 2 Circle + +shape 2 +xCenter 5.0 +yCenter 4.0 +zCenter 4.0 +xRectLength 8.0 +yRectLength 2.0 +zRectLength 2.0 +circleRadius 2.0 + +#=============================================================================== diff --git a/EnhancedSolver/3D-seq/residual.plot b/EnhancedSolver/3D-seq/residual.plot new file mode 100644 index 0000000..36fb011 --- /dev/null +++ b/EnhancedSolver/3D-seq/residual.plot @@ -0,0 +1,9 @@ +set terminal png size 1800,768 enhanced font ,12 +set output 'residual.png' +set datafile separator whitespace +set xlabel "Timestep" +set ylabel "Residual" + +set logscale y 2 + +plot 'residual.dat' using 1:2 title "Residual" \ No newline at end of file diff --git a/BasicSolver/2D-seq-pt/src/allocate.c b/EnhancedSolver/3D-seq/src/allocate.c similarity index 83% rename from BasicSolver/2D-seq-pt/src/allocate.c rename to EnhancedSolver/3D-seq/src/allocate.c index 81e1e9d..cf2efd6 100644 --- a/BasicSolver/2D-seq-pt/src/allocate.c +++ b/EnhancedSolver/3D-seq/src/allocate.c @@ -1,14 +1,17 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ #include +#include #include #include -void* allocate(int alignment, size_t bytesize) +#include "allocate.h" + +void* allocate(size_t alignment, size_t bytesize) { int errorCode; void* ptr; diff --git a/BasicSolver/2D-mpi-v2/src/allocate.h b/EnhancedSolver/3D-seq/src/allocate.h similarity index 64% rename from BasicSolver/2D-mpi-v2/src/allocate.h rename to EnhancedSolver/3D-seq/src/allocate.h index 54cfe06..77f4ba0 100644 --- a/BasicSolver/2D-mpi-v2/src/allocate.h +++ b/EnhancedSolver/3D-seq/src/allocate.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -8,6 +8,6 @@ #define __ALLOCATE_H_ #include -extern void* allocate(int alignment, size_t bytesize); +extern void* allocate(size_t alignment, size_t bytesize); #endif diff --git a/EnhancedSolver/3D-seq/src/discretization.c b/EnhancedSolver/3D-seq/src/discretization.c new file mode 100644 index 0000000..3661c70 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/discretization.c @@ -0,0 +1,1128 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "allocate.h" +#include "discretization.h" +#include "parameter.h" +#include "util.h" + +static double distance( + double i, double j, double k, double iCenter, double jCenter, double kCenter) +{ + return sqrt(pow(iCenter - i, 2) + pow(jCenter - j, 2) + pow(kCenter - k, 2) * 1.0); +} + +static void printConfig(Discretization* d) +{ + printf("Parameters for #%s#\n", d->problem); + printf("BC Left:%d Right:%d Bottom:%d Top:%d Front:%d Back:%d\n", + d->bcLeft, + d->bcRight, + d->bcBottom, + d->bcTop, + d->bcFront, + d->bcBack); + printf("\tReynolds number: %.2f\n", d->re); + printf("\tGx Gy: %.2f %.2f %.2f\n", d->gx, d->gy, d->gz); + printf("Geometry data:\n"); + printf("\tDomain box size (x, y, z): %.2f, %.2f, %.2f\n", + d->grid.xlength, + d->grid.ylength, + d->grid.zlength); + printf("\tCells (x, y, z): %d, %d, %d\n", d->grid.imax, d->grid.jmax, d->grid.kmax); + printf("\tCell size (dx, dy, dz): %f, %f, %f\n", d->grid.dx, d->grid.dy, d->grid.dz); + printf("Timestep parameters:\n"); + printf("\tDefault stepsize: %.2f, Final time %.2f\n", d->dt, d->te); + printf("\tdt bound: %.6f\n", d->dtBound); + printf("\tTau factor: %.2f\n", d->tau); + printf("Iterative parameters:\n"); + printf("\tepsilon (stopping tolerance) : %f\n", d->eps); + printf("\tgamma factor: %f\n", d->gamma); + printf("\tomega (SOR relaxation): %f\n", d->omega); +} + +void initDiscretization(Discretization* d, Parameter* p) +{ + d->problem = p->name; + d->bcLeft = p->bcLeft; + d->bcRight = p->bcRight; + d->bcBottom = p->bcBottom; + d->bcTop = p->bcTop; + d->bcFront = p->bcFront; + d->bcBack = p->bcBack; + + d->grid.imax = p->imax; + d->grid.jmax = p->jmax; + d->grid.kmax = p->kmax; + d->grid.xlength = p->xlength; + d->grid.ylength = p->ylength; + d->grid.zlength = p->zlength; + d->grid.dx = p->xlength / p->imax; + d->grid.dy = p->ylength / p->jmax; + d->grid.dz = p->zlength / p->kmax; + + d->eps = p->eps; + d->omega = p->omg; + d->re = p->re; + d->gx = p->gx; + d->gy = p->gy; + d->gz = p->gz; + d->dt = p->dt; + d->te = p->te; + d->tau = p->tau; + d->gamma = p->gamma; + + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + size_t bytesize = (imax + 2) * (jmax + 2) * (kmax + 2) * sizeof(double); + d->u = allocate(64, bytesize); + d->v = allocate(64, bytesize); + d->w = allocate(64, bytesize); + d->p = allocate(64, bytesize); + d->rhs = allocate(64, bytesize); + d->f = allocate(64, bytesize); + d->g = allocate(64, bytesize); + d->h = allocate(64, bytesize); + d->s = allocate(64, bytesize); + + for (int i = 0; i < (imax + 2) * (jmax + 2) * (kmax + 2); i++) { + d->u[i] = p->u_init; + d->v[i] = p->v_init; + d->w[i] = p->w_init; + d->p[i] = p->p_init; + d->rhs[i] = 0.0; + d->f[i] = 0.0; + d->g[i] = 0.0; + d->h[i] = 0.0; + d->s[i] = FLUID; + } + + double dx = d->grid.dx; + double dy = d->grid.dy; + double dz = d->grid.dz; + + double invSqrSum = 1.0 / (dx * dx) + 1.0 / (dy * dy) + 1.0 / (dz * dz); + d->dtBound = 0.5 * d->re * 1.0 / invSqrSum; + + double xCenter = 0, yCenter = 0, zCenter = 0, radius = 0; + double x1 = 0, x2 = 0, y1 = 0, y2 = 0, z1 = 0, z2 = 0; + + double* s = d->s; + + switch (p->shape) { + case NOSHAPE: + break; + case RECT: + x1 = p->xCenter - p->xRectLength / 2; + x2 = p->xCenter + p->xRectLength / 2; + y1 = p->yCenter - p->yRectLength / 2; + y2 = p->yCenter + p->yRectLength / 2; + z1 = p->zCenter - p->zRectLength / 2; + z2 = p->zCenter + p->zRectLength / 2; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + if ((x1 <= (i * dx)) && ((i * dx) <= x2) && (y1 <= (j * dy)) && + ((j * dy) <= y2) && ((z1 <= (k * dz)) && ((k * dz) <= z2))) { + S(i, j, k) = OBSTACLE; + } + } + } + } + + break; + case CIRCLE: + xCenter = p->xCenter; + yCenter = p->yCenter; + zCenter = p->zCenter; + radius = p->circleRadius; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + if (distance((i * dx), + (j * dy), + (k * dz), + xCenter, + yCenter, + zCenter) <= radius) { + S(i, j, k) = OBSTACLE; + } + } + } + } + break; + default: + break; + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + + /* Assigning enum values to Corners */ + if (S(i - 1, j + 1, k - 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i + 1, j - 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPLEFTCORNER; // + } + if (S(i + 1, j + 1, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j - 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPRIGHTCORNER; // + } + if (S(i - 1, j - 1, k - 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i + 1, j + 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMLEFTCORNER; // + } + if (S(i + 1, j - 1, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j + 1, k + 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMRIGHTCORNER; // + } + if (S(i - 1, j + 1, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i + 1, j - 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPLEFTCORNER; // + } + if (S(i + 1, j + 1, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i - 1, j - 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPRIGHTCORNER; + } + if (S(i - 1, j - 1, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i + 1, j + 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMLEFTCORNER; + } + if (S(i + 1, j - 1, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j, k + 1) == FLUID && + S(i - 1, j + 1, k - 1) == OBSTACLE && S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMRIGHTCORNER; + } + /* Assigning enum values to Lines */ + if (S(i - 1, j, k - 1) == FLUID && S(i, j, k - 1) == FLUID && + S(i - 1, j, k) == FLUID && S(i + 1, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTLEFTLINE; + } + if (S(i + 1, j, k - 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j, k - 1) == FLUID && S(i - 1, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTRIGHTLINE; + } + if (S(i, j + 1, k - 1) == FLUID && S(i, j + 1, k) == FLUID && + S(i, j, k - 1) == FLUID && S(i, j - 1, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTTOPLINE; + } + if (S(i, j - 1, k - 1) == FLUID && S(i, j, k - 1) == FLUID && + S(i, j - 1, k) == FLUID && S(i, j + 1, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTBOTTOMLINE; + } + if (S(i - 1, j + 1, k) == FLUID && S(i, j + 1, k) == FLUID && + S(i - 1, j, k) == FLUID && S(i + 1, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDTOPLEFTLINE; + } + if (S(i + 1, j + 1, k) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j + 1, k) == FLUID && S(i - 1, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDTOPRIGHTLINE; + } + if (S(i - 1, j - 1, k) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i + 1, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDBOTTOMLEFTLINE; + } + if (S(i + 1, j - 1, k) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j - 1, k) == FLUID && S(i - 1, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = MIDBOTTOMRIGHTLINE; + } + if (S(i - 1, j, k + 1) == FLUID && S(i - 1, j, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i + 1, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKLEFTLINE; + } + if (S(i + 1, j, k + 1) == FLUID && S(i + 1, j, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i - 1, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKRIGHTLINE; + } + if (S(i, j + 1, k + 1) == FLUID && S(i, j + 1, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i, j - 1, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKTOPLINE; + } + if (S(i, j - 1, k + 1) == FLUID && S(i, j - 1, k) == FLUID && + S(i, j, k + 1) == FLUID && S(i, j + 1, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKBOTTOMLINE; + } + /* Assigning enum values to Faces */ + if (S(i, j, k - 1) == FLUID && S(i, j, k + 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = FRONTFACE; // + } + if (S(i, j, k + 1) == FLUID && S(i, j, k - 1) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BACKFACE; // + } + if (S(i, j - 1, k) == FLUID && S(i, j + 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = BOTTOMFACE; // + } + if (S(i, j + 1, k) == FLUID && S(i, j - 1, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = TOPFACE; // + } + if (S(i - 1, j, k) == FLUID && S(i + 1, j, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = LEFTFACE; // + } + if (S(i + 1, j, k) == FLUID && S(i - 1, j, k) == OBSTACLE && + S(i, j, k) == OBSTACLE) { + S(i, j, k) = RIGHTFACE; // + } + } + } + } + +#ifdef VERBOSE + printConfig(d); +#endif /* VERBOSE */ +} + +void computeRHS(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + double idx = 1.0 / d->grid.dx; + double idy = 1.0 / d->grid.dy; + double idz = 1.0 / d->grid.dz; + double idt = 1.0 / d->dt; + + double* rhs = d->rhs; + double* f = d->f; + double* g = d->g; + double* h = d->h; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + RHS(i, j, k) = ((F(i, j, k) - F(i - 1, j, k)) * idx + + (G(i, j, k) - G(i, j - 1, k)) * idy + + (H(i, j, k) - H(i, j, k - 1)) * idz) * + idt; + } + } + } +} + +static double maxElement(Discretization* d, double* m) +{ + int size = (d->grid.imax + 2) * (d->grid.jmax + 2) * (d->grid.kmax + 2); + double maxval = DBL_MIN; + + for (int i = 0; i < size; i++) { + maxval = MAX(maxval, fabs(m[i])); + } + + return maxval; +} + +void normalizePressure(Discretization* d) +{ + int size = (d->grid.imax + 2) * (d->grid.jmax + 2) * (d->grid.kmax + 2); + double* p = d->p; + double avgP = 0.0; + + for (int i = 0; i < size; i++) { + avgP += p[i]; + } + avgP /= size; + + for (int i = 0; i < size; i++) { + p[i] = p[i] - avgP; + } +} + +void computeTimestep(Discretization* d) +{ + double dt = d->dtBound; + double dx = d->grid.dx; + double dy = d->grid.dy; + double dz = d->grid.dz; + + double umax = maxElement(d, d->u); + double vmax = maxElement(d, d->v); + double wmax = maxElement(d, d->w); + + if (umax > 0) { + dt = (dt > dx / umax) ? dx / umax : dt; + } + if (vmax > 0) { + dt = (dt > dy / vmax) ? dy / vmax : dt; + } + if (wmax > 0) { + dt = (dt > dz / wmax) ? dz / wmax : dt; + } + + d->dt = dt * d->tau; +} + +void setBoundaryConditions(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + + double* u = d->u; + double* v = d->v; + double* w = d->w; + + switch (d->bcTop) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, jmax, k) = 0.0; + U(i, jmax + 1, k) = -U(i, jmax, k); + W(i, jmax + 1, k) = -W(i, jmax, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, jmax, k) = 0.0; + U(i, jmax + 1, k) = U(i, jmax, k); + W(i, jmax + 1, k) = W(i, jmax, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + U(i, jmax + 1, k) = U(i, jmax, k); + V(i, jmax, k) = V(i, jmax - 1, k); + W(i, jmax + 1, k) = W(i, jmax, k); + } + } + break; + case PERIODIC: + break; + } + + switch (d->bcBottom) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, 0, k) = 0.0; + U(i, 0, k) = -U(i, 1, k); + W(i, 0, k) = -W(i, 1, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + V(i, 0, k) = 0.0; + U(i, 0, k) = U(i, 1, k); + W(i, 0, k) = W(i, 1, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + U(i, 0, k) = U(i, 1, k); + V(i, 0, k) = V(i, 1, k); + W(i, 0, k) = W(i, 1, k); + } + } + break; + case PERIODIC: + break; + } + + switch (d->bcLeft) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = -V(1, j, k); + W(0, j, k) = -W(1, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(0, j, k) = 0.0; + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(0, j, k) = U(1, j, k); + V(0, j, k) = V(1, j, k); + W(0, j, k) = W(1, j, k); + } + } + break; + case PERIODIC: + break; + } + + switch (d->bcRight) { + case NOSLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(imax, j, k) = 0.0; + V(imax + 1, j, k) = -V(imax, j, k); + W(imax + 1, j, k) = -W(imax, j, k); + } + } + break; + case SLIP: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(imax, j, k) = 0.0; + V(imax + 1, j, k) = V(imax, j, k); + W(imax + 1, j, k) = W(imax, j, k); + } + } + break; + case OUTFLOW: + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(imax, j, k) = U(imax - 1, j, k); + V(imax + 1, j, k) = V(imax, j, k); + W(imax + 1, j, k) = W(imax, j, k); + } + } + break; + case PERIODIC: + break; + } + + switch (d->bcFront) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, 0) = -U(i, j, 1); + V(i, j, 0) = -V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, 0) = U(i, j, 1); + V(i, j, 0) = V(i, j, 1); + W(i, j, 0) = W(i, j, 1); + } + } + break; + case PERIODIC: + break; + } + + switch (d->bcBack) { + case NOSLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, kmax + 1) = -U(i, j, kmax); + V(i, j, kmax + 1) = -V(i, j, kmax); + W(i, j, kmax + 1) = 0.0; + } + } + break; + case SLIP: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, kmax + 1) = U(i, j, kmax); + V(i, j, kmax + 1) = V(i, j, kmax); + W(i, j, kmax + 1) = 0.0; + } + } + break; + case OUTFLOW: + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, kmax + 1) = U(i, j, kmax); + V(i, j, kmax + 1) = V(i, j, kmax); + W(i, j, kmax) = W(i, j, kmax - 1); + } + } + break; + case PERIODIC: + break; + } +} + +void setSpecialBoundaryCondition(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + + double mDy = d->grid.dy; + double* u = d->u; + + if (strcmp(d->problem, "dcavity") == 0) { + for (int k = 1; k < kmax; k++) { + for (int i = 1; i < imax; i++) { + U(i, jmax + 1, k) = 2.0 - U(i, jmax, k); + } + } + } else if (strcmp(d->problem, "canal") == 0) { + double ylength = d->grid.ylength; + double y; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + y = mDy * (j - 0.5); + U(0, j, k) = y * (ylength - y) * 4.0 / (ylength * ylength); + } + } + } else if (strcmp(d->problem, "backstep") == 0) { + + double* s = d->s; + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + if (S(1, j, k) == FLUID) U(0, j, k) = 1.0; + else { + U(0, j, k) = 0.0; + U(1, j, k) = 0.0; + } + } + } + } else if (strcmp(d->problem, "karman") == 0) { + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + U(1, j, k) = 1.0; + } + } + } +} + +void computeFG(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* f = d->f; + double* g = d->g; + double* h = d->h; + double* s = d->s; + + double gx = d->gx; + double gy = d->gy; + double gz = d->gz; + double dt = d->dt; + + double gamma = d->gamma; + double inverseRe = 1.0 / d->re; + double inverseDx = 1.0 / d->grid.dx; + double inverseDy = 1.0 / d->grid.dy; + double inverseDz = 1.0 / d->grid.dz; + double du2dx, dv2dy, dw2dz; + double duvdx, duwdx, duvdy, dvwdy, duwdz, dvwdz; + double du2dx2, du2dy2, du2dz2; + double dv2dx2, dv2dy2, dv2dz2; + double dw2dx2, dw2dy2, dw2dz2; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + if ((int)S(i, j, k) == FLUID) { + du2dx = inverseDx * 0.25 * + ((U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) + U(i + 1, j, k)) - + (U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) + U(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i + 1, j, k)) * + (U(i, j, k) - U(i + 1, j, k)) + + fabs(U(i, j, k) + U(i - 1, j, k)) * + (U(i, j, k) - U(i - 1, j, k))); + + duvdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) + U(i, j + 1, k)) - + (V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) + U(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i + 1, j, k)) * + (U(i, j, k) - U(i, j + 1, k)) + + fabs(V(i, j - 1, k) + V(i + 1, j - 1, k)) * + (U(i, j, k) - U(i, j - 1, k))); + + duwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) + U(i, j, k + 1)) - + (W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) + U(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i + 1, j, k)) * + (U(i, j, k) - U(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i + 1, j, k - 1)) * + (U(i, j, k) - U(i, j, k - 1))); + + du2dx2 = inverseDx * inverseDx * + (U(i + 1, j, k) - 2.0 * U(i, j, k) + U(i - 1, j, k)); + du2dy2 = inverseDy * inverseDy * + (U(i, j + 1, k) - 2.0 * U(i, j, k) + U(i, j - 1, k)); + du2dz2 = inverseDz * inverseDz * + (U(i, j, k + 1) - 2.0 * U(i, j, k) + U(i, j, k - 1)); + F(i, j, k) = U(i, j, k) + + dt * (inverseRe * (du2dx2 + du2dy2 + du2dz2) - du2dx - + duvdy - duwdz + gx); + + duvdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) + V(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) + V(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j + 1, k)) * + (V(i, j, k) - V(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j + 1, k)) * + (V(i, j, k) - V(i - 1, j, k))); + + dv2dy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) + V(i, j + 1, k)) - + (V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) + V(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j + 1, k)) * + (V(i, j, k) - V(i, j + 1, k)) + + fabs(V(i, j, k) + V(i, j - 1, k)) * + (V(i, j, k) - V(i, j - 1, k))); + + dvwdz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) + V(i, j, k + 1)) - + (W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) + V(i, j, k + 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j + 1, k)) * + (V(i, j, k) - V(i, j, k + 1)) + + fabs(W(i, j, k - 1) + W(i, j + 1, k - 1)) * + (V(i, j, k) - V(i, j, k + 1))); + + dv2dx2 = inverseDx * inverseDx * + (V(i + 1, j, k) - 2.0 * V(i, j, k) + V(i - 1, j, k)); + dv2dy2 = inverseDy * inverseDy * + (V(i, j + 1, k) - 2.0 * V(i, j, k) + V(i, j - 1, k)); + dv2dz2 = inverseDz * inverseDz * + (V(i, j, k + 1) - 2.0 * V(i, j, k) + V(i, j, k - 1)); + G(i, j, k) = V(i, j, k) + + dt * (inverseRe * (dv2dx2 + dv2dy2 + dv2dz2) - duvdx - + dv2dy - dvwdz + gy); + + duwdx = inverseDx * 0.25 * + ((U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) + W(i + 1, j, k)) - + (U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) + W(i - 1, j, k))) + + gamma * inverseDx * 0.25 * + (fabs(U(i, j, k) + U(i, j, k + 1)) * + (W(i, j, k) - W(i + 1, j, k)) + + fabs(U(i - 1, j, k) + U(i - 1, j, k + 1)) * + (W(i, j, k) - W(i - 1, j, k))); + + dvwdy = inverseDy * 0.25 * + ((V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) + W(i, j + 1, k)) - + (V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) + W(i, j - 1, k))) + + gamma * inverseDy * 0.25 * + (fabs(V(i, j, k) + V(i, j, k + 1)) * + (W(i, j, k) - W(i, j + 1, k)) + + fabs(V(i, j - 1, k + 1) + V(i, j - 1, k)) * + (W(i, j, k) - W(i, j - 1, k))); + + dw2dz = inverseDz * 0.25 * + ((W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) + W(i, j, k + 1)) - + (W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) + W(i, j, k - 1))) + + gamma * inverseDz * 0.25 * + (fabs(W(i, j, k) + W(i, j, k + 1)) * + (W(i, j, k) - W(i, j, k + 1)) + + fabs(W(i, j, k) + W(i, j, k - 1)) * + (W(i, j, k) - W(i, j, k - 1))); + + dw2dx2 = inverseDx * inverseDx * + (W(i + 1, j, k) - 2.0 * W(i, j, k) + W(i - 1, j, k)); + dw2dy2 = inverseDy * inverseDy * + (W(i, j + 1, k) - 2.0 * W(i, j, k) + W(i, j - 1, k)); + dw2dz2 = inverseDz * inverseDz * + (W(i, j, k + 1) - 2.0 * W(i, j, k) + W(i, j, k - 1)); + H(i, j, k) = W(i, j, k) + + dt * (inverseRe * (dw2dx2 + dw2dy2 + dw2dz2) - duwdx - + dvwdy - dw2dz + gz); + } else { + switch ((int)S(i, j, k)) { + case TOPFACE: + G(i, j, k) = 0.0; + break; + case BOTTOMFACE: + G(i, j, k) = 0.0; + break; + case LEFTFACE: + F(i, j, k) = 0.0; + break; + case RIGHTFACE: + F(i, j, k) = 0.0; + break; + case FRONTFACE: + H(i, j, k) = 0.0; + break; + case BACKFACE: + H(i, j, k) = 0.0; + break; + case FRONTLEFTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTRIGHTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case MIDTOPLEFTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case MIDTOPRIGHTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case MIDBOTTOMLEFTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case MIDBOTTOMRIGHTLINE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + break; + case BACKLEFTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKRIGHTLINE: + F(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMLINE: + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTTOPRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case FRONTBOTTOMRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKTOPRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMLEFTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case BACKBOTTOMRIGHTCORNER: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + case OBSTACLE: + F(i, j, k) = 0.0; + G(i, j, k) = 0.0; + H(i, j, k) = 0.0; + break; + } + } + } + } + } + /* ----------------------------- boundary of F --------------------------- + */ + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + F(0, j, k) = U(0, j, k); + F(imax, j, k) = U(imax, j, k); + } + } + + /* ----------------------------- boundary of G --------------------------- + */ + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + G(i, 0, k) = V(i, 0, k); + G(i, jmax, k) = V(i, jmax, k); + } + } + + /* ----------------------------- boundary of G --------------------------- + */ + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + H(i, j, 0) = W(i, j, 0); + H(i, j, kmax) = W(i, j, kmax); + } + } +} + +void adaptUV(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + + double* p = d->p; + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* f = d->f; + double* g = d->g; + double* h = d->h; + + double factorX = d->dt / d->grid.dx; + double factorY = d->dt / d->grid.dy; + double factorZ = d->dt / d->grid.dz; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + U(i, j, k) = F(i, j, k) - (P(i + 1, j, k) - P(i, j, k)) * factorX; + V(i, j, k) = G(i, j, k) - (P(i, j + 1, k) - P(i, j, k)) * factorY; + W(i, j, k) = H(i, j, k) - (P(i, j, k + 1) - P(i, j, k)) * factorZ; + } + } + } +} + +void setObjectBoundaryCondition(Discretization* d) +{ + int imax = d->grid.imax; + int jmax = d->grid.jmax; + int kmax = d->grid.kmax; + double* u = d->u; + double* v = d->v; + double* w = d->w; + double* s = d->s; + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + switch ((int)S(i, j, k)) { + case TOPFACE: + U(i, j, k) = -U(i, j + 1, k); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + + break; + case BOTTOMFACE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case LEFTFACE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i - 1, j, k); + break; + case RIGHTFACE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i + 1, j, k); + break; + case FRONTFACE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = 0.0; + break; + case BACKFACE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = 0.0; + break; + case FRONTLEFTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = -W(i - 1, j, k); + break; + case FRONTRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k - 1); + W(i, j, k) = -W(i + 1, j, k); + break; + case FRONTTOPLINE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTBOTTOMLINE: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case MIDTOPLEFTLINE: + U(i, j, k) = -U(i, j + 1, k); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = 0.0; + break; + case MIDTOPRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = 0.0; + U(i - 1, j, k) = -U(i - 1, j + 1, k); + V(i, j - 1, k) = -V(i + 1, j - 1, k); + W(i, j, k) = 0.0; + break; + case MIDBOTTOMLEFTLINE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = 0.0; + break; + case MIDBOTTOMRIGHTLINE: + U(i, j, k) = -U(i, j - 1, k); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = 0.0; + break; + case BACKLEFTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = -W(i - 1, j, k); + break; + case BACKRIGHTLINE: + U(i, j, k) = 0.0; + V(i, j, k) = -V(i, j, k + 1); + W(i, j, k) = -W(i + 1, j, k); + break; + case BACKTOPLINE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKBOTTOMLINE: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = 0.0; + W(i, j, k) = -W(i, j - 1, k); + break; + case FRONTTOPLEFTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTTOPRIGHTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case FRONTBOTTOMLEFTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case FRONTBOTTOMRIGHTCORNER: + U(i, j, k) = -U(i, j, k - 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case BACKTOPLEFTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKTOPRIGHTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j + 1, k); + break; + case BACKBOTTOMLEFTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i - 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case BACKBOTTOMRIGHTCORNER: + U(i, j, k) = -U(i, j, k + 1); + V(i, j, k) = -V(i + 1, j, k); + W(i, j, k) = -W(i, j - 1, k); + break; + case OBSTACLE: + U(i, j, k) = 0.0; + V(i, j, k) = 0.0; + W(i, j, k) = 0.0; + } + } + } + } +} \ No newline at end of file diff --git a/EnhancedSolver/3D-seq/src/discretization.h b/EnhancedSolver/3D-seq/src/discretization.h new file mode 100644 index 0000000..c86ada5 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/discretization.h @@ -0,0 +1,86 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __DISCRETIZATION_H_ +#define __DISCRETIZATION_H_ + +#include "grid.h" +#include "parameter.h" + +enum OBJECTBOUNDARY { + FLUID = 0, + /* Front Corners */ + FRONTTOPLEFTCORNER, + FRONTTOPRIGHTCORNER, + FRONTBOTTOMLEFTCORNER, + FRONTBOTTOMRIGHTCORNER, + /* Back Corners */ + BACKTOPLEFTCORNER, + BACKTOPRIGHTCORNER, + BACKBOTTOMLEFTCORNER, + BACKBOTTOMRIGHTCORNER, + /* Faces */ + FRONTFACE, + BACKFACE, + LEFTFACE, + RIGHTFACE, + TOPFACE, + BOTTOMFACE, + /* Front Lines remaining after Corners and Faces */ + FRONTLEFTLINE, + FRONTRIGHTLINE, + FRONTTOPLINE, + FRONTBOTTOMLINE, + /* Bottom Lines remaining after Corners and Faces */ + BACKLEFTLINE, + BACKRIGHTLINE, + BACKTOPLINE, + BACKBOTTOMLINE, + /* Mid Lines remaining after Corners and Faces */ + MIDTOPLEFTLINE, + MIDTOPRIGHTLINE, + MIDBOTTOMLEFTLINE, + MIDBOTTOMRIGHTLINE, + /* Local where its an object but not a boundary */ + OBSTACLE, + /*Ghost cells boundary */ + OUTSIDEBOUNDARY +}; + +enum BC { NOSLIP = 1, SLIP, OUTFLOW, PERIODIC }; + + +enum SHAPE { NOSHAPE = 0, RECT, CIRCLE }; + +typedef struct { + /* geometry and grid information */ + Grid grid; + /* arrays */ + double *p, *rhs; + double *f, *g, *h; + double *u, *v, *w; + double* s; + /* parameters */ + double eps, omega; + double re, tau, gamma; + double gx, gy, gz; + /* time stepping */ + double dt, te; + double dtBound; + char* problem; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; +} Discretization; + +extern void initDiscretization(Discretization*, Parameter*); +extern void computeRHS(Discretization*); +extern void normalizePressure(Discretization*); +extern void computeTimestep(Discretization*); +extern void setBoundaryConditions(Discretization*); +extern void setObjectBoundaryCondition(Discretization*); +extern void setSpecialBoundaryCondition(Discretization*); +extern void computeFG(Discretization*); +extern void adaptUV(Discretization*); +#endif diff --git a/BasicSolver/2D-seq-pt/src/grid.h b/EnhancedSolver/3D-seq/src/grid.h similarity index 84% rename from BasicSolver/2D-seq-pt/src/grid.h rename to EnhancedSolver/3D-seq/src/grid.h index c963429..0689ebc 100644 --- a/BasicSolver/2D-seq-pt/src/grid.h +++ b/EnhancedSolver/3D-seq/src/grid.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. diff --git a/BasicSolver/3D-mpi-io/src/likwid-marker.h b/EnhancedSolver/3D-seq/src/likwid-marker.h similarity index 100% rename from BasicSolver/3D-mpi-io/src/likwid-marker.h rename to EnhancedSolver/3D-seq/src/likwid-marker.h diff --git a/EnhancedSolver/3D-seq/src/main.c b/EnhancedSolver/3D-seq/src/main.c new file mode 100644 index 0000000..08be93c --- /dev/null +++ b/EnhancedSolver/3D-seq/src/main.c @@ -0,0 +1,157 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "allocate.h" +#include "discretization.h" +#include "parameter.h" +#include "particletracing.h" +#include "progress.h" +#include "solver.h" +#include "timing.h" +#include "vtkWriter.h" + +#define G(v, i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +static void createBulkArrays( + Discretization* s, double* pg, double* ug, double* vg, double* wg) +{ + int imax = s->grid.imax; + int jmax = s->grid.jmax; + int kmax = s->grid.kmax; + int idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + pg[idx++] = G(s->p, i, j, k); + } + } + } + + idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + ug[idx++] = (G(s->u, i, j, k) + G(s->u, i - 1, j, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + vg[idx++] = (G(s->v, i, j, k) + G(s->v, i, j - 1, k)) / 2.0; + } + } + } + + idx = 0; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + wg[idx++] = (G(s->w, i, j, k) + G(s->w, i, j, k - 1)) / 2.0; + } + } + } +} + +int main(int argc, char** argv) +{ + double timeStart, timeStop; + Parameter p; + Discretization d; + Solver s; + ParticleTracer particletracer; + initParameter(&p); + + FILE* fp; + fp = initResidualWriter(); + + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(EXIT_SUCCESS); + } + + readParameter(&p, argv[1]); + printParameter(&p); + initDiscretization(&d, &p); + initSolver(&s, &d, &p); + initParticleTracer(&particletracer, &p); + printParticleTracerParameters(&particletracer); + +#ifndef VERBOSE + initProgress(d.te); +#endif + + double tau = d.tau; + double te = d.te; + double t = 0.0; + int nt = 0; + double res = 0.0; + + timeStart = getTimeStamp(); + while (t <= te) { + if (tau > 0.0) computeTimestep(&d); + setBoundaryConditions(&d); + setSpecialBoundaryCondition(&d); + setObjectBoundaryCondition(&d); + + computeFG(&d); + computeRHS(&d); + if (nt % 100 == 0) normalizePressure(&d); + res = solve(&s, d.p, d.rhs); + adaptUV(&d); + + trace(&particletracer, d.u, d.v, d.w, d.s, t); + + writeResidual(fp, t, res); + + t += d.dt; + nt++; + +#ifdef VERBOSE + printf("TIME %f , TIMESTEP %f\n", t, d.dt); +#else + printProgress(t); +#endif + } + timeStop = getTimeStamp(); + stopProgress(); + printf("Solution took %.2fs\n", timeStop - timeStart); + + timeStart = getTimeStamp(); + double *pg, *ug, *vg, *wg; + + size_t bytesize = (size_t)(d.grid.imax * d.grid.jmax * d.grid.kmax) * sizeof(double); + + pg = allocate(64, bytesize); + ug = allocate(64, bytesize); + vg = allocate(64, bytesize); + wg = allocate(64, bytesize); + + fclose(fp); + freeParticles(&particletracer); + + createBulkArrays(&d, pg, ug, vg, wg); + VtkOptions opts = { .grid = d.grid }; + vtkOpen(&opts, d.problem); + vtkScalar(&opts, "pressure", pg); + vtkVector(&opts, "velocity", (VtkVector) { ug, vg, wg }); + vtkClose(&opts); + + timeStop = getTimeStamp(); + printf("Result output took %.2fs\n", timeStop - timeStart); + + return EXIT_SUCCESS; +} diff --git a/BasicSolver/3D-mpi-io/src/parameter.c b/EnhancedSolver/3D-seq/src/parameter.c similarity index 72% rename from BasicSolver/3D-mpi-io/src/parameter.c rename to EnhancedSolver/3D-seq/src/parameter.c index d4ea5b6..79d5a7c 100644 --- a/BasicSolver/3D-mpi-io/src/parameter.c +++ b/EnhancedSolver/3D-seq/src/parameter.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. This file is part of nusif-solver. * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. @@ -14,18 +14,21 @@ void initParameter(Parameter* param) { - param->xlength = 1.0; - param->ylength = 1.0; - param->zlength = 1.0; - param->imax = 100; - param->jmax = 100; - param->kmax = 100; - param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.7; - param->re = 100.0; - param->gamma = 0.9; - param->tau = 0.5; + param->xlength = 1.0; + param->ylength = 1.0; + param->zlength = 1.0; + param->imax = 100; + param->jmax = 100; + param->kmax = 100; + param->itermax = 1000; + param->eps = 0.0001; + param->omg = 1.7; + param->re = 100.0; + param->gamma = 0.9; + param->tau = 0.5; + param->levels = 5; + param->presmooth = 5; + param->postsmooth = 5; } void readParameter(Parameter* param, const char* filename) @@ -65,6 +68,7 @@ void readParameter(Parameter* param, const char* filename) PARSE_INT(jmax); PARSE_INT(kmax); PARSE_INT(itermax); + PARSE_INT(levels); PARSE_REAL(eps); PARSE_REAL(omg); PARSE_REAL(re); @@ -86,6 +90,28 @@ void readParameter(Parameter* param, const char* filename) PARSE_REAL(v_init); PARSE_REAL(w_init); PARSE_REAL(p_init); + + /* Added new particle tracing parameters */ + PARSE_INT(numberOfParticles); + PARSE_REAL(startTime); + PARSE_REAL(injectTimePeriod); + PARSE_REAL(writeTimePeriod); + PARSE_REAL(x1); + PARSE_REAL(y1); + PARSE_REAL(z1); + PARSE_REAL(x2); + PARSE_REAL(y2); + PARSE_REAL(z2); + + /* Added obstacle geometry parameters */ + PARSE_INT(shape); + PARSE_REAL(xCenter); + PARSE_REAL(yCenter); + PARSE_REAL(zCenter); + PARSE_REAL(xRectLength); + PARSE_REAL(yRectLength); + PARSE_REAL(zRectLength); + PARSE_REAL(circleRadius); } } @@ -123,4 +149,5 @@ void printParameter(Parameter* param) printf("\tepsilon (stopping tolerance) : %f\n", param->eps); printf("\tgamma (stopping tolerance) : %f\n", param->gamma); printf("\tomega (SOR relaxation): %f\n", param->omg); + printf("\tMultiGrid levels : %d\n", param->levels); } diff --git a/EnhancedSolver/3D-seq/src/parameter.h b/EnhancedSolver/3D-seq/src/parameter.h new file mode 100644 index 0000000..6a54d3a --- /dev/null +++ b/EnhancedSolver/3D-seq/src/parameter.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARAMETER_H_ +#define __PARAMETER_H_ + +typedef struct { + int imax, jmax, kmax; + double xlength, ylength, zlength; + int itermax, levels; + double eps, omg, rho; + double re, tau, gamma; + double te, dt; + double gx, gy, gz; + char* name; + int bcLeft, bcRight, bcBottom, bcTop, bcFront, bcBack; + double u_init, v_init, w_init, p_init; + int presmooth, postsmooth; + + int numberOfParticles; + double startTime, injectTimePeriod, writeTimePeriod; + + double x1, y1, z1, x2, y2, z2; + + int shape; + double xCenter, yCenter, zCenter, xRectLength, yRectLength, zRectLength, circleRadius; +} Parameter; + +void initParameter(Parameter*); +void readParameter(Parameter*, const char*); +void printParameter(Parameter*); +#endif diff --git a/EnhancedSolver/3D-seq/src/particletracing.c b/EnhancedSolver/3D-seq/src/particletracing.c new file mode 100644 index 0000000..72a347f --- /dev/null +++ b/EnhancedSolver/3D-seq/src/particletracing.c @@ -0,0 +1,313 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include +#include + +#include "discretization.h" +#include "vtkWriter.h" + +#define U(i, j, k) u[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define V(i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define W(i, j, k) w[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define S(i, j, k) s[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +static int ts = 0; +unsigned int seed = 32767; +void printParticles(ParticleTracer* particletracer) +{ + for (int i = 0; i < particletracer->totalParticles; ++i) { + printf("Particle position X : %.2f, Y : %.2f, flag : %d\n", + particletracer->particlePool[i].x, + particletracer->particlePool[i].y, + particletracer->particlePool[i].flag); + } +} +void injectParticles(ParticleTracer* particletracer, double* s) +{ + + int imax = particletracer->imax; + int jmax = particletracer->jmax; + int kmax = particletracer->kmax; + + for (int i = 0; i < particletracer->numberOfParticles; ++i) { + + particletracer->particlePool[particletracer->pointer].x = + (((double)rand() / RAND_MAX) * (particletracer->x2 - particletracer->x1)) + + particletracer->x1; + particletracer->particlePool[particletracer->pointer].y = + (((double)rand() / RAND_MAX) * (particletracer->y2 - particletracer->y1)) + + particletracer->y1; + particletracer->particlePool[particletracer->pointer].z = + (((double)rand() / RAND_MAX) * (particletracer->z2 - particletracer->z1)) + + particletracer->z1; + + int i = particletracer->particlePool[particletracer->pointer].x / + particletracer->dx; + int j = particletracer->particlePool[particletracer->pointer].y / + particletracer->dy; + int k = particletracer->particlePool[particletracer->pointer].z / + particletracer->dz; + + if (S(i, j, k) == FLUID) { + particletracer->particlePool[particletracer->pointer].flag = true; + ++(particletracer->pointer); + ++(particletracer->totalParticles); + } else + particletracer->particlePool[particletracer->pointer].flag = false; + } +} + +void advanceParticles(ParticleTracer* particletracer, + double* restrict u, + double* restrict v, + double* restrict w, + double* restrict s, + double time) +{ + int imax = particletracer->imax; + int jmax = particletracer->jmax; + int kmax = particletracer->kmax; + + double dx = particletracer->dx; + double dy = particletracer->dy; + double dz = particletracer->dz; + + double xlength = particletracer->xlength; + double ylength = particletracer->ylength; + double zlength = particletracer->zlength; + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (particletracer->particlePool[i].flag == true) { + double x = particletracer->particlePool[i].x; + double y = particletracer->particlePool[i].y; + double z = particletracer->particlePool[i].z; + + int iCoord = (int)(x / dx) + 1; + int jCoord = (int)((y + 0.5 * dy) / dy) + 1; + int kCoord = (int)((z + 0.5 * dz) / dz) + 1; + + double x1 = (double)(iCoord - 1) * dx; + double y1 = ((double)(jCoord - 1) - 0.5) * dy; + double z1 = ((double)(kCoord - 1) - 0.5) * dz; + + double x2 = (double)iCoord * dx; + double y2 = ((double)jCoord - 0.5) * dy; + double z2 = ((double)kCoord - 0.5) * dz; + + double u_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * U(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * U(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * U(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * U(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * U(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * U(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * U(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * U(iCoord, jCoord, kCoord)); + + double new_x = x + particletracer->dt * u_n; + particletracer->particlePool[i].x = new_x; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)(y / dy) + 1; + kCoord = (int)((z + 0.5 * dz) / dz) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = (double)(jCoord - 1) * dy; + z1 = ((double)(kCoord - 1) - 0.5) * dz; + + x2 = ((double)iCoord - 0.5) * dx; + y2 = (double)jCoord * dy; + z2 = ((double)kCoord - 0.5) * dz; + + double v_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * V(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * V(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * V(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * V(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * V(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * V(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * V(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * V(iCoord, jCoord, kCoord)); + + double new_y = y + particletracer->dt * v_n; + particletracer->particlePool[i].y = new_y; + + iCoord = (int)((x + 0.5 * dx) / dx) + 1; + jCoord = (int)((y + 0.5 * dy) / dy) + 1; + kCoord = (int)(z / dz) + 1; + + x1 = ((double)(iCoord - 1) - 0.5) * dx; + y1 = ((double)(jCoord - 1) - 0.5) * dy; + z1 = (double)(kCoord - 1) * dz; + + x2 = ((double)iCoord - 0.5) * dx; + y2 = ((double)jCoord - 0.5) * dy; + z2 = (double)kCoord * dz; + + double w_n = + (1.0 / (dx * dy * dz)) * + ((x2 - x) * (y2 - y) * (z2 - z) * W(iCoord - 1, jCoord - 1, kCoord - 1) + + (x - x1) * (y2 - y) * (z2 - z) * W(iCoord, jCoord - 1, kCoord - 1) + + (x2 - x) * (y - y1) * (z2 - z) * W(iCoord - 1, jCoord, kCoord - 1) + + (x - x1) * (y - y1) * (z2 - z) * W(iCoord, jCoord, kCoord - 1) + + (x2 - x) * (y2 - y) * (z - z1) * W(iCoord - 1, jCoord - 1, kCoord) + + (x - x1) * (y2 - y) * (z - z1) * W(iCoord, jCoord - 1, kCoord) + + (x2 - x) * (y - y1) * (z - z1) * W(iCoord - 1, jCoord, kCoord) + + (x - x1) * (y - y1) * (z - z1) * W(iCoord, jCoord, kCoord)); + + double new_z = z + particletracer->dt * w_n; + particletracer->particlePool[i].z = new_z; + + if (((new_x < 0.0) || (new_x > xlength) || (new_y < 0.0) || + (new_y > ylength) || (new_z < 0.0) || (new_z > zlength))) { + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + int i_new = new_x / dx, j_new = new_y / dy, k_new = new_z / dz; + + if (S(i_new, j_new, k_new) != FLUID) { + particletracer->particlePool[i].flag = false; + particletracer->removedParticles++; + } + } + } +} + +void freeParticles(ParticleTracer* particletracer) { free(particletracer->particlePool); } + +void writeParticles(ParticleTracer* particletracer) +{ + VtkOptions opts = { .particletracer = particletracer }; + compress(particletracer); + + char filename[50]; + snprintf(filename, 50, "vtk_files/particles%d.vtk", ts); + vtkOpenPT(&opts, filename, ts); + vtkParticle(&opts, "particle"); + vtkClose(&opts); + ++ts; +} + +void initParticleTracer(ParticleTracer* particletracer, Parameter* params) +{ + particletracer->numberOfParticles = params->numberOfParticles; + particletracer->startTime = params->startTime; + particletracer->injectTimePeriod = params->injectTimePeriod; + particletracer->writeTimePeriod = params->writeTimePeriod; + + particletracer->dt = params->dt; + particletracer->dx = params->xlength / params->imax; + particletracer->dy = params->ylength / params->jmax; + particletracer->dz = params->zlength / params->kmax; + + particletracer->xlength = params->xlength; + particletracer->ylength = params->ylength; + particletracer->zlength = params->zlength; + + particletracer->x1 = params->x1; + particletracer->y1 = params->y1; + particletracer->z1 = params->z1; + particletracer->x2 = params->x2; + particletracer->y2 = params->y2; + particletracer->z2 = params->z2; + + particletracer->lastInjectTime = params->startTime; + particletracer->lastUpdateTime = params->startTime; + particletracer->lastWriteTime = params->startTime; + + particletracer->pointer = 0; + particletracer->totalParticles = 0; + particletracer->removedParticles = 0; + + particletracer->imax = params->imax; + particletracer->jmax = params->jmax; + particletracer->kmax = params->kmax; + + particletracer->estimatedNumParticles = ((params->te - params->startTime) + 2) * + params->numberOfParticles; + + particletracer->particlePool = malloc( + sizeof(Particle) * particletracer->estimatedNumParticles); +} + +void printParticleTracerParameters(ParticleTracer* particletracer) +{ + printf("Particle Tracing data:\n"); + printf("\tNumber of particles : %d being injected for every period of %.2f\n", + particletracer->numberOfParticles, + particletracer->injectTimePeriod); + printf("\tstartTime : %.2f\n", particletracer->startTime); + printf("\t(Line along which the particles are to be injected) \n\tx1 : %.2f, y1 : " + "%.2f, z1 : %.2f, x2 : %.2f, y2 : %.2f, z2 : %.2f\n", + particletracer->x1, + particletracer->y1, + particletracer->z1, + particletracer->x2, + particletracer->y2, + particletracer->z2); + printf("\tPointer : %d, TotalParticles : %d\n", + particletracer->pointer, + particletracer->totalParticles); + printf("\tdt : %.2f, dx : %.2f, dy : %.2f, dz : %.2f\n", + particletracer->dt, + particletracer->dx, + particletracer->dy, + particletracer->dz); +} + +void trace(ParticleTracer* particletracer, + double* u, + double* v, + double* w, + double* s, + double time) +{ + if (time >= particletracer->startTime) { + // printParticles(particletracer); + if ((time - particletracer->lastInjectTime) >= particletracer->injectTimePeriod) { + injectParticles(particletracer, s); + particletracer->lastInjectTime = time; + } + if ((time - particletracer->lastWriteTime) >= particletracer->writeTimePeriod) { + writeParticles(particletracer); + particletracer->lastWriteTime = time; + } + advanceParticles(particletracer, u, v, w, s, time); + if (particletracer->removedParticles > (particletracer->totalParticles * 0.2)) { + compress(particletracer); + } + particletracer->lastUpdateTime = time; + } +} + +void compress(ParticleTracer* particletracer) +{ + Particle* memPool = particletracer->particlePool; + Particle tempPool[particletracer->totalParticles]; + int totalParticles = 0; + + for (int i = 0; i < particletracer->totalParticles; ++i) { + if (memPool[i].flag == true) { + tempPool[totalParticles].x = memPool[i].x; + tempPool[totalParticles].y = memPool[i].y; + tempPool[totalParticles].z = memPool[i].z; + tempPool[totalParticles].flag = memPool[i].flag; + ++totalParticles; + } + } + + particletracer->totalParticles = totalParticles; + particletracer->removedParticles = 0; + particletracer->pointer = totalParticles + 1; + + memcpy(particletracer->particlePool, tempPool, totalParticles * sizeof(Particle)); +} diff --git a/EnhancedSolver/3D-seq/src/particletracing.h b/EnhancedSolver/3D-seq/src/particletracing.h new file mode 100644 index 0000000..5f9eadf --- /dev/null +++ b/EnhancedSolver/3D-seq/src/particletracing.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __PARTICLETRACING_H_ +#define __PARTICLETRACING_H_ +#include "allocate.h" +#include "parameter.h" + +#include + +typedef enum COORD { X = 0, Y, NCOORD } COORD; +typedef struct { + double x, y, z; + bool flag; +} Particle; + +typedef struct { + int numberOfParticles, totalParticles; + double startTime, injectTimePeriod, writeTimePeriod, lastInjectTime, lastUpdateTime, + lastWriteTime; + + int estimatedNumParticles, activeParticles, removedParticles; + + double dx, dy, dz, dt; + Particle* particlePool; + + int pointer; + + double imax, jmax, kmax, xlength, ylength, zlength; + + double x1, y1, x2, y2, z1, z2; +} ParticleTracer; + +extern void initParticleTracer(ParticleTracer*, Parameter*); +extern void injectParticles(ParticleTracer*, double*); +extern void advanceParticles(ParticleTracer*, double*, double*, double*, double*, double); +extern void freeParticles(ParticleTracer*); +extern void writeParticles(ParticleTracer*); +extern void printParticleTracerParameters(ParticleTracer*); +extern void printParticles(ParticleTracer*); +extern void compress(ParticleTracer*); +extern void trace(ParticleTracer*, double*, double*, double*, double*, double); +#endif \ No newline at end of file diff --git a/EnhancedSolver/3D-seq/src/progress.c b/EnhancedSolver/3D-seq/src/progress.c new file mode 100644 index 0000000..3523426 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/progress.c @@ -0,0 +1,70 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include +#include + +#include "progress.h" + +static double _end; +static int _current; + +void initProgress(double end) +{ + _end = end; + _current = 0; + + printf("[ ]"); + fflush(stdout); +} + +void printProgress(double current) +{ + int new = (int)rint((current / _end) * 10.0); + + if (new > _current) { + char progress[11]; + _current = new; + progress[0] = 0; + + for (int i = 0; i < 10; i++) { + if (i < _current) { + sprintf(progress + strlen(progress), "#"); + } else { + sprintf(progress + strlen(progress), " "); + } + } + printf("\r[%s]", progress); + } + fflush(stdout); +} + +void stopProgress() +{ + printf("\n"); + fflush(stdout); +} + +FILE* initResidualWriter() +{ + FILE* fp; + fp = fopen("residual.dat", "w"); + + if (fp == NULL) { + printf("Error!\n"); + exit(EXIT_FAILURE); + } + + return fp; + +} + +void writeResidual(FILE* fp, double ts, double res) +{ + fprintf(fp, "%f, %f\n", ts, res); +} \ No newline at end of file diff --git a/BasicSolver/2D-mpi-v1/src/progress.h b/EnhancedSolver/3D-seq/src/progress.h similarity index 58% rename from BasicSolver/2D-mpi-v1/src/progress.h rename to EnhancedSolver/3D-seq/src/progress.h index 9ef2d96..240c279 100644 --- a/BasicSolver/2D-mpi-v1/src/progress.h +++ b/EnhancedSolver/3D-seq/src/progress.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -9,6 +9,7 @@ extern void initProgress(double); extern void printProgress(double); -extern void stopProgress(); - +extern void stopProgress(void); +extern FILE* initResidualWriter(void); +extern void writeResidual(FILE*, double, double); #endif diff --git a/EnhancedSolver/3D-seq/src/solver-mg.c b/EnhancedSolver/3D-seq/src/solver-mg.c new file mode 100644 index 0000000..743292c --- /dev/null +++ b/EnhancedSolver/3D-seq/src/solver-mg.c @@ -0,0 +1,304 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include + +#include "allocate.h" +#include "solver.h" +#include "util.h" + +#define FINEST_LEVEL 0 +#define COARSEST_LEVEL (s->levels - 1) +#define S(i, j, k) s[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define E(i, j, k) e[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define R(i, j, k) r[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define OLD(i, j, k) old[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +static void restrictMG(Solver* s, int level, int imax, int jmax, int kmax) +{ + double* r = s->r[level + 1]; + double* old = s->r[level]; + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; ++i) { + R(i, j, k) = (OLD(2 * i - 1, 2 * j - 1, 2 * k) + + OLD(2 * i, 2 * j - 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k) + + OLD(2 * i - 1, 2 * j, 2 * k) * 2 + + OLD(2 * i, 2 * j, 2 * k) * 8 + + OLD(2 * i + 1, 2 * j, 2 * k) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k) + + OLD(2 * i, 2 * j + 1, 2 * k) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i, 2 * j - 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k - 1) + + OLD(2 * i - 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i, 2 * j, 2 * k - 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k - 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k - 1) + + OLD(2 * i, 2 * j + 1, 2 * k - 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k - 1) + + + OLD(2 * i - 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i, 2 * j - 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j - 1, 2 * k + 1) + + OLD(2 * i - 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i, 2 * j, 2 * k + 1) * 4 + + OLD(2 * i + 1, 2 * j, 2 * k + 1) * 2 + + OLD(2 * i - 1, 2 * j + 1, 2 * k + 1) + + OLD(2 * i, 2 * j + 1, 2 * k + 1) * 2 + + OLD(2 * i + 1, 2 * j + 1, 2 * k + 1)) / + 64.0; + } + } + } +} + +static void prolongate(Solver* s, int level, int imax, int jmax, int kmax) +{ + double* old = s->r[level + 1]; + double* e = s->r[level]; + + for (int k = 2; k < kmax + 1; k += 2) { + for (int j = 2; j < jmax + 1; j += 2) { + for (int i = 2; i < imax + 1; i += 2) { + E(i, j, k) = OLD(i / 2, j / 2, k / 2); + } + } + } +} + +static void correct(Solver* s, double* p, int level, int imax, int jmax, int kmax) +{ + double* e = s->e[level]; + + for (int k = 1; k < kmax + 1; ++k) { + for (int j = 1; j < jmax + 1; ++j) { + for (int i = 1; i < imax + 1; ++i) { + P(i, j, k) += E(i, j, k); + } + } + } +} + +static void setBoundaryCondition(double* p, int imax, int jmax, int kmax) +{ + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } +} + +static double smooth( + Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax) +{ + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + P(i, j, k) -= + factor * + (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } +} + +static double calculateResidual( + Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax) +{ + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double* r = s->r[level]; + double epssq = eps * eps; + int it = 0; + int pass, ksw, jsw, isw; + double res = 1.0; + + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + R(i, + j, + k) = (RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * + idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2)); + + res += (R(i, j, k) * R(i, j, k)); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + res = res / (double)(imax * jmax * kmax); + + return res; +} + +static double multiGrid( + Solver* s, double* p, double* rhs, int level, int imax, int jmax, int kmax) +{ + double res = 0.0; + + // coarsest level + if (level == COARSEST_LEVEL) { + for (int i = 0; i < 5; i++) { + smooth(s, p, rhs, level, imax, jmax, kmax); + } + return res; + } + + // pre-smoothing + for (int i = 0; i < s->presmooth; i++) { + smooth(s, p, rhs, level, imax, jmax, kmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); + } + + res = calculateResidual(s, p, rhs, level, imax, jmax, kmax); + + // restrict + restrictMG(s, level, imax, jmax, kmax); + + // MGSolver on residual and error. + multiGrid(s, + s->e[level + 1], + s->r[level + 1], + level + 1, + imax / 2, + jmax / 2, + kmax / 2); + + // prolongate + prolongate(s, level, imax, jmax, kmax); + + // correct p on finer level using residual + correct(s, p, level, imax, jmax, kmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); + + // post-smoothing + for (int i = 0; i < s->postsmooth; i++) { + smooth(s, p, rhs, level, imax, jmax, kmax); + if (level == FINEST_LEVEL) setBoundaryCondition(p, imax, jmax, kmax); + } + + return res; +} + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->eps = p->eps; + s->omega = p->omg; + s->itermax = p->itermax; + s->levels = p->levels; + s->grid = &d->grid; + s->presmooth = p->presmooth; + s->postsmooth = p->postsmooth; + + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + int levels = s->levels; + printf("Using Multigrid solver with %d levels\n", levels); + + s->r = malloc(levels * sizeof(double*)); + s->e = malloc(levels * sizeof(double*)); + + size_t size = (imax + 2) * (jmax + 2) * (kmax + 2); + + for (int j = 0; j < levels; j++) { + s->r[j] = allocate(64, size * sizeof(double)); + s->e[j] = allocate(64, size * sizeof(double)); + + for (size_t i = 0; i < size; i++) { + s->r[j][i] = 0.0; + s->e[j][i] = 0.0; + } + } +} + +double solve(Solver* s, double* p, double* rhs) +{ + double res = multiGrid(s, p, rhs, 0, s->grid->imax, s->grid->jmax, s->grid->kmax); + +#ifdef VERBOSE + printf("Residuum: %.6f\n", res); +#endif + +return res; +} diff --git a/EnhancedSolver/3D-seq/src/solver-rb.c b/EnhancedSolver/3D-seq/src/solver-rb.c new file mode 100644 index 0000000..6332cce --- /dev/null +++ b/EnhancedSolver/3D-seq/src/solver-rb.c @@ -0,0 +1,101 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include "solver.h" +#include "util.h" + +void initSolver(Solver* s, Discretization* d, Parameter* p) +{ + s->grid = &d->grid; + s->itermax = p->itermax; + s->eps = p->eps; + s->omega = p->omg; +} + +double solve(Solver* s, double* p, double* rhs) +{ + int imax = s->grid->imax; + int jmax = s->grid->jmax; + int kmax = s->grid->kmax; + double eps = s->eps; + int itermax = s->itermax; + double dx2 = s->grid->dx * s->grid->dx; + double dy2 = s->grid->dy * s->grid->dy; + double dz2 = s->grid->dz * s->grid->dz; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double idz2 = 1.0 / dz2; + double factor = s->omega * 0.5 * (dx2 * dy2 * dz2) / + (dy2 * dz2 + dx2 * dz2 + dx2 * dy2); + double epssq = eps * eps; + int it = 0; + double res = 1.0; + int pass, ksw, jsw, isw; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + ksw = 1; + + for (pass = 0; pass < 2; pass++) { + jsw = ksw; + + for (int k = 1; k < kmax + 1; k++) { + isw = jsw; + for (int j = 1; j < jmax + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + double r = + RHS(i, j, k) - + ((P(i + 1, j, k) - 2.0 * P(i, j, k) + P(i - 1, j, k)) * idx2 + + (P(i, j + 1, k) - 2.0 * P(i, j, k) + P(i, j - 1, k)) * + idy2 + + (P(i, j, k + 1) - 2.0 * P(i, j, k) + P(i, j, k - 1)) * + idz2); + + P(i, j, k) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + ksw = 3 - ksw; + } + + for (int j = 1; j < jmax + 1; j++) { + for (int i = 1; i < imax + 1; i++) { + P(i, j, 0) = P(i, j, 1); + P(i, j, kmax + 1) = P(i, j, kmax); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0, k) = P(i, 1, k); + P(i, jmax + 1, k) = P(i, jmax, k); + } + } + + for (int k = 1; k < kmax + 1; k++) { + for (int j = 1; j < jmax + 1; j++) { + P(0, j, k) = P(1, j, k); + P(imax + 1, j, k) = P(imax, j, k); + } + } + + res = res / (double)(imax * jmax * kmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + +#ifdef VERBOSE + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +#endif + +return res; +} diff --git a/EnhancedSolver/3D-seq/src/solver.h b/EnhancedSolver/3D-seq/src/solver.h new file mode 100644 index 0000000..e5f1b38 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/solver.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __SOLVER_H_ +#define __SOLVER_H_ +#include "discretization.h" +#include "grid.h" +#include "parameter.h" + +typedef struct { + /* geometry and grid information */ + Grid* grid; + /* parameters */ + double eps, omega, rho; + int itermax; + int levels; + double **r, **e; + int presmooth, postsmooth; +} Solver; + +extern void initSolver(Solver*, Discretization*, Parameter*); +extern double solve(Solver*, double*, double*); + +#endif diff --git a/BasicSolver/2D-seq-pt/src/timing.c b/EnhancedSolver/3D-seq/src/timing.c similarity index 72% rename from BasicSolver/2D-seq-pt/src/timing.c rename to EnhancedSolver/3D-seq/src/timing.c index c4025a4..78b01c4 100644 --- a/BasicSolver/2D-seq-pt/src/timing.c +++ b/EnhancedSolver/3D-seq/src/timing.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,18 +7,16 @@ #include #include -double getTimeStamp() +double getTimeStamp(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } -double getTimeResolution() +double getTimeResolution(void) { struct timespec ts; clock_getres(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9; } - -double getTimeStamp_() { return getTimeStamp(); } diff --git a/BasicSolver/2D-seq-pt/src/timing.h b/EnhancedSolver/3D-seq/src/timing.h similarity index 55% rename from BasicSolver/2D-seq-pt/src/timing.h rename to EnhancedSolver/3D-seq/src/timing.h index db95329..ed05a8c 100644 --- a/BasicSolver/2D-seq-pt/src/timing.h +++ b/EnhancedSolver/3D-seq/src/timing.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. * All rights reserved. * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. @@ -7,8 +7,7 @@ #ifndef __TIMING_H_ #define __TIMING_H_ -extern double getTimeStamp(); -extern double getTimeResolution(); -extern double getTimeStamp_(); +extern double getTimeStamp(void); +extern double getTimeResolution(void); #endif // __TIMING_H_ diff --git a/EnhancedSolver/3D-seq/src/util.h b/EnhancedSolver/3D-seq/src/util.h new file mode 100644 index 0000000..f2a6eab --- /dev/null +++ b/EnhancedSolver/3D-seq/src/util.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. + * Use of this source code is governed by a MIT-style + * license that can be found in the LICENSE file. + */ +#ifndef __UTIL_H_ +#define __UTIL_H_ +#define HLINE \ + "----------------------------------------------------------------------------\n" + +#ifndef MIN +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif +#ifndef ABS +#define ABS(a) ((a) >= 0 ? (a) : -(a)) +#endif + +#define P(i, j, k) p[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define F(i, j, k) f[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define G(i, j, k) g[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define H(i, j, k) h[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define U(i, j, k) u[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define V(i, j, k) v[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define W(i, j, k) w[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define S(i, j, k) s[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] +#define RHS(i, j, k) rhs[(k) * (imax + 2) * (jmax + 2) + (j) * (imax + 2) + (i)] + +#endif // __UTIL_H_ diff --git a/EnhancedSolver/3D-seq/src/vtkWriter.c b/EnhancedSolver/3D-seq/src/vtkWriter.c new file mode 100644 index 0000000..6c6140e --- /dev/null +++ b/EnhancedSolver/3D-seq/src/vtkWriter.c @@ -0,0 +1,198 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#include +#include +#include + +#include "vtkWriter.h" +#define G(v, i, j, k) v[(k)*imax * jmax + (j)*imax + (i)] + +static float floatSwap(float f) +{ + union { + float f; + char b[4]; + } dat1, dat2; + + dat1.f = f; + dat2.b[0] = dat1.b[3]; + dat2.b[1] = dat1.b[2]; + dat2.b[2] = dat1.b[1]; + dat2.b[3] = dat1.b[0]; + return dat2.f; +} + +static void writeHeader(VtkOptions* o) +{ + fprintf(o->fh, "# vtk DataFile Version 3.0\n"); + fprintf(o->fh, "PAMPI cfd solver output\n"); + if (o->fmt == ASCII) { + fprintf(o->fh, "ASCII\n"); + } else if (o->fmt == BINARY) { + fprintf(o->fh, "BINARY\n"); + } + + fprintf(o->fh, "DATASET STRUCTURED_POINTS\n"); + fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid.imax, o->grid.jmax, o->grid.kmax); + fprintf(o->fh, + "ORIGIN %f %f %f\n", + o->grid.dx * 0.5, + o->grid.dy * 0.5, + o->grid.dz * 0.5); + fprintf(o->fh, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz); + fprintf(o->fh, "POINT_DATA %d\n", o->grid.imax * o->grid.jmax * o->grid.kmax); +} + +void vtkOpen(VtkOptions* o, char* problem) +{ + char filename[50]; + snprintf(filename, 50, "%s.vtk", problem); + o->fh = fopen(filename, "w"); + writeHeader(o); + + printf("Writing VTK output for %s\n", problem); +} + +void vtkScalar(VtkOptions* o, char* name, double* s) +{ + int imax = o->grid.imax; + int jmax = o->grid.jmax; + int kmax = o->grid.kmax; + + printf("Register scalar %s\n", name); + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + fprintf(o->fh, "SCALARS %s float\n", name); + fprintf(o->fh, "LOOKUP_TABLE default\n"); + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, "%f\n", G(s, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((float[1]) { floatSwap(G(s, i, j, k)) }, + sizeof(float), + 1, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +void vtkVector(VtkOptions* o, char* name, VtkVector vec) +{ + int imax = o->grid.imax; + int jmax = o->grid.jmax; + int kmax = o->grid.kmax; + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + + fprintf(o->fh, "VECTORS %s float\n", name); + + for (int k = 0; k < kmax; k++) { + for (int j = 0; j < jmax; j++) { + for (int i = 0; i < imax; i++) { + if (o->fmt == ASCII) { + fprintf(o->fh, + "%f %f %f\n", + G(vec.u, i, j, k), + G(vec.v, i, j, k), + G(vec.w, i, j, k)); + } else if (o->fmt == BINARY) { + fwrite((float[3]) { floatSwap(G(vec.u, i, j, k)), + floatSwap(G(vec.v, i, j, k)), + floatSwap(G(vec.w, i, j, k)) }, + sizeof(float), + 3, + o->fh); + } + } + } + } + if (o->fmt == BINARY) fprintf(o->fh, "\n"); +} + +void vtkClose(VtkOptions* o) +{ + fclose(o->fh); + o->fh = NULL; +} + +static void writeHeaderPT(VtkOptions* o, int ts) +{ + fprintf(o->fh, "# vtk DataFile Version 3.0\n"); + fprintf(o->fh, "PAMPI cfd solver particle tracing file\n"); + if (o->fmt == ASCII) { + fprintf(o->fh, "ASCII\n"); + } else if (o->fmt == BINARY) { + fprintf(o->fh, "BINARY\n"); + } + + fprintf(o->fh, "DATASET UNSTRUCTURED_GRID\n"); + fprintf(o->fh, "FIELD FieldData 2\n"); + fprintf(o->fh, "TIME 1 1 double\n"); + fprintf(o->fh, "%d\n", ts); + fprintf(o->fh, "CYCLE 1 1 int\n"); + fprintf(o->fh, "1\n"); +} + +void vtkOpenPT(VtkOptions* o, char* problem, int ts) +{ + o->fh = fopen(problem, "w"); + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + + writeHeaderPT(o, ts); + + // printf("Writing VTK output for %s\n", problem); +} + +void vtkParticle(VtkOptions* o, char* name) +{ + Particle* particlePool = o->particletracer->particlePool; + + if (o->fh == NULL) { + printf("vtkWriter not initialize! Call vtkOpen first!\n"); + exit(EXIT_FAILURE); + } + + fprintf(o->fh, "POINTS %d float\n", o->particletracer->totalParticles); + + for (int i = 0; i < o->particletracer->totalParticles; ++i) { + double x = particlePool[i].x; + double y = particlePool[i].y; + double z = particlePool[i].z; + fprintf(o->fh, "%.2f %.2f %.2f\n", x, y, z); + } + + fprintf(o->fh, + "CELLS %d %d\n", + o->particletracer->totalParticles, + 2 * o->particletracer->totalParticles); + + for (int i = 0; i < o->particletracer->totalParticles; ++i) { + fprintf(o->fh, "1 %d\n", i); + } + + fprintf(o->fh, "CELL_TYPES %d\n", o->particletracer->totalParticles); + + for (int i = 0; i < o->particletracer->totalParticles; ++i) { + fprintf(o->fh, "1\n"); + } +} diff --git a/EnhancedSolver/3D-seq/src/vtkWriter.h b/EnhancedSolver/3D-seq/src/vtkWriter.h new file mode 100644 index 0000000..2bc9503 --- /dev/null +++ b/EnhancedSolver/3D-seq/src/vtkWriter.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) NHR@FAU, University Erlangen-Nuremberg. + * All rights reserved. This file is part of nusif-solver. + * Use of this source code is governed by a MIT style + * license that can be found in the LICENSE file. + */ +#ifndef __VTKWRITER_H_ +#define __VTKWRITER_H_ +#include + +#include "grid.h" +#include "particletracing.h" + +typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat; + +typedef struct VtkOptions { + VtkFormat fmt; + Grid grid; + FILE* fh; + ParticleTracer* particletracer; +} VtkOptions; + +typedef struct VtkVector { + double *u, *v, *w; +} VtkVector; + +extern void vtkOpen(VtkOptions* opts, char* filename); +extern void vtkVector(VtkOptions* opts, char* name, VtkVector vec); +extern void vtkScalar(VtkOptions* opts, char* name, double* p); +extern void vtkClose(VtkOptions* opts); + +extern void vtkOpenPT(VtkOptions* opts, char* filename, int ts); +extern void vtkParticle(VtkOptions* opts, char* name); +#endif // __VTKWRITER_H_ diff --git a/PoissonSolver/2D-mpi/README.md b/PoissonSolver/2D-mpi/README.md index b0a80a6..2a898e3 100644 --- a/PoissonSolver/2D-mpi/README.md +++ b/PoissonSolver/2D-mpi/README.md @@ -2,7 +2,7 @@ ## Build -1. Configure the toolchain and additional options in `config.mk`: +1. Configure the tool chain and additional options in `config.mk`: ``` # Supported: GCC, CLANG, ICC TAG ?= GCC @@ -22,7 +22,7 @@ The verbosity options enable detailed output about affinity settings, allocation make ``` -You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. +You can build multiple tool chains in the same directory, but notice that the Makefile is only acting on the one currently set. Intermediate build results are located in the `` directory. To output the executed commands use: diff --git a/PoissonSolver/2D-mpi/src/main.c b/PoissonSolver/2D-mpi/src/main.c index c797be3..981cac2 100644 --- a/PoissonSolver/2D-mpi/src/main.c +++ b/PoissonSolver/2D-mpi/src/main.c @@ -4,18 +4,17 @@ * Use of this source code is governed by a MIT-style * license that can be found in the LICENSE file. */ -#include -#include -#include -#include #include +#include #include +#include +#include +#include #include "parameter.h" #include "solver.h" - -int main (int argc, char** argv) +int main(int argc, char** argv) { int rank; Parameter params; @@ -25,13 +24,13 @@ int main (int argc, char** argv) MPI_Comm_rank(MPI_COMM_WORLD, &rank); initParameter(¶ms); - if ( argc != 2 ) { - printf("Usage: %s \n",argv[0]); + if (argc != 2) { + printf("Usage: %s \n", argv[0]); exit(EXIT_SUCCESS); } readParameter(¶ms, argv[1]); - if ( rank == 0 ) { + if (rank == 0) { printParameter(¶ms); } diff --git a/PoissonSolver/2D-mpi/src/parameter.c b/PoissonSolver/2D-mpi/src/parameter.c index 6c08af9..0a4b455 100644 --- a/PoissonSolver/2D-mpi/src/parameter.c +++ b/PoissonSolver/2D-mpi/src/parameter.c @@ -12,41 +12,47 @@ #include "util.h" #define MAXLINE 4096 -void initParameter(Parameter *param) { +void initParameter(Parameter* param) +{ param->xlength = 1.0; param->ylength = 1.0; - param->imax = 100; - param->jmax = 100; + param->imax = 100; + param->jmax = 100; param->itermax = 1000; - param->eps = 0.0001; - param->omg = 1.8; + param->eps = 0.0001; + param->omg = 1.8; } -void readParameter(Parameter *param, const char *filename) { - FILE *fp = fopen(filename, "r"); +void readParameter(Parameter* param, const char* filename) +{ + FILE* fp = fopen(filename, "r"); char line[MAXLINE]; int i; - if(!fp) { + if (!fp) { fprintf(stderr, "Could not open parameter file: %s\n", filename); exit(EXIT_FAILURE); } - while(!feof(fp)) { + while (!feof(fp)) { line[0] = '\0'; fgets(line, MAXLINE, fp); - for(i = 0; line[i] != '\0' && line[i] != '#'; i++); + for (i = 0; line[i] != '\0' && line[i] != '#'; i++) + ; line[i] = '\0'; - char *tok = strtok(line, " "); - char *val = strtok(NULL, " "); + char* tok = strtok(line, " "); + char* val = strtok(NULL, " "); - #define PARSE_PARAM(p,f) if(strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { param->p = f(val); } - #define PARSE_STRING(p) PARSE_PARAM(p, strdup) - #define PARSE_INT(p) PARSE_PARAM(p, atoi) - #define PARSE_REAL(p) PARSE_PARAM(p, atof) +#define PARSE_PARAM(p, f) \ + if (strncmp(tok, #p, sizeof(#p) / sizeof(#p[0]) - 1) == 0) { \ + param->p = f(val); \ + } +#define PARSE_STRING(p) PARSE_PARAM(p, strdup) +#define PARSE_INT(p) PARSE_PARAM(p, atoi) +#define PARSE_REAL(p) PARSE_PARAM(p, atof) - if(tok != NULL && val != NULL) { + if (tok != NULL && val != NULL) { PARSE_REAL(xlength); PARSE_REAL(ylength); PARSE_INT(imax); @@ -60,7 +66,8 @@ void readParameter(Parameter *param, const char *filename) { fclose(fp); } -void printParameter(Parameter *param) { +void printParameter(Parameter* param) +{ printf("Parameters:\n"); printf("Geometry data:\n"); printf("\tDomain box size (x, y): %e, %e\n", param->xlength, param->ylength); diff --git a/PoissonSolver/2D-mpi/src/parameter.h b/PoissonSolver/2D-mpi/src/parameter.h index d57ce19..6344a31 100644 --- a/PoissonSolver/2D-mpi/src/parameter.h +++ b/PoissonSolver/2D-mpi/src/parameter.h @@ -8,7 +8,7 @@ #define __PARAMETER_H_ typedef struct { - double xlength, ylength; + double xlength, ylength; int imax, jmax; int itermax; double eps, omg, gamma; diff --git a/PoissonSolver/2D-mpi/src/solver.c b/PoissonSolver/2D-mpi/src/solver.c index 7264c44..62f135e 100644 --- a/PoissonSolver/2D-mpi/src/solver.c +++ b/PoissonSolver/2D-mpi/src/solver.c @@ -4,49 +4,53 @@ * Use of this source code is governed by a MIT style * license that can be found in the LICENSE file. */ -#include -#include #include #include +#include +#include -#include "solver.h" -#include "parameter.h" #include "allocate.h" +#include "parameter.h" +#include "solver.h" -#define PI 3.14159265358979323846 -#define P(i,j) p[(j)*(imax+2) + (i)] -#define RHS(i,j) rhs[(j)*(imax+2) + (i)] +#define PI 3.14159265358979323846 +#define P(i, j) p[(j) * (imax + 2) + (i)] +#define RHS(i, j) rhs[(j) * (imax + 2) + (i)] static int sizeOfRank(int rank, int size, int N) { - return N/size + ((N%size>rank) ? 1 : 0); + return N / size + ((N % size > rank) ? 1 : 0); } static void print(Solver* solver) { double* p = solver->p; - int imax = solver->imax; + int imax = solver->imax; - printf("### RANK %d #######################################################\n", solver->rank); - for( int j=0; j < solver->jmaxLocal+2; j++ ) { + printf("### RANK %d #######################################################\n", + solver->rank); + for (int j = 0; j < solver->jmaxLocal + 2; j++) { printf("%02d: ", j); - for( int i=0; i < solver->imax+2; i++ ) { + for (int i = 0; i < solver->imax + 2; i++) { printf("%12.8f ", P(i, j)); } printf("\n"); } - fflush( stdout ); + fflush(stdout); } static void exchange(Solver* solver) { - MPI_Request requests[4] = { MPI_REQUEST_NULL, MPI_REQUEST_NULL, MPI_REQUEST_NULL, MPI_REQUEST_NULL }; + MPI_Request requests[4] = { MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL, + MPI_REQUEST_NULL }; /* exchange ghost cells with top neighbor */ if (solver->rank + 1 < solver->size) { - int top = solver->rank + 1; - double* src = solver->p + (solver->jmaxLocal) * (solver->imax+2) + 1; - double* dst = solver->p + (solver->jmaxLocal+1) * (solver->imax+2) + 1; + int top = solver->rank + 1; + double* src = solver->p + (solver->jmaxLocal) * (solver->imax + 2) + 1; + double* dst = solver->p + (solver->jmaxLocal + 1) * (solver->imax + 2) + 1; MPI_Isend(src, solver->imax, MPI_DOUBLE, top, 1, MPI_COMM_WORLD, &requests[0]); MPI_Irecv(dst, solver->imax, MPI_DOUBLE, top, 2, MPI_COMM_WORLD, &requests[1]); @@ -54,222 +58,382 @@ static void exchange(Solver* solver) /* exchange ghost cells with bottom neighbor */ if (solver->rank > 0) { - int bottom = solver->rank - 1; - double* src = solver->p + (solver->imax+2) + 1; + int bottom = solver->rank - 1; + double* src = solver->p + (solver->imax + 2) + 1; double* dst = solver->p + 1; - MPI_Isend(src, solver->imax, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]); - MPI_Irecv(dst, solver->imax, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]); + MPI_Isend(src, solver->imax, MPI_DOUBLE, bottom, 2, MPI_COMM_WORLD, &requests[2]); + MPI_Irecv(dst, solver->imax, MPI_DOUBLE, bottom, 1, MPI_COMM_WORLD, &requests[3]); } MPI_Waitall(4, requests, MPI_STATUSES_IGNORE); } -void getResult(Solver *solver) +void getResult(Solver* solver) { double* Pall = NULL; int *rcvCounts, *displs; - if ( solver->rank == 0 ) { - Pall = allocate(64, (solver->imax+2) * (solver->jmax+2) * sizeof(double)); - rcvCounts = (int*) malloc(solver->size * sizeof(int)); - displs = (int*) malloc(solver->size * sizeof(int)); - rcvCounts[0] = solver->jmaxLocal * (solver->imax+2); - displs[0] = 0; - int cursor = rcvCounts[0]; + if (solver->rank == 0) { + Pall = allocate(64, (solver->imax + 2) * (solver->jmax + 2) * sizeof(double)); + rcvCounts = (int*)malloc(solver->size * sizeof(int)); + displs = (int*)malloc(solver->size * sizeof(int)); + rcvCounts[0] = solver->jmaxLocal * (solver->imax + 2); + displs[0] = 0; + int cursor = rcvCounts[0]; - for ( int i=1; i < solver->size; i++ ) { - rcvCounts[i] = sizeOfRank(i, solver->size, solver->jmax) * (solver->imax+2); - displs[i] = cursor; + for (int i = 1; i < solver->size; i++) { + rcvCounts[i] = sizeOfRank(i, solver->size, solver->jmax) * (solver->imax + 2); + displs[i] = cursor; cursor += rcvCounts[i]; } } - int cnt = solver->jmaxLocal*(solver->imax+2); - double* sendbuffer = solver->p + (solver->imax+2); - MPI_Gatherv(sendbuffer, cnt, MPI_DOUBLE, Pall, - rcvCounts, displs, MPI_DOUBLE, 0, MPI_COMM_WORLD); + int cnt = solver->jmaxLocal * (solver->imax + 2); + double* sendbuffer = solver->p + (solver->imax + 2); + MPI_Gatherv(sendbuffer, + cnt, + MPI_DOUBLE, + Pall, + rcvCounts, + displs, + MPI_DOUBLE, + 0, + MPI_COMM_WORLD); - if ( solver->rank == 0 ) { + if (solver->rank == 0) { writeResult(solver, Pall, "p.dat"); } } -void initSolver(Solver *solver, Parameter *params, int problem) +void initSolver(Solver* solver, Parameter* params, int problem) { MPI_Comm_rank(MPI_COMM_WORLD, &(solver->rank)); MPI_Comm_size(MPI_COMM_WORLD, &(solver->size)); - solver->imax = params->imax; - solver->jmax = params->jmax; + solver->imax = params->imax; + solver->jmax = params->jmax; solver->jmaxLocal = sizeOfRank(solver->rank, solver->size, solver->jmax); - printf("RANK %d: %d\n", solver->rank, solver->jmaxLocal); + printf("RANK %d: imaxLocal : %d, jmaxLocal : %d\n", + solver->rank, + solver->imax, + solver->jmaxLocal); - solver->dx = params->xlength/params->imax; - solver->dy = params->ylength/params->jmax; - solver->ys = solver->rank * solver->jmaxLocal * solver->dy; - solver->eps = params->eps; - solver->omega = params->omg; + solver->dx = params->xlength / params->imax; + solver->dy = params->ylength / params->jmax; + solver->ys = solver->rank * solver->jmaxLocal * solver->dy; + solver->eps = params->eps; + solver->omega = params->omg; solver->itermax = params->itermax; - int imax = solver->imax; - int jmax = solver->jmax; + int imax = solver->imax; + int jmax = solver->jmax; int jmaxLocal = solver->jmaxLocal; - solver->p = allocate(64, (imax+2) * (jmaxLocal+2) * sizeof(double)); - solver->rhs = allocate(64, (imax+2) * (jmax+2) * sizeof(double)); + solver->p = allocate(64, (imax + 2) * (jmaxLocal + 2) * sizeof(double)); + solver->rhs = allocate(64, (imax + 2) * (jmax + 2) * sizeof(double)); - double dx = solver->dx; - double dy = solver->dy; - double* p = solver->p; + double dx = solver->dx; + double dy = solver->dy; + double* p = solver->p; double* rhs = solver->rhs; - for( int j=0; jys + j * dy; - for( int i=0; iimax; - int rank = solver->rank; + int imax = solver->imax; + int rank = solver->rank; double* p = solver->p; -/* for( int j=0; j < solver->jmaxLocal+2; j++ ) { */ -/* for( int i=0; i < solver->imax+2; i++ ) { */ -/* P(i, j) = (double) rank; */ -/* } */ -/* } */ + /* for( int j=0; j < solver->jmaxLocal+2; j++ ) { */ + /* for( int i=0; i < solver->imax+2; i++ ) { */ + /* P(i, j) = (double) rank; */ + /* } */ + /* } */ -/* for ( int i=0; i < solver->size; i++) { */ -/* if ( i == rank ) { */ -/* print(solver); */ -/* } */ -/* MPI_Barrier(MPI_COMM_WORLD); */ -/* } */ + /* for ( int i=0; i < solver->size; i++) { */ + /* if ( i == rank ) { */ + /* print(solver); */ + /* } */ + /* MPI_Barrier(MPI_COMM_WORLD); */ + /* } */ -/* if ( rank == 0 ) { */ -/* printf("##########################################################\n"); */ -/* printf("## Exchange ghost layers\n"); */ -/* printf("##########################################################\n"); */ -/* } */ -/* exchange(solver); */ + /* if ( rank == 0 ) { */ + /* printf("##########################################################\n"); */ + /* printf("## Exchange ghost layers\n"); */ + /* printf("##########################################################\n"); */ + /* } */ + /* exchange(solver); */ - for ( int i=0; i < solver->size; i++) { - if ( i == rank ) { - print(solver); + for (int i = 0; i < solver->size; i++) { + if (i == rank) { + print(solver); } MPI_Barrier(MPI_COMM_WORLD); } } -int solve(Solver *solver) +int solve(Solver* solver) { double r; int it = 0; - double res; + double res, res1; - int imax = solver->imax; - int jmax = solver->jmax; + int imax = solver->imax; + int jmax = solver->jmax; int jmaxLocal = solver->jmaxLocal; - double eps= solver->eps; - double omega = solver->omega; - int itermax = solver->itermax; + double eps = solver->eps; + double omega = solver->omega; + int itermax = solver->itermax; - double dx2 = solver->dx * solver->dx; - double dy2 = solver->dy * solver->dy; - double idx2 = 1.0/dx2; - double idy2 = 1.0/dy2; - double factor = omega * 0.5 * (dx2*dy2) / (dx2+dy2); - double* p = solver->p; - double* rhs = solver->rhs; + double dx2 = solver->dx * solver->dx; + double dy2 = solver->dy * solver->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* p = solver->p; + double* rhs = solver->rhs; + double epssq = eps * eps; res = eps + 1.0; - while((res >= eps) && (it < itermax)) { + while ((res >= epssq) && (it < itermax)) { res = 0.0; exchange(solver); - for( int j=1; jrank == 0 ) { - for( int i=1; irank == 0) { + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); } } - if ( solver->rank == (solver->size-1) ) { - for( int i=1; irank == (solver->size - 1)) { + for (int i = 1; i < imax + 1; i++) { + P(i, jmaxLocal + 1) = P(i, jmaxLocal); } } - for( int j=1; jrank == 0 ) { - printf("%d Residuum: %e\n",it, res); + if (solver->rank == 0) { + printf("%d Residuum: %e\n", it, res1); } #endif it++; } - if ( solver->rank == 0 ) { - printf("Solver took %d iterations\n",it); + if (solver->rank == 0) { + printf("Solver took %d iterations\n", it); } - if( res < eps ){ + if (res < eps) { return 1; - } else{ + } else { return 0; } } +int solveRB(Solver* solver) +{ + double r; + int it = 0; + double res, res1; + + int imax = solver->imax; + int jmax = solver->jmax; + int jmaxLocal = solver->jmaxLocal; + double eps = solver->eps; + double omega = solver->omega; + int itermax = solver->itermax; + + double dx2 = solver->dx * solver->dx; + double dy2 = solver->dy * solver->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* p = solver->p; + double* rhs = solver->rhs; + int pass, jsw, isw; + double epssq = eps * eps; + + res = eps + 1.0; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + exchange(solver); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + } + + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } + MPI_Allreduce(&res, &res1, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + res = res1; + res = res / (double)(imax * jmax); +#ifdef DEBUG + printf("%d Residuum: %e\n", it, res); +#endif + it++; + } + + if (solver->rank == 0) { + printf("Solver took %d iterations\n", it); + } + if (res < eps) { + return 1; + } else { + return 0; + } +} + +int solveRBA(Solver* solver) +{ + double r; + int it = 0; + double res; + + int imax = solver->imax; + int jmax = solver->jmax; + int jmaxLocal = solver->jmaxLocal; + double eps = solver->eps; + double omega = solver->omega; + int itermax = solver->itermax; + + double dx2 = solver->dx * solver->dx; + double dy2 = solver->dy * solver->dy; + double idx2 = 1.0 / dx2; + double idy2 = 1.0 / dy2; + double factor = omega * 0.5 * (dx2 * dy2) / (dx2 + dy2); + double* p = solver->p; + double* rhs = solver->rhs; + int pass, jsw, isw; + double rho = solver->rho; + double epssq = eps * eps; + + res = eps + 1.0; + + while ((res >= epssq) && (it < itermax)) { + res = 0.0; + jsw = 1; + + for (pass = 0; pass < 2; pass++) { + isw = jsw; + exchange(solver); + + for (int j = 1; j < jmaxLocal + 1; j++) { + for (int i = isw; i < imax + 1; i += 2) { + + double r = RHS(i, j) - + ((P(i + 1, j) - 2.0 * P(i, j) + P(i - 1, j)) * idx2 + + (P(i, j + 1) - 2.0 * P(i, j) + P(i, j - 1)) * idy2); + + P(i, j) -= (omega * factor * r); + res += (r * r); + } + isw = 3 - isw; + } + jsw = 3 - jsw; + omega = (it == 0 && pass == 0 ? 1.0 / (1.0 - 0.5 * rho * rho) + : 1.0 / (1.0 - 0.25 * rho * rho * omega)); + } + + for (int i = 1; i < imax + 1; i++) { + P(i, 0) = P(i, 1); + P(i, jmaxLocal + 1) = P(i, jmaxLocal); + } + + for (int j = 1; j < jmaxLocal + 1; j++) { + P(0, j) = P(1, j); + P(imax + 1, j) = P(imax, j); + } + + res = res / (double)(imax * jmax); +#ifdef DEBUG + printf("%d Residuum: %e Omega: %e\n", it, res, omega); +#endif + it++; + } + + printf("Final omega: %f\n", omega); + printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); +} + void writeResult(Solver* solver, double* m, char* filename) { - int imax = solver->imax; - int jmax = solver->jmax; + int imax = solver->imax; + int jmax = solver->jmax; double* p = solver->p; - FILE *fp; - fp= fopen(filename, "w"); + FILE* fp; + fp = fopen(filename, "w"); - if (fp== NULL) { + if (fp == NULL) { printf("Error!\n"); exit(EXIT_FAILURE); } - for( int j=0; jomega); } void solveRB(Solver* solver) @@ -168,7 +169,8 @@ void solveRB(Solver* solver) it++; } - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + // printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + printf("%d, %f\n", it, solver->omega); } void solveRBA(Solver* solver) @@ -232,8 +234,9 @@ void solveRBA(Solver* solver) it++; } - printf("Final omega: %f\n", omega); - printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + // printf("Final omega: %f\n", omega); + // printf("Solver took %d iterations to reach %f\n", it, sqrt(res)); + printf("%d, %f\n", it, omega); } void writeResult(Solver* solver)