NuSiF-Solver/BasicSolver/2D-mpi/src/comm.c

/*
 * Copyright (C)  NHR@FAU, University Erlangen-Nuremberg.
 * All rights reserved. This file is part of nusif-solver.
 * Use of this source code is governed by a MIT style
 * license that can be found in the LICENSE file.
 */
#include <stdio.h>
#include <stdlib.h>

#include "comm.h"

// subroutines local to this module
int sizeOfRank(int rank, int size, int N)
{
    return N / size + ((N % size > rank) ? 1 : 0);
}

void commReduction(double* v, int op)
{
#ifdef _MPI
    if (op == MAX) {
        MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
    } else if (op == SUM) {
        MPI_Allreduce(MPI_IN_PLACE, v, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
    }
#endif
}

void commPrintConfig(Comm* c)
{
#ifdef _MPI
    fflush(stdout);
    MPI_Barrier(MPI_COMM_WORLD);
    if (commIsMaster(c)) {
        printf("Communication setup:\n");
    }

    for (int i = 0; i < c->size; i++) {
        if (i == c->rank) {
            printf("\tRank %d of %d\n", c->rank, c->size);
            printf("\tNeighbours (bottom, top, left, right): %d %d, %d, %d\n",
                c->neighbours[BOTTOM],
                c->neighbours[TOP],
                c->neighbours[LEFT],
                c->neighbours[RIGHT]);
            printf("\tCoordinates (i,j) %d %d\n", c->coords[IDIM], c->coords[JDIM]);
            printf("\tLocal domain size (i,j) %dx%d\n", c->imaxLocal, c->jmaxLocal);
            fflush(stdout);
        }
    }
    MPI_Barrier(MPI_COMM_WORLD);
#endif
}

void commInit(Comm* c, int argc, char** argv)
{
#ifdef _MPI
    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &(c->rank));
    MPI_Comm_size(MPI_COMM_WORLD, &(c->size));
#else
    c->rank = 0;
    c->size = 1;
#endif
}

void commTestInit(Comm* c, double* p, double* f, double* g)
{
    int imax = c->imaxLocal;
    int jmax = c->jmaxLocal;
    int rank = c->rank;

    for (int j = 0; j < jmax + 2; j++) {
        for (int i = 0; i < imax + 2; i++) {
            p[j * (imax + 2) + i] = rank;
            f[j * (imax + 2) + i] = rank;
            g[j * (imax + 2) + i] = rank;
        }
    }
}

static void testWriteFile(char* filename, double* grid, int imax, int jmax)
{
    FILE* fp = fopen(filename, "w");

    if (fp == NULL) {
        printf("Error!\n");
        exit(EXIT_FAILURE);
    }

    for (int j = 0; j < jmax + 2; j++) {
        for (int i = 0; i < imax + 2; i++) {
            fprintf(fp, "%f ", grid[j * (imax + 2) + i]);
        }
        fprintf(fp, "\n");
    }

    fclose(fp);
}

void commTestWrite(Comm* c, double* p, double* f, double* g)
{
    int imax = c->imaxLocal;
    int jmax = c->jmaxLocal;
    int rank = c->rank;

    char filename[30];
    snprintf(filename, 30, "ptest-%d.dat", rank);
    testWriteFile(filename, p, imax, jmax);

    snprintf(filename, 30, "ftest-%d.dat", rank);
    testWriteFile(filename, f, imax, jmax);

    snprintf(filename, 30, "gtest-%d.dat", rank);
    testWriteFile(filename, g, imax, jmax);
}

void commFinalize(Comm* c)
{
#ifdef _MPI
    MPI_Finalize();
#endif
}