From 394e4643ec3eb5786300bb7325dde5dc9ac202b7 Mon Sep 17 00:00:00 2001
From: Jan Eitzinger <jan.treibig@gmail.com>
Date: Sun, 5 Feb 2023 11:22:05 +0100
Subject: [PATCH] Start MPI-IO VTK Writer

---
 BasicSolver/3D-mpi-io/src/vtkWriter.c | 59 +++++++++++++--------------
 BasicSolver/3D-mpi-io/src/vtkWriter.h |  8 +---
 2 files changed, 31 insertions(+), 36 deletions(-)

diff --git a/BasicSolver/3D-mpi-io/src/vtkWriter.c b/BasicSolver/3D-mpi-io/src/vtkWriter.c
index 5b4d36b..d7aaff7 100644
--- a/BasicSolver/3D-mpi-io/src/vtkWriter.c
+++ b/BasicSolver/3D-mpi-io/src/vtkWriter.c
@@ -6,8 +6,10 @@
  */
 #include <stdbool.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
+#include "mpi.h"
 #include "vtkWriter.h"
 #define G(v, i, j, k) v[(k)*imax * jmax + (j)*imax + (i)]
 
@@ -28,39 +30,45 @@ static float floatSwap(float f)
 
 static void writeHeader(VtkOptions* o)
 {
-    fprintf(o->fh, "# vtk DataFile Version 3.0\n");
-    fprintf(o->fh, "PAMPI cfd solver output\n");
-    if (o->fmt == ASCII) {
-        fprintf(o->fh, "ASCII\n");
-    } else if (o->fmt == BINARY) {
-        fprintf(o->fh, "BINARY\n");
-    }
+    const size_t MAX_HEADER = 200;
 
-    fprintf(o->fh, "DATASET STRUCTURED_POINTS\n");
-    fprintf(o->fh, "DIMENSIONS %d %d %d\n", o->grid.imax, o->grid.jmax, o->grid.kmax);
-    fprintf(o->fh,
+    char* header = (char*)malloc(MAX_HEADER);
+    char* cursor = header;
+
+    cursor += sprintf(cursor, "# vtk DataFile Version 3.0\n");
+    cursor += sprintf(cursor, "PAMPI cfd solver output\n");
+    cursor += sprintf(cursor, "BINARY\n");
+
+    cursor += sprintf(cursor, "DATASET STRUCTURED_POINTS\n");
+    cursor += sprintf(cursor,
+        "DIMENSIONS %d %d %d\n",
+        o->grid.imax,
+        o->grid.jmax,
+        o->grid.kmax);
+    cursor += sprintf(cursor,
         "ORIGIN %f %f %f\n",
         o->grid.dx * 0.5,
         o->grid.dy * 0.5,
         o->grid.dz * 0.5);
-    fprintf(o->fh, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz);
-    fprintf(o->fh, "POINT_DATA %d\n", o->grid.imax * o->grid.jmax * o->grid.kmax);
+    cursor += sprintf(cursor, "SPACING %f %f %f\n", o->grid.dx, o->grid.dy, o->grid.dz);
+    cursor += sprintf(cursor,
+        "POINT_DATA %d\n",
+        o->grid.imax * o->grid.jmax * o->grid.kmax);
 }
 
 void vtkOpen(VtkOptions* o, char* problem)
 {
     char filename[50];
 
-    if (o->mode == UNIX) {
-        if (commIsMaster(&o->comm)) {
-            snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size);
-            o->fh = fopen(filename, "w");
-            writeHeader(o);
-        }
-    } else if (o->mode == MPI) {
-    }
+    snprintf(filename, 50, "%s-p%d.vtk", problem, o->comm.size);
+    MPI_File_open(o->comm.comm,
+        filename,
+        MPI_MODE_WRONLY | MPI_MODE_CREATE,
+        MPI_INFO_NULL,
+        &o->fh);
 
     if (commIsMaster(&o->comm)) printf("Writing VTK output for %s\n", problem);
+    writeHeader(o);
 }
 
 static void writeScalar(VtkOptions* o, double* s)
@@ -153,13 +161,4 @@ void vtkVector(VtkOptions* o, char* name, VtkVector vec)
     }
 }
 
-void vtkClose(VtkOptions* o)
-{
-    if (o->mode == UNIX) {
-        if (commIsMaster(&o->comm)) {
-            fclose(o->fh);
-            o->fh = NULL;
-        }
-    } else if (o->mode == MPI) {
-    }
-}
+void vtkClose(VtkOptions* o) { MPI_File_close(o->fh); }
diff --git a/BasicSolver/3D-mpi-io/src/vtkWriter.h b/BasicSolver/3D-mpi-io/src/vtkWriter.h
index 7a91dbb..7756826 100644
--- a/BasicSolver/3D-mpi-io/src/vtkWriter.h
+++ b/BasicSolver/3D-mpi-io/src/vtkWriter.h
@@ -6,19 +6,15 @@
  */
 #ifndef __VTKWRITER_H_
 #define __VTKWRITER_H_
+#include <mpi.h>
 #include <stdio.h>
 
 #include "comm.h"
 #include "grid.h"
 
-typedef enum VtkFormat { ASCII = 0, BINARY } VtkFormat;
-typedef enum VtkMode { UNIX = 0, MPI } VtkMode;
-
 typedef struct VtkOptions {
-    VtkFormat fmt;
-    VtkMode mode;
     Grid grid;
-    FILE* fh;
+    MPI_File fh;
     Comm comm;
 } VtkOptions;