From f295f54fca9dfc7860060713b8aa84480495bb6c Mon Sep 17 00:00:00 2001
From: Rafael Ravedutti <rafaelravedutti@gmail.com>
Date: Mon, 11 Oct 2021 16:57:02 +0200
Subject: [PATCH] Add useful data volume calculation for force kernel

Signed-off-by: Rafael Ravedutti <rafaelravedutti@gmail.com>
---
 src/force.c             |  9 +++++----
 src/includes/neighbor.h |  1 +
 src/main-stub.c         |  6 +++---
 src/main.c              | 15 ++++++++++-----
 src/neighbor.c          |  1 +
 5 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/force.c b/src/force.c
index dcd2633..4cc9a0a 100644
--- a/src/force.c
+++ b/src/force.c
@@ -33,7 +33,7 @@
 #endif
 
 #ifndef TRACER_CONDITION
-#   define TRACER_CONDITION                 (!(timestep % every))
+#   define TRACER_CONDITION                 (!(timestep % param->every))
 #endif
 
 #ifdef MEM_TRACER
@@ -118,10 +118,9 @@
 #   define DIST_TRACE(l, e)
 #endif
 
-double computeForce(Parameter *param, Atom *atom, Neighbor *neighbor, int first_exec, int timestep, int every) {
+double computeForce(Parameter *param, Atom *atom, Neighbor *neighbor, int first_exec, int timestep) {
     MEM_TRACER_INIT;
     INDEX_TRACER_INIT;
-    double S = getTimeStamp();
     int Nlocal = atom->Nlocal;
     int* neighs;
     MD_FLOAT* fx = atom->fx; MD_FLOAT* fy = atom->fy; MD_FLOAT* fz = atom->fz;
@@ -138,11 +137,13 @@ double computeForce(Parameter *param, Atom *atom, Neighbor *neighbor, int first_
     }
 
     INDEX_TRACE_NATOMS(Nlocal, atom->Nghost, neighbor->maxneighs);
+    double S = getTimeStamp();
     LIKWID_MARKER_START("force");
     #pragma omp parallel for
     for(int i = 0; i < Nlocal; i++) {
         neighs = &neighbor->neighbors[i * neighbor->maxneighs];
         int numneighs = neighbor->numneigh[i];
+        neighbor->totalneighs += numneighs; // Maybe remove this for real time measurements
         MD_FLOAT xtmp = atom_x(i);
         MD_FLOAT ytmp = atom_y(i);
         MD_FLOAT ztmp = atom_z(i);
@@ -217,8 +218,8 @@ double computeForce(Parameter *param, Atom *atom, Neighbor *neighbor, int first_
         MEM_TRACE(fz[i], 'W');
     }
     LIKWID_MARKER_STOP("force");
-
     double E = getTimeStamp();
+
     INDEX_TRACER_END;
     MEM_TRACER_END;
     return E-S;
diff --git a/src/includes/neighbor.h b/src/includes/neighbor.h
index c9bad95..9c35e8c 100644
--- a/src/includes/neighbor.h
+++ b/src/includes/neighbor.h
@@ -31,6 +31,7 @@ typedef struct {
     int* neighbors;
     int maxneighs;
     int* numneigh;
+    long long int totalneighs;
 } Neighbor;
 
 extern void initNeighbor(Neighbor*, Parameter*);
diff --git a/src/main-stub.c b/src/main-stub.c
index 79dc298..a96f92c 100644
--- a/src/main-stub.c
+++ b/src/main-stub.c
@@ -16,7 +16,7 @@
 #define LATTICE_DISTANCE    10.0
 #define NEIGH_DISTANCE      1.0
 
-extern double computeForce(Parameter*, Atom*, Neighbor*, int, int, int);
+extern double computeForce(Parameter*, Atom*, Neighbor*, int, int);
 
 void init(Parameter *param) {
     param->epsilon = 1.0;
@@ -207,12 +207,12 @@ int main(int argc, const char *argv[]) {
     DEBUG("Building neighbor lists...\n");
     buildNeighbor(atom, &neighbor);
     DEBUG("Computing forces...\n");
-    computeForce(&param, atom, &neighbor, 1, 0, param.every);
+    computeForce(&param, atom, &neighbor, 1, 0);
 
     double S, E;
     S = getTimeStamp();
     for(int i = 0; i < param.ntimes; i++) {
-        computeForce(&param, atom, &neighbor, 0, i + 1, param.every);
+        computeForce(&param, atom, &neighbor, 0, i + 1);
     }
     E = getTimeStamp();
     double T_accum = E-S;
diff --git a/src/main.c b/src/main.c
index 2e2c67b..9ac72e2 100644
--- a/src/main.c
+++ b/src/main.c
@@ -47,7 +47,7 @@ typedef enum {
     NUMTIMER
 } timertype;
 
-extern double computeForce(Parameter*, Atom*, Neighbor*, int, int, int);
+extern double computeForce(Parameter*, Atom*, Neighbor*, int, int);
 
 void init(Parameter *param)
 {
@@ -206,14 +206,13 @@ int main (int argc, char** argv)
 
     setup(&param, &atom, &neighbor);
     computeThermo(0, &param, &atom);
-    computeForce(&param, &atom, &neighbor, 1, 0, param.every);
+    computeForce(&param, &atom, &neighbor, 1, 0);
 
     timer[FORCE] = 0.0;
     timer[NEIGH] = 0.0;
     timer[TOTAL] = getTimeStamp();
 
     for(int n = 0; n < param.ntimes; n++) {
-
         initialIntegrate(&param, &atom);
 
         if((n + 1) % param.every) {
@@ -222,7 +221,7 @@ int main (int argc, char** argv)
             timer[NEIGH] += reneighbour(&param, &atom, &neighbor);
         }
 
-        timer[FORCE] += computeForce(&param, &atom, &neighbor, 0, n + 1, param.every);
+        timer[FORCE] += computeForce(&param, &atom, &neighbor, 0, n + 1);
         finalIntegrate(&param, &atom);
 
         if(!((n + 1) % param.nstat) && (n+1) < param.ntimes) {
@@ -247,7 +246,13 @@ int main (int argc, char** argv)
     printf(HLINE);
     printf("Performance: %.2f million atom updates per second\n",
             1e-6 * (double) atom.Natoms * param.ntimes / timer[TOTAL]);
-
+    double force_useful_volume = 1e-9 * ( (double)(atom.Nlocal * (param.ntimes + 1)) * (sizeof(MD_FLOAT) * 6 + sizeof(int)) +
+                                          (double)(neighbor.totalneighs) * (sizeof(MD_FLOAT) * 3 + sizeof(int)) );
+#ifdef EXPLICIT_TYPES
+    force_useful_volume += 1e-9 * (double)((atom.Nlocal * (param.ntimes + 1)) + neighbor.totalneighs) * sizeof(int);
+#endif
+    printf("total_neighs = %lld/%.2f\n", neighbor.totalneighs, (double)(neighbor.totalneighs));
+    printf("Useful read data volume for force computation: %.2fGB\n", force_useful_volume);
     LIKWID_MARKER_CLOSE;
     return EXIT_SUCCESS;
 }
diff --git a/src/neighbor.c b/src/neighbor.c
index 91c7ab8..451342c 100644
--- a/src/neighbor.c
+++ b/src/neighbor.c
@@ -69,6 +69,7 @@ void initNeighbor(Neighbor *neighbor, Parameter *param)
     neighbor->maxneighs = 100;
     neighbor->numneigh = NULL;
     neighbor->neighbors = NULL;
+    neighbor->totalneighs = 0;
 }
 
 void setupNeighbor()