cache_utils/analyse_csv.py

@@ -3,26 +3,24 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-License-Identifier: MIT
 
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+#import tikzplotlib
+import wquantiles as wq
+import numpy as np
 import argparse
-import warnings
+
-import time
-import json
 import sys
 import os
 
-import matplotlib.style as mplstyle
+import json
-import matplotlib.pyplot as plt
+import warnings
-import wquantiles as wq
-import seaborn as sns
-import pandas as pd
-import numpy as np
-#import tikzplotlib
 
+warnings.filterwarnings('ignore')
+print("warnings are filtered, enable them back if you are having some trouble")
 
-t = time.time()
+sns.set_theme()
-def print_timed(*args, **kwargs):
-    print(f"[{round(time.time()-t, 1):>8}]", *args, **kwargs)
-
 
 def dict_to_json(d):
     if isinstance(d, dict):
@@ -88,9 +86,6 @@ parser.add_argument(
 
 args = parser.parse_args()
 
-warnings.filterwarnings('ignore')
-print_timed("warnings are filtered, enable them back if you are having some trouble")
-
 img_dir = os.path.dirname(args.path)+"/figs/"
 os.makedirs(img_dir, exist_ok=True)
 
@@ -119,7 +114,7 @@ df = pd.read_csv(args.path + "-results_lite.csv.bz2",
         converters={'address': convert64, 'hash': convert8},
         )
 
-print_timed(f"Loaded columns : {list(df.keys())}")
+print(f"Loaded columns : {list(df.keys())}")
 
 sample_columns = [
 "clflush_remote_hit",
@@ -148,30 +143,24 @@ if args.slice_remap:
 core_mapping = pd.read_csv(args.path + ".cores.csv")
 
 def remap_core(key):
-    column = core_mapping.columns.get_loc(key)
     def remap(core):
-        return core_mapping.iat[core, column]
+        remapped = core_mapping.iloc[core]
+        return remapped[key]
 
     return remap
 
 
-columns = [
+df["main_socket"] = df["main_core"].apply(remap_core("socket"))
-    ("main_socket", "main_core", "socket")
+df["main_core_fixed"] = df["main_core"].apply(remap_core("core"))
-    ("main_core_fixed", "main_core", "core")
+df["main_ht"] = df["main_core"].apply(remap_core("hthread"))
-    ("main_ht", "main_core", "hthread")
+df["helper_socket"] = df["helper_core"].apply(remap_core("socket"))
-    ("helper_socket", "helper_core", "socket")
+df["helper_core_fixed"] = df["helper_core"].apply(remap_core("core"))
-    ("helper_core_fixed", "helper_core", "core")
+df["helper_ht"] = df["helper_core"].apply(remap_core("hthread"))
-    ("helper_ht", "helper_core", "hthread")
-]
-for (col, icol, key) in columns:
-    df[col] = df[icol].apply(remap_core(key))
-    print_timed(f"Column {col} added")
 
 
 if args.slice_remap:
     slice_remap = lambda h: slice_mapping["slice_group"].iloc[h]
     df["slice_group"] = df["hash"].apply(slice_remap)
-    print_timed(f"Column slice_group added")
 else:
     df["slice_group"] = df["hash"]
 
@@ -183,10 +172,9 @@ def get_graphing_bounds():
     return int(((min(q10s) - 10) // 10) * 10), int(((max(q90s) + 19) // 10) * 10)
 
 
-mplstyle.use("fast")
-
 graph_lower, graph_upper = get_graphing_bounds()
-print_timed(f"graphing between {graph_lower}, {graph_upper}")
+print("graphing between {}, {}".format(graph_lower, graph_upper))
+
 
 def plot(filename, g=None):
     if args.no_plot:
@@ -194,7 +182,6 @@ def plot(filename, g=None):
             g.savefig(img_dir+filename)
         else:
             plt.savefig(img_dir+filename)
-        print_timed(f"Saved {filename}")
         plt.close()
     plt.show()
 
@@ -246,7 +233,31 @@ def export_stats_csv():
         return maxi-mini
     
     def compute_stat(x, key):
-        return wq.median(x["time"], x[key])
+        def compute_median(x):
+            return wq.median(x["time"], x[key])
+
+        filtered_x = x[(x[key] != 0)]
+        mini, maxi = filtered_x["time"].min(), filtered_x["time"].max()
+
+        miss_spread = get_spread(x, "clflush_miss_n")
+        
+        if maxi-mini < 3*miss_spread:
+            med = compute_median(x)
+            return [med, med]
+
+        if key == "clflush_remote_hit":
+            """print(
+                "double for core {}:{}@{}, helper {}:{}@{}".format(
+                    x["main_core_fixed"].unique()[0],
+                    x["main_ht"].unique()[0],
+                    x["main_socket"].unique()[0],
+                    x["helper_core_fixed"].unique()[0],
+                    x["helper_ht"].unique()[0],
+                    x["helper_socket"].unique()[0],
+                )
+            )"""
+        center = mini + (maxi-mini)/2
+        return [compute_median(filtered_x[(filtered_x["time"] < center)]), compute_median(filtered_x[(filtered_x["time"] >= center)])]
     
     df_grouped = df.groupby(["main_core", "helper_core", "hash"])
     
@@ -265,6 +276,8 @@ def export_stats_csv():
         "clflush_shared_hit": hit_shared.values
     })
 
+    stats = stats.explode(['clflush_miss_n', 'clflush_remote_hit', 'clflush_local_hit_n', 'clflush_shared_hit'])
+
     stats.to_csv(args.path + ".stats.csv", index=False)
 
 
@@ -295,4 +308,4 @@ if not args.stats:
 if not os.path.exists(args.path + ".stats.csv") or args.stats:
     export_stats_csv()
 else:
-    print_timed("Skipping .stats.csv export")
+    print("Skipping .stats.csv export")

cache_utils/analyse_medians.py

@@ -15,9 +15,7 @@ import pandas as pd
 import seaborn as sns
 from scipy import optimize
 import matplotlib.pyplot as plt
-import matplotlib.style as mplstyle
 
-mplstyle.use("fast")
 
 warnings.filterwarnings("ignore")
 print("warnings are filtered, enable them back if you are having some trouble")
@@ -56,23 +54,14 @@ parser.add_argument(
     help="Create slice{} directories with segmented grid",
 )
 
-parser.add_argument(
-    "--no-slice-remap",
-    dest="slice_remap",
-    action="store_false",
-    default=True,
-    help="Don't remap the slices"
-)
-
 args = parser.parse_args()
 
 img_dir = os.path.dirname(args.path) + "/figs/"
 os.makedirs(img_dir, exist_ok=True)
 
 assert os.path.exists(args.path + ".stats.csv")
+assert os.path.exists(args.path + ".slices.csv")
 assert os.path.exists(args.path + ".cores.csv")
-if args.slice_remap:
-    assert os.path.exists(args.path + ".slices.csv")
 
 stats = pd.read_csv(
     args.path + ".stats.csv",
@@ -95,8 +84,7 @@ stats = pd.read_csv(
     },
 )
 
-if args.slice_remap:
+slice_mapping = pd.read_csv(args.path + ".slices.csv")
-    slice_mapping = pd.read_csv(args.path + ".slices.csv")
 core_mapping = pd.read_csv(args.path + ".cores.csv")
 
 # print("core mapping:\n", core_mapping.to_string())
@@ -141,12 +129,9 @@ stats["helper_ht"] = stats["helper_core"].apply(remap_core("hthread"))
 
 # slice_mapping = {3: 0, 1: 1, 2: 2, 0: 3}
 
-if args.slice_remap:
+stats["slice_group"] = stats["hash"].apply(
-    stats["slice_group"] = stats["hash"].apply(
+    lambda h: slice_mapping["slice_group"].iloc[h]
-        lambda h: slice_mapping["slice_group"].iloc[h]
+)
-    )
-else:
-    stats["slice_group"] = stats["hash"]
 
 graph_lower_miss = int((min_time_miss // 10) * 10)
 graph_upper_miss = int(((max_time_miss + 9) // 10) * 10)
@@ -401,26 +386,15 @@ def facet_grid(
             "clflush_miss_n",
         ],
         colors=["y", "r", "g", "b"],
-        separate_hthreads=False,
         title=None,
     ):
     """
     Creates a facet grid showing all points
     """
-    if separate_hthreads:
-        colors=["y", "r", "g", "b"]
-        for el in shown:
-            for helper, main in itertools.product((0, 1), (0, 1)):
-                df[el+f"_m{main}h{helper}"] = df[(df["main_ht"] == main) & (df["helper_ht"] == helper)][el]
-
     grid = sns.FacetGrid(df, row=row, col=col)
 
     for i, el in enumerate(shown):
-        if separate_hthreads:
+        grid.map(draw_fn, third, el, color=colors[i % len(colors)])
-            for helper, main in itertools.product((0, 1), (0, 1)):
-                grid.map(draw_fn, third, el+f"_m{main}h{helper}", color=colors[(helper+2*main) % len(colors)])# marker=['+', 'x'][helper])
-        else:
-            grid.map(draw_fn, third, el, color=colors[i % len(colors)])
 
     if title is not None:
         plot(title, g=grid)
@@ -434,7 +408,7 @@ def all_facets(df, pre="", post="", *args, **kwargs):
     """
 
     facet_grid(
-        df, "helper_core_fixed", "main_core_fixed", "slice_group",
+        df, "main_core_fixed", "helper_core_fixed", "slice_group",
         title=f"{pre}facet_slice{post}.png", *args, **kwargs
     )
     facet_grid(
@@ -442,58 +416,47 @@ def all_facets(df, pre="", post="", *args, **kwargs):
         title=f"{pre}facet_main{post}.png", *args, **kwargs
     )
     facet_grid(
-        df, "main_core_fixed", "slice_group", "helper_core_fixed",
+        df, "slice_group", "main_core_fixed", "helper_core_fixed",
         title=f"{pre}facet_helper{post}.png", *args, **kwargs
     )
 
 
-def do_facet(main: int, helper: int, line: bool, metrics: str):
+def do_facet(main: int, helper: int, line: bool):
-    """
-    - metrics: hit, miss or all
-    """
     df = stats.copy(deep=True)
 
-    print(f"Doing all facets {main}x{helper} {metrics}")
+    print(f"Doing all facets {main}x{helper}")
     filtered_df = stats[
-        (stats["main_socket"] == main)
+        (stats["main_core_fixed"] // (num_core / 2) == main)
-        & (stats["helper_socket"] == helper)
+        & (stats["helper_core_fixed"] // (num_core / 2) == helper)
     ]
     method = "line" if line else "pt"
-    shown = []
-    colors = []
-    if metrics == "hit" or metrics == "all":
-        shown.append("clflush_remote_hit")
-        colors.append("r")
-    if metrics == "miss" or metrics == "all":
-        shown.append("clflush_miss_n")
-        colors.append("b")
-
     all_facets(
         filtered_df,
-        pre=f"{metrics}_{method}_",
+        pre=f"hit_{method}_",
         post=f"_m{main}h{helper}",
-        shown=shown,
+        shown=["clflush_remote_hit"],
-        colors=colors,
+        colors=["r"],
         draw_fn=sns.lineplot if line else sns.scatterplot
     )
+    all_facets(
+        filtered_df,
+        pre=f"miss_{method}_",
+        post=f"_m{main}h{helper}",
+        shown=["clflush_miss_n"],
+        colors=["b"],
+        draw_fn=sns.lineplot if line else sns.scatterplot
+    )
+
 
 
 if args.rslice:
     rslice()
 
 # do_predictions(stats)
-# all_facets(stats, shown=["clflush_remote_hit"], colors=["r"])
+# all_facets(stats, "")
 
 
 
 with Pool(8) as pool:
-    pool.starmap(
+    pool.starmap(do_facet, itertools.product((0, 1), (0, 1), (True, False)))
-        do_facet,
-        itertools.product(
-            stats["main_socket"].unique(),
-            stats["helper_socket"].unique(),
-            (True, False),
-            ("hit", "miss")
-        )
-    )