From 24d920785ebe764ef48e405e497230808df9d6ba Mon Sep 17 00:00:00 2001 From: augustin64 Date: Thu, 13 Jun 2024 11:02:20 +0200 Subject: [PATCH] reformat with black --- cache_utils/analyse_medians.py | 309 +++++++++++++++++++++++---------- 1 file changed, 213 insertions(+), 96 deletions(-) diff --git a/cache_utils/analyse_medians.py b/cache_utils/analyse_medians.py index 67dc028..b526083 100644 --- a/cache_utils/analyse_medians.py +++ b/cache_utils/analyse_medians.py @@ -3,18 +3,19 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: MIT -import pandas as pd -import matplotlib.pyplot as plt -import seaborn as sns -import numpy as np -from scipy import optimize -import argparse -import sys import os - +import sys +import argparse import warnings -warnings.filterwarnings('ignore') +import numpy as np +import pandas as pd +import seaborn as sns +from scipy import optimize +import matplotlib.pyplot as plt + + +warnings.filterwarnings("ignore") print("warnings are filtered, enable them back if you are having some trouble") # TODO @@ -40,7 +41,7 @@ parser.add_argument( dest="no_plot", action="store_true", default=False, - help="No visible plot (save figures to files)" + help="No visible plot (save figures to files)", ) parser.add_argument( @@ -48,37 +49,38 @@ parser.add_argument( dest="rslice", action="store_true", default=False, - help="Create slice{} directories with segmented grid" + help="Create slice{} directories with segmented grid", ) args = parser.parse_args() -img_dir = os.path.dirname(args.path)+"/figs/" +img_dir = os.path.dirname(args.path) + "/figs/" os.makedirs(img_dir, exist_ok=True) assert os.path.exists(args.path + ".stats.csv") assert os.path.exists(args.path + ".slices.csv") assert os.path.exists(args.path + ".cores.csv") -stats = pd.read_csv(args.path + ".stats.csv", - dtype={ - "main_core": np.int8, - "helper_core": np.int8, - # "address": int, - "hash": np.int8, - # "time": np.int16, - "clflush_remote_hit": np.float64, - "clflush_shared_hit": np.float64, - # "clflush_miss_f": np.int32, - # "clflush_local_hit_f": np.int32, - "clflush_miss_n": np.float64, - "clflush_local_hit_n": np.float64, - # "reload_miss": np.int32, - # "reload_remote_hit": np.int32, - # "reload_shared_hit": np.int32, - # "reload_local_hit": np.int32 - } - ) +stats = pd.read_csv( + args.path + ".stats.csv", + dtype={ + "main_core": np.int8, + "helper_core": np.int8, + # "address": int, + "hash": np.int8, + # "time": np.int16, + "clflush_remote_hit": np.float64, + "clflush_shared_hit": np.float64, + # "clflush_miss_f": np.int32, + # "clflush_local_hit_f": np.int32, + "clflush_miss_n": np.float64, + "clflush_local_hit_n": np.float64, + # "reload_miss": np.int32, + # "reload_remote_hit": np.int32, + # "reload_shared_hit": np.int32, + # "reload_local_hit": np.int32 + }, +) slice_mapping = pd.read_csv(args.path + ".slices.csv") core_mapping = pd.read_csv(args.path + ".cores.csv") @@ -99,17 +101,23 @@ def remap_core(key): return remap + def plot(filename, g=None): if args.no_plot: if g is not None: - g.savefig(img_dir+filename) + g.savefig(img_dir + filename) else: - plt.savefig(img_dir+filename) - # tikzplotlib.save(img_dir+filename+".tex", axis_width=r'0.175\textwidth', axis_height=r'0.25\textwidth') + plt.savefig(img_dir + filename) + # tikzplotlib.save( + # img_dir+filename+".tex", + # axis_width=r'0.175\textwidth', + # axis_height=r'0.25\textwidth' + # ) print(filename, "saved") plt.close() plt.show() + stats["main_socket"] = stats["main_core"].apply(remap_core("socket")) stats["main_core_fixed"] = stats["main_core"].apply(remap_core("core")) stats["main_ht"] = stats["main_core"].apply(remap_core("hthread")) @@ -119,7 +127,9 @@ stats["helper_ht"] = stats["helper_core"].apply(remap_core("hthread")) # slice_mapping = {3: 0, 1: 1, 2: 2, 0: 3} -stats["slice_group"] = stats["hash"].apply(lambda h: slice_mapping["slice_group"].iloc[h]) +stats["slice_group"] = stats["hash"].apply( + lambda h: slice_mapping["slice_group"].iloc[h] +) graph_lower_miss = int((min_time_miss // 10) * 10) graph_upper_miss = int(((max_time_miss + 9) // 10) * 10) @@ -144,153 +154,248 @@ num_core = len(stats["main_core_fixed"].unique()) def miss_topology(main_core_fixed, slice_group, C, h): return C + h * abs(main_core_fixed - slice_group) + h * abs(slice_group + 1) + def miss_topology_df(x, C, h): - return x.apply(lambda x, C, h: miss_topology(x["main_core_fixed"], x["slice_group"], C, h), args=(C, h), axis=1) + func = lambda x, C, h: miss_topology(x["main_core_fixed"], x["slice_group"], C, h) + return x.apply(func, args=(C, h), axis=1) memory = -1 gpu_if_any = num_core + def exclusive_hit_topology_gpu(main_core, slice_group, helper_core, C, h1, h2): round_trip = gpu_if_any - memory - if slice_group <= num_core/2: + if slice_group <= num_core / 2: # send message towards higher cores first if helper_core < slice_group: - r = C + h1 * abs(main_core - slice_group) + h2 * abs(round_trip - (helper_core - memory)) + r = ( + C + + h1 * abs(main_core - slice_group) + + h2 * abs(round_trip - (helper_core - memory)) + ) else: - r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) + r = ( + C + + h1 * abs(main_core - slice_group) + + h2 * abs(helper_core - slice_group) + ) else: # send message toward lower cores first if helper_core > slice_group: r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - memory) else: - r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) + r = ( + C + + h1 * abs(main_core - slice_group) + + h2 * abs(helper_core - slice_group) + ) return r + def exclusive_hit_topology_gpu_df(x, C, h1, h2): - return x.apply(lambda x, C, h1, h2: exclusive_hit_topology_gpu(x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2), args=(C, h1, h2), axis=1) + def func(x, C, h1, h2): + return exclusive_hit_topology_gpu( + x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2 + ) + + return x.apply(func, args=(C, h1, h2), axis=1) + def exclusive_hit_topology_gpu2(main_core, slice_group, helper_core, C, h1, h2): round_trip = gpu_if_any + 1 - memory - if slice_group <= num_core/2: + if slice_group <= num_core / 2: # send message towards higher cores first if helper_core < slice_group: - r = C + h1 * abs(main_core - slice_group) + h2 * abs(round_trip - (helper_core - memory)) + r = ( + C + + h1 * abs(main_core - slice_group) + + h2 * abs(round_trip - (helper_core - memory)) + ) else: - r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) + r = ( + C + + h1 * abs(main_core - slice_group) + + h2 * abs(helper_core - slice_group) + ) else: # send message toward lower cores first if helper_core > slice_group: r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - memory) else: - r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) + r = ( + C + + h1 * abs(main_core - slice_group) + + h2 * abs(helper_core - slice_group) + ) return r + def exclusive_hit_topology_gpu2_df(x, C, h1, h2): - return x.apply(lambda x, C, h1, h2: exclusive_hit_topology_gpu2(x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2), args=(C, h1, h2), axis=1) + def func(x, C, h1, h2): + return exclusive_hit_topology_gpu2( + x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2 + ) + + return x.apply(func, args=(C, h1, h2), axis=1) + # unlikely def exclusive_hit_topology_nogpu(main_core, slice_group, helper_core, C, h1, h2): - round_trip = (num_core-1) - memory + round_trip = (num_core - 1) - memory - if slice_group <= num_core/2: + if slice_group <= num_core / 2: # send message towards higher cores first if helper_core < slice_group: - r = C + h1 * abs(main_core - slice_group) + h2 * abs(round_trip - (helper_core - memory)) + r = ( + C + + h1 * abs(main_core - slice_group) + + h2 * abs(round_trip - (helper_core - memory)) + ) else: - r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) + r = ( + C + + h1 * abs(main_core - slice_group) + + h2 * abs(helper_core - slice_group) + ) else: # send message toward lower cores first if helper_core > slice_group: r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - memory) else: - r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) + r = ( + C + + h1 * abs(main_core - slice_group) + + h2 * abs(helper_core - slice_group) + ) return r + def exclusive_hit_topology_nogpu_df(x, C, h1, h2): - return x.apply(lambda x, C, h1, h2: exclusive_hit_topology_nogpu(x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2), args=(C, h1, h2), axis=1) + def func(x, C, h1, h2): + return exclusive_hit_topology_nogpu( + x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2 + ) + + return x.apply(func, args=(C, h1, h2), axis=1) + def remote_hit_topology_2(x, C, h): main_core = x["main_core_fixed"] slice_group = x["slice_group"] helper_core = x["helper_core_fixed"] - return C + h * abs(main_core - slice_group) + h * abs(slice_group - helper_core) + h * abs(helper_core - main_core) + return ( + C + + h * abs(main_core - slice_group) + + h * abs(slice_group - helper_core) + + h * abs(helper_core - main_core) + ) + def shared_hit_topology_1(x, C, h): main_core = x["main_core_fixed"] slice_group = x["slice_group"] helper_core = x["helper_core_fixed"] - return C + h * abs(main_core - slice_group) + h * max(abs(slice_group - main_core), abs(slice_group - helper_core)) + return ( + C + + h * abs(main_core - slice_group) + + h * max(abs(slice_group - main_core), abs(slice_group - helper_core)) + ) def do_predictions(df): - res_miss = optimize.curve_fit(miss_topology_df, df[["main_core_fixed", "slice_group"]], df["clflush_miss_n"]) + res_miss = optimize.curve_fit( + miss_topology_df, df[["main_core_fixed", "slice_group"]], df["clflush_miss_n"] + ) # print("Miss topology:") # print(res_miss) - res_gpu = optimize.curve_fit(exclusive_hit_topology_gpu_df, df[["main_core_fixed", "slice_group", "helper_core_fixed"]], df["clflush_remote_hit"]) + res_gpu = optimize.curve_fit( + exclusive_hit_topology_gpu_df, + df[["main_core_fixed", "slice_group", "helper_core_fixed"]], + df["clflush_remote_hit"], + ) # print("Exclusive hit topology (GPU):") # print(res_gpu) - #res_gpu2 = optimize.curve_fit(exclusive_hit_topology_gpu2_df, df[["main_core_fixed", "slice_group", "helper_core_fixed"]], df["clflush_remote_hit"]) - #print("Exclusive hit topology (GPU2):") - #print(res_gpu2) + # res_gpu2 = optimize.curve_fit( + # exclusive_hit_topology_gpu2_df, + # df[["main_core_fixed", "slice_group", "helper_core_fixed"]], + # df["clflush_remote_hit"] + # ) + # print("Exclusive hit topology (GPU2):") + # print(res_gpu2) - #res_no_gpu = optimize.curve_fit(exclusive_hit_topology_nogpu_df, df[["main_core_fixed", "slice_group", "helper_core_fixed"]], df["clflush_remote_hit"]) - #print("Exclusive hit topology (No GPU):") - #print(res_no_gpu) + # res_no_gpu = optimize.curve_fit( + # exclusive_hit_topology_nogpu_df, + # df[["main_core_fixed", "slice_group", "helper_core_fixed"]], + # df["clflush_remote_hit"] + # ) + # print("Exclusive hit topology (No GPU):") + # print(res_no_gpu) + df["predicted_miss"] = miss_topology_df(df, *(res_miss[0])) - # df["predicted_miss"] = miss_topology_df(df, *(res_miss[0])) - - - #df["predicted_remote_hit_no_gpu"] = exclusive_hit_topology_nogpu_df(df, *(res_no_gpu[0])) + # df["predicted_remote_hit_no_gpu"] = exclusive_hit_topology_nogpu_df(df, *(res_no_gpu[0])) df["predicted_remote_hit_gpu"] = exclusive_hit_topology_gpu_df(df, *(res_gpu[0])) - #df["predicted_remote_hit_gpu2"] = exclusive_hit_topology_gpu_df(df, *(res_gpu2[0])) - + # df["predicted_remote_hit_gpu2"] = exclusive_hit_topology_gpu_df(df, *(res_gpu2[0])) df_A0 = df[df["main_core_fixed"] == 0] - figure_median_E_A0 = sns.FacetGrid(df_A0, col="slice_group") - figure_median_E_A0.map(sns.scatterplot, 'helper_core_fixed', 'clflush_remote_hit', color="r") - figure_median_E_A0.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_gpu', color="r") - figure_median_E_A0.set_titles(col_template="$S$ = {col_name}") + figure_A0 = sns.FacetGrid(df_A0, col="slice_group") + figure_A0.map(sns.scatterplot, "helper_core_fixed", "clflush_remote_hit", color="r") + figure_A0.map( + sns.lineplot, "helper_core_fixed", "predicted_remote_hit_gpu", color="r" + ) + figure_A0.set_titles(col_template="$S$ = {col_name}") plot("medians_remote_hit.png") g2 = sns.FacetGrid(df, row="main_core_fixed", col="slice_group") - g2.map(sns.scatterplot, 'helper_core_fixed', 'clflush_remote_hit', color="r") - g2.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_gpu', color="r") - #g2.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_gpu2', color="g") - #g2.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_no_gpu', color="g") + g2.map(sns.scatterplot, "helper_core_fixed", "clflush_remote_hit", color="r") + g2.map(sns.lineplot, "helper_core_fixed", "predicted_remote_hit_gpu", color="r") + # g2.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_gpu2', color="g") + # g2.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_no_gpu', color="g") plot("medians_remote_hit_grid.png", g=g2) def rslice(): for core in stats["main_core_fixed"].unique(): - os.makedirs(img_dir+f"slices{core}", exist_ok=True) - for slice in stats["slice_group"].unique(): - df = stats[(stats["slice_group"] == slice) & (stats["main_core_fixed"] == core)] - fig = sns.scatterplot(df, x="helper_core_fixed", y="clflush_remote_hit", color="r") - fig.set(title=f"main_core={core} slice={slice}") - plt.savefig(img_dir+f"slices{core}/"+str(slice)+".png") + os.makedirs(img_dir + f"slices{core}", exist_ok=True) + for slice_ in stats["slice_group"].unique(): + df = stats[ + (stats["slice_group"] == slice_) & (stats["main_core_fixed"] == core) + ] + fig = sns.scatterplot( + df, x="helper_core_fixed", y="clflush_remote_hit", color="r" + ) + fig.set(title=f"main_core={core} slice={slice_}") + plt.savefig(img_dir + f"slices{core}/" + str(slice_) + ".png") plt.close() def facet_grid( - df, row, col, third, - draw_fn=sns.scatterplot, - shown=["clflush_shared_hit", "clflush_remote_hit", "clflush_local_hit_n", "clflush_miss_n"], - colors=["y", "r", "g", "b"], - title=None - ): + df, + row, + col, + third, + draw_fn=sns.scatterplot, + shown=[ + "clflush_shared_hit", + "clflush_remote_hit", + "clflush_local_hit_n", + "clflush_miss_n", + ], + colors=["y", "r", "g", "b"], + title=None, +): """ Creates a facet grid showing all points """ grid = sns.FacetGrid(df, row=row, col=col) for i, el in enumerate(shown): - grid.map(draw_fn, third, el, color=colors[i%len(colors)]) + grid.map(draw_fn, third, el, color=colors[i % len(colors)]) if title is not None: plot(title, g=grid) @@ -299,15 +404,24 @@ def facet_grid( return grid -def all_facets(df, id, *args, **kwargs): +def all_facets(df, id_, *args, **kwargs): """ df : panda dataframe - id: the str to append to filenames + id_: the str to append to filenames """ - facet_grid(df, "main_core_fixed", "helper_core_fixed", "slice_group", title=f"medians_facet_{id}s.png", *args, **kwargs) - facet_grid(df, "helper_core_fixed", "slice_group", "main_core_fixed", title=f"medians_facet_{id}c.png", *args, **kwargs) - facet_grid(df, "slice_group", "main_core_fixed", "helper_core_fixed", title=f"medians_facet_{id}h.png", *args, **kwargs) + facet_grid( + df, "main_core_fixed", "helper_core_fixed", "slice_group", + title=f"medians_facet_{id_}s.png", *args, **kwargs + ) + facet_grid( + df, "helper_core_fixed", "slice_group", "main_core_fixed", + title=f"medians_facet_{id}c.png", *args, **kwargs + ) + facet_grid( + df, "slice_group", "main_core_fixed", "helper_core_fixed", + title=f"medians_facet_{id}h.png", *args, **kwargs + ) if args.rslice: @@ -319,5 +433,8 @@ all_facets(stats, "") for main in (0, 1): for helper in (0, 1): print(f"Doing all facets {main}x{helper}") - filtered_df = stats[(stats["main_core_fixed"]//(num_core/2) == main) & (stats["helper_core_fixed"]//(num_core/2) == helper)] - all_facets(filtered_df, f"m{main}h{helper}_") \ No newline at end of file + filtered_df = stats[ + (stats["main_core_fixed"] // (num_core / 2) == main) + & (stats["helper_core_fixed"] // (num_core / 2) == helper) + ] + all_facets(filtered_df, f"m{main}h{helper}_")