l3-ENSL/dendrobates-t-azureus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add all_facets

Browse Source 
Segment code in multiple functions
Reformat with black
This commit is contained in:
augustin64 2024-06-13 11:08:14 +02:00
parent f894161143
commit 31fc0236f8
1 changed files with 248 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

331
cache_utils/analyse_medians.py
View File

@ -3,19 +3,19 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-License-Identifier: MIT # SPDX-License-Identifier: MIT
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sys import exit
import numpy as np
from scipy import optimize
import argparse
import sys
import os import os
import sys
import argparse
import warnings import warnings
warnings.filterwarnings('ignore') import numpy as np
import pandas as pd
import seaborn as sns
from scipy import optimize
import matplotlib.pyplot as plt
warnings.filterwarnings("ignore")
print("warnings are filtered, enable them back if you are having some trouble") print("warnings are filtered, enable them back if you are having some trouble")
# TODO # TODO
@ -41,7 +41,7 @@ parser.add_argument(
dest="no_plot", dest="no_plot",
action="store_true", action="store_true",
default=False, default=False,
help="No visible plot (save figures to files)" help="No visible plot (save figures to files)",
) )
parser.add_argument( parser.add_argument(
@ -49,7 +49,7 @@ parser.add_argument(
dest="rslice", dest="rslice",
action="store_true", action="store_true",
default=False, default=False,
help="Create slice{} directories with segmented grid" help="Create slice{} directories with segmented grid",
) )
args = parser.parse_args() args = parser.parse_args()
@ -61,7 +61,8 @@ assert os.path.exists(args.path + ".stats.csv")
assert os.path.exists(args.path + ".slices.csv") assert os.path.exists(args.path + ".slices.csv")
assert os.path.exists(args.path + ".cores.csv") assert os.path.exists(args.path + ".cores.csv")
stats = pd.read_csv(args.path + ".stats.csv", stats = pd.read_csv(
args.path + ".stats.csv",
dtype={ dtype={
"main_core": np.int8, "main_core": np.int8,
"helper_core": np.int8, "helper_core": np.int8,
@ -78,14 +79,14 @@ stats = pd.read_csv(args.path + ".stats.csv",
# "reload_remote_hit": np.int32, # "reload_remote_hit": np.int32,
# "reload_shared_hit": np.int32, # "reload_shared_hit": np.int32,
# "reload_local_hit": np.int32 # "reload_local_hit": np.int32
} },
) )
slice_mapping = pd.read_csv(args.path + ".slices.csv") slice_mapping = pd.read_csv(args.path + ".slices.csv")
core_mapping = pd.read_csv(args.path + ".cores.csv") core_mapping = pd.read_csv(args.path + ".cores.csv")
print("core mapping:\n", core_mapping.to_string()) # print("core mapping:\n", core_mapping.to_string())
print("slice mapping:\n", slice_mapping.to_string()) # print("slice mapping:\n", slice_mapping.to_string())
# print("core {} is mapped to '{}'".format(4, repr(core_mapping.iloc[4]))) # print("core {} is mapped to '{}'".format(4, repr(core_mapping.iloc[4])))
@ -100,15 +101,23 @@ def remap_core(key):
return remap return remap
def plot(filename, g=None): def plot(filename, g=None):
if args.no_plot: if args.no_plot:
if g is not None: if g is not None:
g.savefig(img_dir + filename) g.savefig(img_dir + filename)
else: else:
plt.savefig(img_dir + filename) plt.savefig(img_dir + filename)
# tikzplotlib.save(
# img_dir+filename+".tex",
# axis_width=r'0.175\textwidth',
# axis_height=r'0.25\textwidth'
# )
print(filename, "saved")
plt.close() plt.close()
plt.show() plt.show()
stats["main_socket"] = stats["main_core"].apply(remap_core("socket")) stats["main_socket"] = stats["main_core"].apply(remap_core("socket"))
stats["main_core_fixed"] = stats["main_core"].apply(remap_core("core")) stats["main_core_fixed"] = stats["main_core"].apply(remap_core("core"))
stats["main_ht"] = stats["main_core"].apply(remap_core("hthread")) stats["main_ht"] = stats["main_core"].apply(remap_core("hthread"))
@ -118,17 +127,15 @@ stats["helper_ht"] = stats["helper_core"].apply(remap_core("hthread"))
# slice_mapping = {3: 0, 1: 1, 2: 2, 0: 3} # slice_mapping = {3: 0, 1: 1, 2: 2, 0: 3}
stats["slice_group"] = stats["hash"].apply(lambda h: slice_mapping["slice_group"].iloc[h]) stats["slice_group"] = stats["hash"].apply(
lambda h: slice_mapping["slice_group"].iloc[h]
)
graph_lower_miss = int((min_time_miss // 10) * 10) graph_lower_miss = int((min_time_miss // 10) * 10)
graph_upper_miss = int(((max_time_miss + 9) // 10) * 10) graph_upper_miss = int(((max_time_miss + 9) // 10) * 10)
print("Graphing from {} to {}".format(graph_lower_miss, graph_upper_miss)) # print("Graphing from {} to {}".format(graph_lower_miss, graph_upper_miss))
g_ = sns.FacetGrid(stats, col="main_core_fixed", row="slice_group")
g_.map(sns.histplot, 'clflush_miss_n', bins=range(graph_lower_miss, graph_upper_miss), color="b", edgecolor="b", alpha=0.2)
plot("medians_miss_grid.png", g=g_)
# also explains remote # also explains remote
# shared needs some thinking as there is something weird happening there. # shared needs some thinking as there is something weird happening there.
@ -138,23 +145,19 @@ plot("medians_miss_grid.png", g=g_)
# #
print(stats.head()) # print(stats.head())
num_core = len(stats["main_core_fixed"].unique()) num_core = len(stats["main_core_fixed"].unique())
print("Found {}".format(num_core)) # print("Found {}".format(num_core))
def miss_topology(main_core_fixed, slice_group, C, h): def miss_topology(main_core_fixed, slice_group, C, h):
return C + h * abs(main_core_fixed - slice_group) + h * abs(slice_group + 1) return C + h * abs(main_core_fixed - slice_group) + h * abs(slice_group + 1)
def miss_topology_df(x, C, h): def miss_topology_df(x, C, h):
return x.apply(lambda x, C, h: miss_topology(x["main_core_fixed"], x["slice_group"], C, h), args=(C, h), axis=1) func = lambda x, C, h: miss_topology(x["main_core_fixed"], x["slice_group"], C, h)
return x.apply(func, args=(C, h), axis=1)
res_miss = optimize.curve_fit(miss_topology_df, stats[["main_core_fixed", "slice_group"]], stats["clflush_miss_n"])
print("Miss topology:")
print(res_miss)
memory = -1 memory = -1
@ -167,20 +170,37 @@ def exclusive_hit_topology_gpu(main_core, slice_group, helper_core, C, h1, h2):
if slice_group <= num_core / 2: if slice_group <= num_core / 2:
# send message towards higher cores first # send message towards higher cores first
if helper_core < slice_group: if helper_core < slice_group:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(round_trip - (helper_core - memory)) r = (
C
+ h1 * abs(main_core - slice_group)
+ h2 * abs(round_trip - (helper_core - memory))
)
else: else:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) r = (
C
+ h1 * abs(main_core - slice_group)
+ h2 * abs(helper_core - slice_group)
)
else: else:
# send message toward lower cores first # send message toward lower cores first
if helper_core > slice_group: if helper_core > slice_group:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - memory) r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - memory)
else: else:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) r = (
C
+ h1 * abs(main_core - slice_group)
+ h2 * abs(helper_core - slice_group)
)
return r return r
def exclusive_hit_topology_gpu_df(x, C, h1, h2): def exclusive_hit_topology_gpu_df(x, C, h1, h2):
return x.apply(lambda x, C, h1, h2: exclusive_hit_topology_gpu(x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2), args=(C, h1, h2), axis=1) def func(x, C, h1, h2):
return exclusive_hit_topology_gpu(
x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2
)
return x.apply(func, args=(C, h1, h2), axis=1)
def exclusive_hit_topology_gpu2(main_core, slice_group, helper_core, C, h1, h2): def exclusive_hit_topology_gpu2(main_core, slice_group, helper_core, C, h1, h2):
@ -189,20 +209,37 @@ def exclusive_hit_topology_gpu2(main_core, slice_group, helper_core, C, h1, h2):
if slice_group <= num_core / 2: if slice_group <= num_core / 2:
# send message towards higher cores first # send message towards higher cores first
if helper_core < slice_group: if helper_core < slice_group:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(round_trip - (helper_core - memory)) r = (
C
+ h1 * abs(main_core - slice_group)
+ h2 * abs(round_trip - (helper_core - memory))
)
else: else:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) r = (
C
+ h1 * abs(main_core - slice_group)
+ h2 * abs(helper_core - slice_group)
)
else: else:
# send message toward lower cores first # send message toward lower cores first
if helper_core > slice_group: if helper_core > slice_group:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - memory) r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - memory)
else: else:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) r = (
C
+ h1 * abs(main_core - slice_group)
+ h2 * abs(helper_core - slice_group)
)
return r return r
def exclusive_hit_topology_gpu2_df(x, C, h1, h2): def exclusive_hit_topology_gpu2_df(x, C, h1, h2):
return x.apply(lambda x, C, h1, h2: exclusive_hit_topology_gpu2(x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2), args=(C, h1, h2), axis=1) def func(x, C, h1, h2):
return exclusive_hit_topology_gpu2(
x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2
)
return x.apply(func, args=(C, h1, h2), axis=1)
# unlikely # unlikely
@ -212,123 +249,187 @@ def exclusive_hit_topology_nogpu(main_core, slice_group, helper_core, C, h1, h2)
if slice_group <= num_core / 2: if slice_group <= num_core / 2:
# send message towards higher cores first # send message towards higher cores first
if helper_core < slice_group: if helper_core < slice_group:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(round_trip - (helper_core - memory)) r = (
C
+ h1 * abs(main_core - slice_group)
+ h2 * abs(round_trip - (helper_core - memory))
)
else: else:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) r = (
C
+ h1 * abs(main_core - slice_group)
+ h2 * abs(helper_core - slice_group)
)
else: else:
# send message toward lower cores first # send message toward lower cores first
if helper_core > slice_group: if helper_core > slice_group:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - memory) r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - memory)
else: else:
r = C + h1 * abs(main_core - slice_group) + h2 * abs(helper_core - slice_group) r = (
C
+ h1 * abs(main_core - slice_group)
+ h2 * abs(helper_core - slice_group)
)
return r return r
def exclusive_hit_topology_nogpu_df(x, C, h1, h2): def exclusive_hit_topology_nogpu_df(x, C, h1, h2):
return x.apply(lambda x, C, h1, h2: exclusive_hit_topology_nogpu(x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2), args=(C, h1, h2), axis=1) def func(x, C, h1, h2):
return exclusive_hit_topology_nogpu(
x["main_core_fixed"], x["slice_group"], x["helper_core_fixed"], C, h1, h2
#res_no_gpu = optimize.curve_fit(exclusive_hit_topology_nogpu_df, stats[["main_core_fixed", "slice_group", "helper_core_fixed"]], stats["clflush_remote_hit"]) )
#print("Exclusive hit topology (No GPU):")
#print(res_no_gpu)
res_gpu = optimize.curve_fit(exclusive_hit_topology_gpu_df, stats[["main_core_fixed", "slice_group", "helper_core_fixed"]], stats["clflush_remote_hit"])
print("Exclusive hit topology (GPU):")
print(res_gpu)
#res_gpu2 = optimize.curve_fit(exclusive_hit_topology_gpu2_df, stats[["main_core_fixed", "slice_group", "helper_core_fixed"]], stats["clflush_remote_hit"])
#print("Exclusive hit topology (GPU2):")
#print(res_gpu2)
return x.apply(func, args=(C, h1, h2), axis=1)
def remote_hit_topology_2(x, C, h): def remote_hit_topology_2(x, C, h):
main_core = x["main_core_fixed"] main_core = x["main_core_fixed"]
slice_group = x["slice_group"] slice_group = x["slice_group"]
helper_core = x["helper_core_fixed"] helper_core = x["helper_core_fixed"]
return C + h * abs(main_core - slice_group) + h * abs(slice_group - helper_core) + h * abs(helper_core - main_core) return (
C
+ h * abs(main_core - slice_group)
+ h * abs(slice_group - helper_core)
+ h * abs(helper_core - main_core)
)
def shared_hit_topology_1(x, C, h): def shared_hit_topology_1(x, C, h):
main_core = x["main_core_fixed"] main_core = x["main_core_fixed"]
slice_group = x["slice_group"] slice_group = x["slice_group"]
helper_core = x["helper_core_fixed"] helper_core = x["helper_core_fixed"]
return C + h * abs(main_core - slice_group) + h * max(abs(slice_group - main_core), abs(slice_group - helper_core)) return (
C
+ h * abs(main_core - slice_group)
+ h * max(abs(slice_group - main_core), abs(slice_group - helper_core))
)
def plot_func(function, *params): def do_predictions(df):
def plot_it(x, **kwargs): res_miss = optimize.curve_fit(
# plot_x = [] miss_topology_df, df[["main_core_fixed", "slice_group"]], df["clflush_miss_n"]
# plot_y = [] )
# for x in set(x): # print("Miss topology:")
# plot_y.append(function(x, *params)) # print(res_miss)
# plot_x = x
print(x)
plot_y = function(x, *params)
sns.lineplot(x, plot_y, **kwargs)
return plot_it
stats["predicted_miss"] = miss_topology_df(stats, *(res_miss[0])) res_gpu = optimize.curve_fit(
exclusive_hit_topology_gpu_df,
df[["main_core_fixed", "slice_group", "helper_core_fixed"]],
df["clflush_remote_hit"],
)
# print("Exclusive hit topology (GPU):")
# print(res_gpu)
figure_median_I = sns.FacetGrid(stats, col="main_core_fixed") # res_gpu2 = optimize.curve_fit(
figure_median_I.map(sns.scatterplot, 'slice_group', 'clflush_miss_n', color="b") # exclusive_hit_topology_gpu2_df,
figure_median_I.map(sns.lineplot, 'slice_group', 'predicted_miss', color="b") # df[["main_core_fixed", "slice_group", "helper_core_fixed"]],
figure_median_I.set_titles(col_template="$A$ = {col_name}") # df["clflush_remote_hit"]
figure_median_I.tight_layout() # )
# print("Exclusive hit topology (GPU2):")
# print(res_gpu2)
# import tikzplotlib # res_no_gpu = optimize.curve_fit(
# exclusive_hit_topology_nogpu_df,
# df[["main_core_fixed", "slice_group", "helper_core_fixed"]],
# df["clflush_remote_hit"]
# )
# print("Exclusive hit topology (No GPU):")
# print(res_no_gpu)
# tikzplotlib.save("fig-median-I.tex", axis_width=r'0.175\textwidth', axis_height=r'0.25\textwidth') df["predicted_miss"] = miss_topology_df(df, *(res_miss[0]))
plot("medians_miss.png")
#stats["predicted_remote_hit_no_gpu"] = exclusive_hit_topology_nogpu_df(stats, *(res_no_gpu[0])) # df["predicted_remote_hit_no_gpu"] = exclusive_hit_topology_nogpu_df(df, *(res_no_gpu[0]))
stats["predicted_remote_hit_gpu"] = exclusive_hit_topology_gpu_df(stats, *(res_gpu[0])) df["predicted_remote_hit_gpu"] = exclusive_hit_topology_gpu_df(df, *(res_gpu[0]))
#stats["predicted_remote_hit_gpu2"] = exclusive_hit_topology_gpu_df(stats, *(res_gpu2[0])) # df["predicted_remote_hit_gpu2"] = exclusive_hit_topology_gpu_df(df, *(res_gpu2[0]))
df_A0 = df[df["main_core_fixed"] == 0]
stats_A0 = stats[stats["main_core_fixed"] == 0] figure_A0 = sns.FacetGrid(df_A0, col="slice_group")
figure_median_E_A0 = sns.FacetGrid(stats_A0, col="slice_group") figure_A0.map(sns.scatterplot, "helper_core_fixed", "clflush_remote_hit", color="r")
figure_median_E_A0.map(sns.scatterplot, 'helper_core_fixed', 'clflush_remote_hit', color="r") figure_A0.map(
figure_median_E_A0.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_gpu', color="r") sns.lineplot, "helper_core_fixed", "predicted_remote_hit_gpu", color="r"
figure_median_E_A0.set_titles(col_template="$S$ = {col_name}") )
figure_A0.set_titles(col_template="$S$ = {col_name}")
# tikzplotlib.save("fig-median-E-A0.tex", axis_width=r'0.175\textwidth', axis_height=r'0.25\textwidth')
plot("medians_remote_hit.png") plot("medians_remote_hit.png")
g2 = sns.FacetGrid(df, row="main_core_fixed", col="slice_group")
g = sns.FacetGrid(stats, row="main_core_fixed") g2.map(sns.scatterplot, "helper_core_fixed", "clflush_remote_hit", color="r")
g.map(sns.scatterplot, 'slice_group', 'clflush_miss_n', color="b") g2.map(sns.lineplot, "helper_core_fixed", "predicted_remote_hit_gpu", color="r")
g.map(sns.scatterplot, 'slice_group', 'clflush_local_hit_n', color="g")
plot("medians_miss_v_localhit_core.png", g=g)
g0 = sns.FacetGrid(stats, row="slice_group")
g0.map(sns.scatterplot, 'main_core_fixed', 'clflush_miss_n', color="b")
g0.map(sns.scatterplot, 'main_core_fixed', 'clflush_local_hit_n', color="g") # this gives away the trick I think !
# possibility of sending a general please discard this everyone around one of the ring + wait for ACK - direction depends on the core.
plot("medians_miss_v_localhit_slice.png", g=g0)
g2 = sns.FacetGrid(stats, row="main_core_fixed", col="slice_group")
g2.map(sns.scatterplot, 'helper_core_fixed', 'clflush_remote_hit', color="r")
g2.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_gpu', color="r")
# g2.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_gpu2', color="g") # g2.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_gpu2', color="g")
# g2.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_no_gpu', color="g") # g2.map(sns.lineplot, 'helper_core_fixed', 'predicted_remote_hit_no_gpu', color="g")
#g2.map(plot_func(exclusive_hit_topology_nogpu_df, *(res_no_gpu[0])), 'helper_core_fixed', color="g")
plot("medians_remote_hit_grid.png", g=g2) plot("medians_remote_hit_grid.png", g=g2)
if args.rslice: def rslice():
for core in stats["main_core_fixed"].unique(): for core in stats["main_core_fixed"].unique():
os.makedirs(img_dir + f"slices{core}", exist_ok=True) os.makedirs(img_dir + f"slices{core}", exist_ok=True)
for slice in stats["slice_group"].unique(): for slice_ in stats["slice_group"].unique():
df = stats[(stats["slice_group"] == slice) & (stats["main_core_fixed"] == core)] df = stats[
fig = sns.scatterplot(df, x="helper_core_fixed", y="clflush_remote_hit", color="r") (stats["slice_group"] == slice_) & (stats["main_core_fixed"] == core)
fig.set(title=f"main_core={core} slice={slice}") ]
plt.savefig(img_dir+f"slices{core}/"+str(slice)+".png") fig = sns.scatterplot(
df, x="helper_core_fixed", y="clflush_remote_hit", color="r"
)
fig.set(title=f"main_core={core} slice={slice_}")
plt.savefig(img_dir + f"slices{core}/" + str(slice_) + ".png")
plt.close() plt.close()
g3 = sns.FacetGrid(stats, row="main_core_fixed", col="slice_group") def facet_grid(
g3.map(sns.scatterplot, 'helper_core_fixed', 'clflush_shared_hit', color="y") df, row, col, third,
plot("medians_sharedhit.png", g=g3) draw_fn=sns.scatterplot,
# more ideas needed shown=[
"clflush_shared_hit",
"clflush_remote_hit",
"clflush_local_hit_n",
"clflush_miss_n",
],
colors=["y", "r", "g", "b"],
title=None,
):
"""
Creates a facet grid showing all points
"""
grid = sns.FacetGrid(df, row=row, col=col)
for i, el in enumerate(shown):
grid.map(draw_fn, third, el, color=colors[i % len(colors)])
if title is not None:
plot(title, g=grid)
return grid
def all_facets(df, id_, *args, **kwargs):
"""
df : panda dataframe
id_: the str to append to filenames
"""
facet_grid(
df, "main_core_fixed", "helper_core_fixed", "slice_group",
title=f"medians_facet_{id_}s.png", *args, **kwargs
)
facet_grid(
df, "helper_core_fixed", "slice_group", "main_core_fixed",
title=f"medians_facet_{id}c.png", *args, **kwargs
)
facet_grid(
df, "slice_group", "main_core_fixed", "helper_core_fixed",
title=f"medians_facet_{id}h.png", *args, **kwargs
)
if args.rslice:
rslice()
do_predictions(stats)
all_facets(stats, "")
for main in (0, 1):
for helper in (0, 1):
print(f"Doing all facets {main}x{helper}")
filtered_df = stats[
(stats["main_core_fixed"] // (num_core / 2) == main)
& (stats["helper_core_fixed"] // (num_core / 2) == helper)
]
all_facets(filtered_df, f"m{main}h{helper}_")