From d3aa4d131752ce9cd6b518f2ee4be92dcbb62bb7 Mon Sep 17 00:00:00 2001 From: Guillaume DIDIER Date: Wed, 3 Jul 2024 09:56:22 +0200 Subject: [PATCH 1/6] Added debug information. --- cache_slice/src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cache_slice/src/lib.rs b/cache_slice/src/lib.rs index 4dbfb73..86ed321 100644 --- a/cache_slice/src/lib.rs +++ b/cache_slice/src/lib.rs @@ -99,6 +99,7 @@ fn monitor_core(addr: *const u8, cpu: u8, mut max_cbox: usize) -> Result= max_cbox { false @@ -106,16 +107,22 @@ fn monitor_core(addr: *const u8, cpu: u8, mut max_cbox: usize) -> Result Result Result Date: Wed, 3 Jul 2024 11:04:23 +0200 Subject: [PATCH 2/6] Get the number of available CBox on Intel Core using MSR_UNC_CBO_CONFIG --- cache_slice/src/arch.rs | 7 ++++++- cache_slice/src/lib.rs | 25 ++++++++++++------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/cache_slice/src/arch.rs b/cache_slice/src/arch.rs index d9e6bf3..e8e3a8b 100644 --- a/cache_slice/src/arch.rs +++ b/cache_slice/src/arch.rs @@ -135,6 +135,7 @@ pub struct XeonPerfCounters { } pub struct CorePerfCounters { + pub msr_unc_cbo_config: u64, pub max_slice: u16, pub msr_unc_perf_global_ctr: u64, pub val_enable_ctrs: u64, @@ -221,6 +222,7 @@ const BROADWELL_XEON: XeonPerfCounters = XeonPerfCounters { // TODO find appropriate values const ALDER_LAKE_TO_RAPTOR_LAKE_CORE: CorePerfCounters = CorePerfCounters { + msr_unc_cbo_config: 0x396, max_slice: 10, msr_unc_perf_global_ctr: 0x2ff0, val_enable_ctrs: 0x20000000, // To validate @@ -233,6 +235,7 @@ const ALDER_LAKE_TO_RAPTOR_LAKE_CORE: CorePerfCounters = CorePerfCounters { // TODO verify his on ICELAKE, and appropriate values. Also deal with backport Cypress Cove ? const CANNON_LAKE_TO_TIGER_LAKE_CORE: CorePerfCounters = CorePerfCounters { + msr_unc_cbo_config: 0x396, max_slice: 8, // To validate msr_unc_perf_global_ctr: 0xe01, val_enable_ctrs: 0x20000000, // To validate @@ -244,6 +247,7 @@ const CANNON_LAKE_TO_TIGER_LAKE_CORE: CorePerfCounters = CorePerfCounters { }; const SKYLAKE_KABYLAKE_CORE: CorePerfCounters = CorePerfCounters { + msr_unc_cbo_config: 0x396, max_slice: 7, msr_unc_perf_global_ctr: 0xe01, val_enable_ctrs: 0x20000000, @@ -257,7 +261,8 @@ const SKYLAKE_KABYLAKE_CORE: CorePerfCounters = CorePerfCounters { // This is documented in Intel SDM, 20.3.4.6 (in March 2024 edition) const SANDYBRIDGE_TO_BROADWELL_CORE: CorePerfCounters = CorePerfCounters { - max_slice: 0, + msr_unc_cbo_config: 0x396, + max_slice: 4, msr_unc_perf_global_ctr: 0x391, // Go in MSR_UNC_PERF_GLOBAL_CTR EN (bit 29) set to one, and route PMI to core 1-4 upon overflow. val_enable_ctrs: 0x2000000f, diff --git a/cache_slice/src/lib.rs b/cache_slice/src/lib.rs index 86ed321..5f67184 100644 --- a/cache_slice/src/lib.rs +++ b/cache_slice/src/lib.rs @@ -3,7 +3,7 @@ use std::arch::x86_64::_mm_clflush; use crate::arch::CpuClass::{IntelCore, IntelXeon, IntelXeonSP}; use crate::arch::{get_performance_counters_core, get_performance_counters_xeon}; -use crate::Error::UnsupportedCPU; +use crate::Error::{InvalidParameter, UnsupportedCPU}; use crate::msr::{read_msr_on_cpu, write_msr_on_cpu}; pub mod msr; @@ -39,7 +39,7 @@ unsafe fn monitor_xeon(addr: *const u8, cpu: u8, max_cbox: usize) -> Result Result Result, Error> { +fn monitor_core(addr: *const u8, cpu: u8) -> Result, Error> { // Note, we need to add the workaround for one missing perf counter here. let performance_counters = if let Some(p) = get_performance_counters_core() { p @@ -97,15 +97,13 @@ fn monitor_core(addr: *const u8, cpu: u8, mut max_cbox: usize) -> Result= max_cbox { - false - } else { - return Err(Error::InvalidParameter); - }; + eprint!("Finding the number of CBox available... "); + let max_cbox = read_msr_on_cpu(performance_counters.msr_unc_cbo_config, cpu)? & 0xF; // TODO magic number (mask for bit 3:0) + eprintln!("{}", max_cbox); + + if max_cbox > performance_counters.max_slice as u64 { + return Err(InvalidParameter); + } eprintln!("Disabling counters"); write_msr_on_cpu(performance_counters.msr_unc_perf_global_ctr, cpu, performance_counters.val_disable_ctrs)?; @@ -150,10 +148,11 @@ fn monitor_core(addr: *const u8, cpu: u8, mut max_cbox: usize) -> Result Result, Error> { match arch::determine_cpu_class() { Some(IntelCore) => { - unsafe { monitor_core(addr, cpu, max_cbox as usize) } + unsafe { monitor_core(addr, cpu) } } Some(IntelXeon) => { unsafe { monitor_xeon(addr, cpu, max_cbox as usize) } From 35b3ec71431199a6a5e8f8f9693709e24ef3ffc0 Mon Sep 17 00:00:00 2001 From: Guillaume DIDIER Date: Wed, 3 Jul 2024 11:14:32 +0200 Subject: [PATCH 3/6] u64 and usize are not the same type, sorry --- cache_slice/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cache_slice/src/lib.rs b/cache_slice/src/lib.rs index 5f67184..fff0857 100644 --- a/cache_slice/src/lib.rs +++ b/cache_slice/src/lib.rs @@ -98,10 +98,10 @@ fn monitor_core(addr: *const u8, cpu: u8) -> Result, Error> { }; eprint!("Finding the number of CBox available... "); - let max_cbox = read_msr_on_cpu(performance_counters.msr_unc_cbo_config, cpu)? & 0xF; // TODO magic number (mask for bit 3:0) + let max_cbox = (read_msr_on_cpu(performance_counters.msr_unc_cbo_config, cpu)? & 0xF) as usize; // TODO magic number (mask for bit 3:0) eprintln!("{}", max_cbox); - if max_cbox > performance_counters.max_slice as u64 { + if max_cbox > performance_counters.max_slice as usize { return Err(InvalidParameter); } From c99e244b71b6b55fa6814efcd61d137e5f6ceb06 Mon Sep 17 00:00:00 2001 From: Guillaume DIDIER Date: Wed, 3 Jul 2024 11:22:55 +0200 Subject: [PATCH 4/6] Only print detailed information in debug mode --- cache_slice/src/lib.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cache_slice/src/lib.rs b/cache_slice/src/lib.rs index fff0857..828fd3a 100644 --- a/cache_slice/src/lib.rs +++ b/cache_slice/src/lib.rs @@ -96,30 +96,36 @@ fn monitor_core(addr: *const u8, cpu: u8) -> Result, Error> { } else { return Err(UnsupportedCPU); }; - + #[cfg(debug_assertions)] eprint!("Finding the number of CBox available... "); let max_cbox = (read_msr_on_cpu(performance_counters.msr_unc_cbo_config, cpu)? & 0xF) as usize; // TODO magic number (mask for bit 3:0) + #[cfg(debug_assertions)] eprintln!("{}", max_cbox); if max_cbox > performance_counters.max_slice as usize { return Err(InvalidParameter); } + #[cfg(debug_assertions)] eprintln!("Disabling counters"); write_msr_on_cpu(performance_counters.msr_unc_perf_global_ctr, cpu, performance_counters.val_disable_ctrs)?; + #[cfg(debug_assertions)] eprint!("Resetting counters..."); for i in 0..max_cbox { eprint!(" {i}"); write_msr_on_cpu(performance_counters.msr_unc_cbo_per_ctr0[i], cpu, performance_counters.val_reset_ctrs)?; } + #[cfg(debug_assertions)] eprintln!(" ok"); + #[cfg(debug_assertions)] eprintln!("Selecting events"); for i in 0..max_cbox { write_msr_on_cpu(performance_counters.msr_unc_cbo_perfevtsel0[i], cpu, performance_counters.val_select_evt_core)?; } + #[cfg(debug_assertions)] eprintln!("enabling counters"); write_msr_on_cpu(performance_counters.msr_unc_perf_global_ctr, cpu, performance_counters.val_enable_ctrs)?; @@ -131,6 +137,7 @@ fn monitor_core(addr: *const u8, cpu: u8) -> Result, Error> { */ // Read counters + #[cfg(debug_assertions)] eprintln!("Gathering results"); let mut results = Vec::new(); for i in 0..max_cbox { @@ -142,6 +149,7 @@ fn monitor_core(addr: *const u8, cpu: u8) -> Result, Error> { } } + #[cfg(debug_assertions)] eprintln!("disabling counters again"); write_msr_on_cpu(performance_counters.msr_unc_perf_global_ctr, cpu, performance_counters.val_disable_ctrs)?; From 52ef3902f3fd5a7461efcc019aa4929c540b0cc7 Mon Sep 17 00:00:00 2001 From: Guillaume DIDIER Date: Wed, 3 Jul 2024 11:28:03 +0200 Subject: [PATCH 5/6] Fix forgotten debug print --- cache_slice/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cache_slice/src/lib.rs b/cache_slice/src/lib.rs index 828fd3a..63af753 100644 --- a/cache_slice/src/lib.rs +++ b/cache_slice/src/lib.rs @@ -113,6 +113,7 @@ fn monitor_core(addr: *const u8, cpu: u8) -> Result, Error> { #[cfg(debug_assertions)] eprint!("Resetting counters..."); for i in 0..max_cbox { + #[cfg(debug_assertions)] eprint!(" {i}"); write_msr_on_cpu(performance_counters.msr_unc_cbo_per_ctr0[i], cpu, performance_counters.val_reset_ctrs)?; } From 4ace7c8f7ecbaa257b2ba9012bed166cf793738d Mon Sep 17 00:00:00 2001 From: Guillaume DIDIER Date: Fri, 16 Aug 2024 09:56:00 +0200 Subject: [PATCH 6/6] Increase the size of the test area --- cache_slice/src/bin/scan.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache_slice/src/bin/scan.rs b/cache_slice/src/bin/scan.rs index dddd9f3..4ec4f53 100644 --- a/cache_slice/src/bin/scan.rs +++ b/cache_slice/src/bin/scan.rs @@ -8,7 +8,7 @@ pub fn main() { let nb_cores = core_per_package(); println!("Found {} cores", nb_cores); - let target = vec![0x0123456789abcdefu64; 64]; + let target = vec![0x0123456789abcdefu64; 1024]; let old = sched_getaffinity(Pid::from_raw(0)).unwrap(); let mut core_set = Vec::new();