隔核绑核、服务器优化
# 关闭服务
NetworkManager 超线程
# 查看核隔离情况
$ more /proc/cmdline
BOOT_IMAGE=/vmlinuz-3.10.0-1062.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root r
d.lvm.lv=centos/swap rhgb quiet skew_tick=1 isolcpus=2-3 intel_pstate=disable nosoftlockup
2
3
# 隔核
https://blog.csdn.net/Frederick_Fung/article/details/116698835 (opens new window)
# 修改/etc/default/grub: GRUB_CMDLINE_LINUX="... isolcpus=1-2,7-8" 隔离1,2,7,8核心
vim /etc/default/grub
# 更新配置 centos
grub2-mkconfig -o /boot/grub2/grub.cfg
# 更新配置 ubuntu
update-grub
# 重启
reboot
# 查看结果
cat /proc/cmdline
2
3
4
5
6
7
8
9
10
11
12
13
14
# 绑核
https://blog.csdn.net/lanyzh0909/article/details/50404664
#include<sched.h>
// 绑定进程到cpu上
void set_cpu_affinity(int id)
{
cpu_set_t mask;
CPU_ZERO(&mask);
CPU_SET(id, &mask);
if (sched_setaffinity(0, sizeof(mask), &mask) == -1)
{
fprintf(stderr, "warning: could not set CPU affinity/n");
}
}
// 绑定线程到cpu上
int pthread_setaffinity_np(pthread_t thread, size_t cpusetsize, const cpu_set_t *cpuset);
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# 可能有用的优化
# 看起来重要的
isolcpus=2-17 nohz_full=2-17 nohz=on rcu_nocbs=2-17 rcu_nocb_poll intel_idle.max_cstate=0 processor.max_cstate=0 intel_pstate=disable idle=poll pcie_aspm.policy=performance irqaffinity=0
# 看起来不重要的
mce=ignore_ce ipmi_si.force_kipmid=0 nmi_watchdog=0 hpet=disabled clocksource=tsc tsc=reliable audit=0 acpi_irq_nobalance noht iommu=off intel_iommu=off noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off mitigations=off
# 其他可选(未必是优化):
transparent_hugepage=never
# 解释
noht 禁用超线程
nohz_full=2-17 nohz=on 禁用时间片时钟中断 查看时间片长度(通常是100ms):
cat /proc/sys/kernel/sched_rr_timeslice_ms
# 关闭下面的服务
参考https://cloud.tencent.com/developer/article/1583554 (opens new window)
for SERVICE in
avahi-daemon.service crond.service dnsmasq.service
firewalld.service lvm2-monitor.service postfix.service
rpcgssd.service rpcidmapd.service rpcsvcgssd.service
wpa_supplicant.service irqbalance.service
do systemctl disable $SERVICE
systemctl stop $SERVICE
done
2
3
4
5
6
7
8
以及 cpupower cpuspeed cpufreqd powerd irqbalance firewalld
# 查看隔核效果
命令行查看:
watch -d cat /proc/interrupts
测试程序查看
irq_test.cpp:
#include <iostream>
#include <unistd.h>
#include <stdint.h>
#include <stdio.h>
using namespace std;
uint64_t now() { return __builtin_ia32_rdtsc(); }
int test() {
unit64_t last = now();
unit64_t start = now();
int irq_times = 0;
int irq_total_cycle = 0;
unit64_t total_run_cycle = 1024*1024*1024;
unit64_t irq_threshold = 300;
while(1) {
unit64_t cur = now(); //point B
unit64_t diff = cur - last;
if(diff > irq_threshold) { //check diff from point A -> B
irq_times++;
irq_total_cycle += diff;
cur = now();
if(cur-start > total_run_cycle) break;
}
last = cur; //point A
}
printf("total_run_cycle=%ld, irq_times=%ld, irq_threshold=%d, irq_avg=%ld\n",
total_run_cycle, irq_times, irq_threshold, irq_total_cycle/irq_times);
}
int main()
{
test();
test();
test();
}
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
编译运行
g++ irq_test.cpp
taskset -c 5 ./a.out
2