隔核绑核、服务器优化

# 关闭服务

NetworkManager 超线程

# 查看核隔离情况

$ more /proc/cmdline 
BOOT_IMAGE=/vmlinuz-3.10.0-1062.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root r
d.lvm.lv=centos/swap rhgb quiet skew_tick=1 isolcpus=2-3 intel_pstate=disable nosoftlockup

1
2
3

# 隔核

https://blog.csdn.net/Frederick_Fung/article/details/116698835 (opens new window)

# 修改/etc/default/grub: GRUB_CMDLINE_LINUX="... isolcpus=1-2,7-8" 隔离1，2，7，8核心
vim /etc/default/grub 

# 更新配置 centos
grub2-mkconfig -o /boot/grub2/grub.cfg 

# 更新配置 ubuntu
update-grub

#  重启
reboot

#  查看结果
cat /proc/cmdline

1
2
3
4
5
6
7
8
9
10
11
12
13
14

# 绑核

https://blog.csdn.net/lanyzh0909/article/details/50404664

#include<sched.h>

// 绑定进程到cpu上
void set_cpu_affinity(int id)
{
    cpu_set_t mask;
    CPU_ZERO(&mask);
    CPU_SET(id, &mask);
    if (sched_setaffinity(0, sizeof(mask), &mask) == -1)
    {
        fprintf(stderr, "warning: could not set CPU affinity/n");
    }
}

// 绑定线程到cpu上
int pthread_setaffinity_np(pthread_t thread, size_t cpusetsize, const cpu_set_t *cpuset);

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

# 可能有用的优化

# 看起来重要的

isolcpus=2-17 nohz_full=2-17 nohz=on rcu_nocbs=2-17 rcu_nocb_poll intel_idle.max_cstate=0 processor.max_cstate=0 intel_pstate=disable idle=poll pcie_aspm.policy=performance irqaffinity=0

# 看起来不重要的

mce=ignore_ce ipmi_si.force_kipmid=0 nmi_watchdog=0 hpet=disabled clocksource=tsc tsc=reliable audit=0 acpi_irq_nobalance noht iommu=off intel_iommu=off noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off mitigations=off

# 其他可选（未必是优化）：

transparent_hugepage=never

# 解释

noht 禁用超线程
nohz_full=2-17 nohz=on 禁用时间片时钟中断查看时间片长度（通常是100ms）： cat /proc/sys/kernel/sched_rr_timeslice_ms

# 关闭下面的服务

参考https://cloud.tencent.com/developer/article/1583554 (opens new window)

for SERVICE in
   avahi-daemon.service     crond.service        dnsmasq.service
   firewalld.service         lvm2-monitor.service     postfix.service
   rpcgssd.service          rpcidmapd.service          rpcsvcgssd.service
   wpa_supplicant.service    irqbalance.service
do systemctl  disable $SERVICE
    systemctl stop $SERVICE
done

1
2
3
4
5
6
7
8

以及 cpupower cpuspeed cpufreqd powerd irqbalance firewalld

# 查看隔核效果

命令行查看：

watch -d cat /proc/interrupts

测试程序查看

irq_test.cpp:

#include <iostream>
#include <unistd.h>
#include <stdint.h>
#include <stdio.h>
using namespace std;

uint64_t now() { return __builtin_ia32_rdtsc(); }

int test() {
  unit64_t last = now();
  unit64_t start = now();
  int irq_times = 0;
  int irq_total_cycle = 0;

  unit64_t total_run_cycle = 1024*1024*1024;
  unit64_t irq_threshold = 300;
  while(1) {
    unit64_t cur = now(); //point B
    unit64_t diff = cur - last;
    if(diff > irq_threshold) { //check diff from point A -> B 
      irq_times++;
      irq_total_cycle += diff;
      cur = now();
      if(cur-start > total_run_cycle) break;
    }
    last = cur; //point A
  }
  printf("total_run_cycle=%ld, irq_times=%ld, irq_threshold=%d, irq_avg=%ld\n",
    total_run_cycle, irq_times, irq_threshold, irq_total_cycle/irq_times);
}
int main()
{
    test();
    test();
    test();
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

编译运行

g++ irq_test.cpp
taskset -c 5 ./a.out

1
2

编辑

上次更新: 2023/05/07, 17:27:54

← numa网卡绑定 popen底层原理和仿照实现-execl→