docker 容器报错 too many open files

docker 容器报错 too many open files

• 使用ulimit命令查看用户最大文件打开数

[root@test ~]# ulimit -a
core file size          (blocks, -c) unlimited
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 62990
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 65535
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 62990
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

• 查看docker 进程的PID

[root@test ~]# systemctl status docker|grep PID
 Main PID: 3217 (dockerd)

[root@test ~]#cat /proc/3217/limits 
Limit                     Soft Limit           Hard Limit           Units     
Max cpu time              unlimited            unlimited            seconds   
Max file size             unlimited            unlimited            bytes     
Max data size             unlimited            unlimited            bytes     
Max stack size            8388608              unlimited            bytes     
Max core file size        unlimited            unlimited            bytes     
Max resident set          unlimited            unlimited            bytes     
Max processes             unlimited            unlimited            processes 
Max open files            65536                65536              files     
Max locked memory         65536                65536                bytes     
Max address space         unlimited            unlimited            bytes     
Max file locks            unlimited            unlimited            locks     
Max pending signals       31820                31820                signals   
Max msgqueue size         819200               819200               bytes     
Max nice priority         0                    0                    
Max realtime priority     0                    0                    
Max realtime timeout      unlimited            unlimited            us  

• 查看PID为3217的进程打开的文件数量

[root@test ~]# ls -htrl /proc/3217/fd|wc -l
65535 #已经达到了设置了65535

• 动态修改当前Docker进程的nofile限制

[root@test ~]#  prlimit --pid 3217 --nofile=655360:655360 #将Docker进程的nofile限制调整为655360 
[root@test ~]# ulimit  -a
core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 31820
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 655350
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 31820
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

[root@test ~]# systemctl status docker #再次查询Docker进程状态发现问题已修复

• 修改内核配置文件file-max

[root@test ~]#  vim /etc/sysctl.conf
fs.file-max=655350 #新增
[root@test ~]#  sysctl  -p
net.ipv6.conf.all.disable_ipv6 = 0
net.ipv6.conf.default.disable_ipv6 = 0
net.ipv6.conf.lo.disable_ipv6 = 0
fs.file-max=655350

• 该问题后续影响
  • 该docker节点是k8s主节点，导致k8s调度分配不创建pods，最终重启服务器，才恢复，特此记录下。