https://blog.csdn.net/qq_31747765/article/details/108922204

yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpmcurl -s -L https://nvidia.github.io/nvidia-docker/centos7/x86_64/nvidia-docker.repo | sudo tee /etc/yum.repos.d/nvidia-docker.repo

$ sudo yum install nvidia-detect$ nvidia-detect -v
Probing for supported NVIDIA devices...
[10de:1e04] NVIDIA Corporation TU102 [GeForce RTX 2080 Ti]
This device requires the current 440.64 NVIDIA driver kmod-nvidia
[1af4:1050] Red Hat, Inc. Virtio GPULinux查看显卡信息：(ps：若找不到lspci命令，可以安装 yum install pciutils)
lspci | grep -i vga
使用nvidia GPU可以：
lspci | grep -i nvidia
查看显卡驱动
cat /proc/driver/nvidia/version

NIVID官网:http://www.geforce.cn/drivers

yum search --showduplicates nvidia-docker

$ cat /etc/docker/daemon.json
{"registry-mirrors": ["https://xxxxxxx"],   #替换成自己的harbor仓库地址"live-restore": true,"default-shm-size": "128M","max-concurrent-downloads": 10,"oom-score-adjust": -1000,"debug": false,"exec-opts": ["native.cgroupdriver=cgroupfs"],"log-driver": "json-file","log-opts": {"max-size": "20m","max-file": "3"},"storage-driver": "overlay2","default-runtime": "nvidia","storage-opts": ["overlay2.override_kernel_check=true"],"runtimes": {"nvidia": {"path": "/usr/bin/nvidia-container-runtime","runtimeArgs": []}}
}

# 资料 https://unix.stackexchange.com/questions/115289/driver-install-kernel-source-not-found
yum -y install kernel-devel kernel-header

monitor.shGPU跨平台通用监控脚本功能： Useage: monitor.sh fast|mem|gpu|temp|all|[pathToLog sleepTimeNum]注意： ./monitor.sh fast速度最快#!/bin/bash#. /etc/profile#. ~/.bash_profile#. ~/.bashrc# 判断nvidia-smi命令是否存在/usr/bin/nvidia-smi > /dev/nullif [ $? -eq 0 ]thenecho 'nvidia-smi check pass' `date`elseecho 'nvidia-smi not exists'exit 1fi# 获取GPU Countfunction get_gpu_list(){count=`nvidia-smi -L|wc -l`echo $count}#获取GPU id对应uuidfunction get_uuid(){uuid=`nvidia-smi -q -i $1|grep 'UUID'|awk '{print $4}'`echo $uuid}#获取显存使用率 function get_memory_usage(){usage=`nvidia-smi -q -d MEMORY -i $1|grep -E 'Total|Used'|head -2|awk '{print $3}'|xargs echo|awk '{print $2/$1}'`echo $usage}#获取内存详细信息function get_memory_detail(){detail=`nvidia-smi -q -d MEMORY -i $1|grep -E 'Total|Used|Free'|head -3|awk '{print $3}'|xargs echo`echo $detail}#获取GPU使用率function get_volatile_gpu(){vol=`nvidia-smi -q -d UTILIZATION -i $1 |grep -A 5 "GPU Utilization"|tail -1|awk '{print $3}'` echo $vol}#获取GPU Current 温度function get_temperature(){temp=`nvidia-smi -q -d Temperature -i $1|grep 'GPU Current'|awk '{print $5}'`echo $temp}#获取Pod_idfunction get_pod_id(){echo `hostname`}#数据output#output $1 $2 $3 $4 $5#$1 字段名 $2 pod_id $3 gpu编号 $4 uuid $5 监控值function output(){echo $1"{podid=\""$2"\",gpu=\""$3"\",uuid=\""$4"\"}" $5}#输出mem prometheus格式数据#dcgm_mem_usage{pod_id="localhost"}function mem_prm(){for((i=0;i<`get_gpu_list`;i++))doname="dcgm_mem_usage"pod_id=`get_pod_id`uuid=`get_uuid $i`value=`get_memory_usage $i`output $name $pod_id $i $uuid $valuedone}#输出mem detail prometheus格式数据#dcgm_mem_detail{pod_id="localhost"}function mem_detail_prm(){for((i=0;i<`get_gpu_list`;i++))dopod_id=`get_pod_id`uuid=`get_uuid $i`value=`get_memory_detail $i`output "dcgm_fb_total" $pod_id $i $uuid `echo $value|awk '{print $1}'`output "dcgm_fb_used" $pod_id $i $uuid `echo $value|awk '{print $2}'`output "dcgm_fb_free" $pod_id $i $uuid `echo $value|awk '{print $3}'`done}#输出gpu prometheus格式数据#dcgm_gpu_utilization{...}function gpu_prm(){for((i=0;i<`get_gpu_list`;i++))doname="dcgm_gpu_utilization"pod_id=`get_pod_id`uuid=`get_uuid $i`value=`get_volatile_gpu $i`output $name $pod_id $i $uuid $valuedone}#输出温度 prometheus格式数据#dcgm_temp{...}function temp_prm(){for((i=0;i<`get_gpu_list`;i++))doname="dcgm_temp"pod_id=`get_pod_id`uuid=`get_uuid $i`value=`get_temperature $i`output $name $pod_id $i $uuid $valuedone}function allinone(){mem_prmmem_detail_prmgpu_prmtemp_prm}#快速获取function fast(){nvidia-smi -q > /tmp/1num=0count=0uuid=''first=0for i in `cat /tmp/1|grep -E 'Minor Number|UUID|GPU Current Temp|Gpu|Total|Used|Free'|cut -d ':' -f2|awk '{print $1}'`doif [ $num -eq 0 ];thenuuid=$ielif [ $num -eq 1 ];thencount=$ielif [ $num -eq 2 ];thenif [ $first -lt 13 ];thenecho '# HELP dcgm_fb_total Framebuffer memory total (in MiB).'echo '# TYPE dcgm_fb_total gauge'fioutput 'dcgm_fb_total' ${HOSTNAME} $count $uuid $ielif [ $num -eq 3 ];thenif [ $first -lt 13 ];thenecho '# HELP dcgm_fb_used Framebuffer memory used (in MiB).'echo '# TYPE dcgm_fb_used gauge'fioutput 'dcgm_fb_used' ${HOSTNAME} $count $uuid $ielif [ $num -eq 4 ];thenif [ $first -lt 13 ];thenecho '# HELP dcgm_fb_free Framebuffer memory free (in MiB).'echo '# TYPE dcgm_fb_free gauge'fioutput 'dcgm_fb_free' ${HOSTNAME} $count $uuid $ielif [ $num -eq 8 ];thenif [ $first -lt 13 ];thenecho '# HELP dcgm_gpu_utilization GPU utilization (in %).'echo '# TYPE dcgm_gpu_utilization gauge'fioutput 'dcgm_gpu_utilization' ${HOSTNAME} $count $uuid $ielif [ $num -eq 13 ];thenif [ $first -le 13 ];thenecho '# HELP dcgm_gpu_temp GPU temperature (in C).'echo '# TYPE dcgm_gpu_temp gauge'fioutput 'dcgm_gpu_temp' ${HOSTNAME} $count $uuid $ifiif [ $num -eq 13 ];thennum=0else((num++))fi((first++))done}case $1 in"help")echo 'Useage: monitor.sh fast|mem|gpu|temp|all|[pathToLog sleepTimeNum]';;"mem")mem_prm     mem_detail_prm;;"gpu")gpu_prm;;"temp")temp_prm;;"fast")fast;;"all")allinone;;"onebyone")if [ ! -n "$1" ];thenif [ ! -d "/run/prometheus" ];thenmkdir -p /run/prometheusfiwhile true;do allinone > /run/prometheus/`hostname`_dcgm.prom;sleep 15;doneelseif [ ! -n "$2" ];thenwhile true;do allinone > $1;sleep 15;doneelsewhile true;do allinone > $1;sleep $2;donefifi;;*)if [ ! -n "$1" ];thenif [ ! -d "/run/prometheus" ];thenmkdir -p /run/prometheusfiwhile true;do fast > /run/prometheus/`hostname`_dcgm.prom;sleep 15;doneelseif [ ! -n "$2" ];thenwhile true;do fast > $1;sleep 15;doneelsewhile true;do fast > $1;sleep $2;donefifi;;esac

$ xfs_info /
meta-data=/dev/sda3              isize=512    agcount=4, agsize=3080064 blks=                       sectsz=512   attr=2, projid32bit=1=                       crc=1        finobt=0 spinodes=0
data     =                       bsize=4096   blocks=12320256, imaxpct=25=                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=1
log      =internal               bsize=4096   blocks=6015, version=2=                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0

mkfs.xfs -n ftype=1 /dev/sdxxxx

logvol / --fstype xfs --mkfsoptions='-n ftype=1'