OpenMMLab使用北京超算云计算中心进行图像分类

1.环境配置

根据 mmclassification 的环境要求，需要⽤ anaconda、cuda、gcc 等基础环境模块。在 N30 分区可以使⽤ module avail 命令可以使⽤模块信息

1 $ module avail
--------------------------------------------------------------------------------
------- /usr/share/Modules/modulefiles -----------------------------------------
-----------------------------------------------
2
3 dot module-git module-info modules null use.own
4
--------------------------------------------------------------------------------
----------- /data/apps/modulefiles ---------------------------------------------
-----------------------------------------------
5
alphafold/2.0.1 fftw/3.3.9-ompi-float lapack/3.10.1 openmpi/4.1.1_ucx1.9
6
alphafold/2.1.1 gcc/11.2 libevent/2.1.12
openmpi/4.1.1_ucx1.9_cuda11.2
7
alphafold/parafold-2.1.1 gcc/5.4 localcolabfold/1.4 p7zip/16.02
8
anaconda/2020.11 gcc/6.3 namd2/2.14-verbs-linux-x86_64-gcc-smp-CUDA p7zip/21.02
9
anaconda/2021.05 gcc/7.3 nccl/2.11.4-1_cuda11.1 plumed/2.7.2
10
anaconda/2022.10 gcc/8.3 nccl/2.11.4-1_cuda11.2 pmix/3.2.2
11
arias/1.36.0 gcc/9.3 nccl/2.11.4-1_cuda11.4
pnetcdf/1.12.2/openmpi_gcc9.3
12
blas/3.10.0 go/1.18.2 netcdf-c/4.8.1/openmpi_gcc9.3
qe/6.8_nvhpc21.9_openmpi4.0.5_cuda11.4_ucx1.9
13
cmake/3.22.0 gromacs/2021.2_nompi nvhpc/21.5 rar/611
14
cp2k/9.1_openmpi-414_gcc-8.3_cuda-11.3 gromacs/2021.2-parallel nvhpc/21.9 singularity/2.6.0
15
cuda/11.1 gromacs/2021.2-plumed_nompi nvhpc/nvhpc-byo-compiler/21.5 singularity/3.10.0
16
cuda/11.2 gromacs/2021.5_dev_fep_mpi nvhpc/nvhpc-byo-compiler/21.9 singularity/3.9.9
17
cuda/11.3 gromacs/2022_nompi nvhpc/nvhpc-nompi/21.5 SPONGE/1.2.6
18
cuda/11.4 hdf5/1.12.1 nvhpc/nvhpc-nompi/21.9 tensorboard/2.3.0
19
cuda/11.6 hwloc/2.1.0 nvhpc-byo-compiler/21.5 ucx/1.8
20
cuda/11.7 intel/parallelstudio/2017.1.5 nvhpc-byo-compiler/21.9 ucx/1.9
21
cudnn/8.2.1_cuda11.x intel/parallelstudio/2019.3.0 nvhpc-nompi/21.5 ucx/1.9_cuda11.2
22
cudnn/8.2.4_cuda11.4 intel/parallelstudio/2021.1.1 nvhpc-nompi/21.9 ucx/1.9_cuda11.4
23
cudnn/8.5.0_cuda11.x jupyter/lab oneFlow/0.8.0 zlib/1.2.11
24
dos2unix/6.0.3 jupyter/notebook openmpi/4.0.5_nvhpc21.9_ucx1.9_cuda11.4
25
fftw/3.3.9 lammps/27May2021 openmpi/4.1.1

1. 加载 anaconda ，创建⼀个 python 3.8 的环境。

1 # 加载 anaconda/2021.05
2 module load anaconda/2021.05
3
4 # 创建 python=3.8 的环境
5 conda create --name opennmmlab_mmclassification python=3.8
6
7 # 激活环境
8 source activate opennmmlab_mmclassification

2. 安装 torch，torch 参考需求。注意在 RTX3090 的GPU上，cuda 版本需要 ≥ 11.1 。如下安装的 torch 是 1.10.0+cu111 。使⽤ pip 安装的torch 不包括 cuda，所以需要使⽤ module 加载 cuda/11.1 模块。

1 # 加载 cuda/11.1
2 module load cuda/11.1
3
4 # 安装 torch
pip install torch==1.10.0+cu111 torchvision==0.11.0+cu111 torchaudio==0.10.0 -f
https://download.pytorch.org/whl/torch_stable.html

3. 安装 mmcv-full 模块，mmcv-full 模块安装时候需要注意 torch 和 cuda 版本。

pip install mmcv-full==1.7.0 -f
https://download.openmmlab.com/mmcv/dist/cu111/torch1.10/index.html

4. 安装 openmmlab/mmclassification 模块，建议通过下载编译的⽅式进⾏安装；安装该模块需要 gcc ≥ 5，使⽤ module 加载⼀个 gcc ，例如 module load gcc/7.3 。

1 # 加载 gcc/7.3 模块
2 module load gcc/7.3
3
4 # git 下载 mmclassification 代码
5 git clone https://github.com/open-mmlab/mmclassification.git
6
7 # 编译安装
8 cd mmclassification
9 pip install -e .

5. 准备 shell 脚本，将环境信息预先保存在脚本中。

1 #!/bin/bash
2 # 加载模块
3 module load anaconda/2021.05
4 module load cuda/11.1
5 module load gcc/7.3
6
7 # 激活环境
8 source activate opennmmlab_mmclassification

2.数据集

flower 数据集包含 5 种类别的花卉图像:雏菊 daisy 588张，蒲公英 dandelion 556张，玫瑰 rose 583张，向⽇葵 sunflower 536张，郁⾦⾹ tulip 585张。数据集下载链接:

国际⽹

国内⽹提取码: 9x5u

2.1 划分数据集

将数据集按照 8:2 的⽐例划分成训练和验证⼦数据集，并将数据集整理成 ImageNet的格式将训练⼦集和验证⼦集放到 train 和 val ⽂件夹下。

数据集划分代码 split_data.py 如下，执⾏：

python split_data.py [源数据集路径] [⽬标数据集路径]

1 import os
2 import sys
3 import shutil
4 import numpy as np
5
6
7 def load_data(data_path):
8 count = 0
9 data = {}
10 for dir_name in os.listdir(data_path):
11 dir_path = os.path.join(data_path, dir_name)
12 if not os.path.isdir(dir_path):
13 continue
14
15 data[dir_name] = []
16 for file_name in os.listdir(dir_path):
17 file_path = os.path.join(dir_path, file_name)
18 if not os.path.isfile(file_path):
19 continue
20 data[dir_name].append(file_path)
21
22 count += len(data[dir_name])
23 print("{} :{}".format(dir_name, len(data[dir_name])))
24
25 print("total of image : {}".format(count))
26 return data
27
28
29 def copy_dataset(src_img_list, data_index, target_path):
30 target_img_list = []
31 for index in data_index:
32 src_img = src_img_list[index]
33 img_name = os.path.split(src_img)[-1]
34
35 shutil.copy(src_img, target_path)
36 target_img_list.append(os.path.join(target_path, img_name))
37 return target_img_list
38
39
40 def write_file(data, file_name):
41 if isinstance(data, dict):
42 write_data = []
43 for lab, img_list in data.items():
44 for img in img_list:
45 write_data.append("{} {}".format(img, lab))
46 else:
47 write_data = data
48
49 with open(file_name, "w") as f:
50 for line in write_data:
51 f.write(line + "\n")
52
53 print("{} write over!".format(file_name))
54
55
56 def split_data(src_data_path, target_data_path, train_rate=0.8):
57 src_data_dict = load_data(src_data_path)
58
59 classes = []
60 train_dataset, val_dataset = {}, {}
61 train_count, val_count = 0, 0
62 for i, (cls_name, img_list) in enumerate(src_data_dict.items()):
63 img_data_size = len(img_list)random_index = np.random.choice(img_data_size, img_data_size,
replace=False)
64
65
66 train_data_size = int(img_data_size * train_rate)
67 train_data_index = random_index[:train_data_size]
68 val_data_index = random_index[train_data_size:]
69
70 train_data_path = os.path.join(target_data_path, "train", cls_name)
71 val_data_path = os.path.join(target_data_path, "val", cls_name)
72 os.makedirs(train_data_path, exist_ok=True)
73 os.makedirs(val_data_path, exist_ok=True)
74
75 classes.append(cls_name)train_dataset[i] = copy_dataset(img_list, train_data_index,
train_data_path)
76
77 val_dataset[i] = copy_dataset(img_list, val_data_index, val_data_path)
78print("target {} train:{}, val:{}".format(cls_name,
len(train_dataset[i]), len(val_dataset[i])))
79
80 train_count += len(train_dataset[i])
81 val_count += len(val_dataset[i])
82print("train size:{}, val size:{}, total:{}".format(train_count, val_count,
train_count + val_count))
83
84
85 write_file(classes, os.path.join(target_data_path, "classes.txt"))
86 write_file(train_dataset, os.path.join(target_data_path, "train.txt"))
87 write_file(val_dataset, os.path.join(target_data_path, "val.txt"))
88
89
90 def main():
91 src_data_path = sys.argv[1]
92 target_data_path = sys.argv[2]
93 split_data(src_data_path, target_data_path, train_rate=0.8)
94
95
96 if __name__ == '__main__':
97 main()

3. MMCls 配置⽂件

构建配置⽂件可以使⽤继承机制，从 configs/__base__ 中继承 ImageNet 预训练的任何模型，ImageNet 的数据集配置，学习率策略等。

如下内容可命名为 resnet18_b32_flower.py，在 mmclassification/configs 下创建 resnet18 ⽬录，将该⽂件放到⾥⾯。

_base_ = ['../_base_/models/resnet18.py', '../_base_/datasets/imagenet_bs32.py',
'../_base_/default_runtime.py']
1
2
3 model = dict(
4 head=dict(
5 num_classes=5,
6 topk = (1,)
7 ))
8
9 data = dict(
10 samples_per_gpu = 32,
11 workers_per_gpu = 2,
12 train = dict(
13 data_prefix = 'data/flower/train',
14 ann_file = 'data/flower/train.txt',
15 classes = 'data/flower/classes.txt'
16 ),
17 val = dict(
18 data_prefix = 'data/flower/val',
19 ann_file = 'data/flower/val.txt',
20 classes = 'data/flower/classes.txt'
21 )
22 )
23
24 optimizer = dict(type='SGD', lr=0.001, momentum=0.9, weight_decay=0.0001)
25 optimizer_config = dict(grad_clip=None)
26
27 lr_config = dict(
28 policy='step',
29 step=[1])
30
31 runner = dict(type='EpochBasedRunner', max_epochs=100)
32
33 # 预训练模型
load_from =
'/HOME/shenpg/run/openmmlab/mmclassification/checkpoints/resnet18_batch256_image
net_20200708-34ab8f90.pth'