setns对当前进程无效问题的排查(getpid获取值不变)

1）复现流程及lxc的处理

demo1程序与执行结果如下，此时在容器内部看不到执行的程序。

int main()
{int ret, fd, pid;printf("father pid old:%d\n", getpid());fd = open("/dev/ns", O_RDWR);ret = ioctl(fd, 24635); // parm is dst ns process's pidprintf("father pid old:%d\n", getpid());sleep(5);return 0;
}

# ./a.out
father pid old:26169
father pid old:26169

demo2程序与执行结果如下，此时容器内还是看不到执行程序，但是这里getpid()获取到的值就为0了，对比上边，世界上没有这么玄乎的事，或许是缓存的问题？这是第一个疑问。

int main()
{int ret, fd, pid;fd = open("/dev/ns", O_RDWR);ret = ioctl(fd, 24635); // parm is dst ns process's pidprintf("father pid old:%d\n", getpid());sleep(5);return 0;
}

# ./a.out
father pid old:0

demo3程序与执行结果如下，这时在容器内部能看到子进程，为什么子进程实现了pid ns的切换，父进程却没有实现？这是第二个疑问。

int main()
{int ret, fd, pid;fd = open("/dev/ns", O_RDWR);ret = ioctl(fd, 24635); // parm is dst ns process's pidprintf("father pid old:%d\n", getpid());pid = fork();if(0 == pid) {printf("son pid:%d\n", getpid());sleep(5);} else {printf("father pid:%d\n", getpid());wait();}return 0;
}

# ./a.out
father pid old:0
father pid:0
son pid:87PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND                                         1 root      20   0   22036   2144   1624 S   0.0  0.1   0:00.01 bash                                            57 root      20   0   23672   1516   1160 R   0.0  0.1   0:00.66 top                                             87 root      20   0    3760    204    120 S   0.0  0.0   0:00.00 a.out

lxc attach流程如下，实现思路和我们上边的测试demo一致，看来这确实是linux机制的问题。

setns()
pid = fork();
if(!pid) { // son
execve(bash);
}

2）pid缓存问题

编写了demo4：

int main()
{int ret, fd, pid;printf("father pid:%d\n", getpid());printf("father pid:%d\n", getpid());printf("father pid:%d\n", getpid());printf("father pid:%d\n", getpid());fd = open("/dev/ns", O_RDWR);ret = ioctl(fd, 24635); // parm is dst ns process's pidprintf("father pid old:%d\n", getpid());pid = fork();if(0 == pid) {printf("son pid:%d\n", getpid());sleep(5);} else {printf("father pid:%d\n", getpid());wait();}return 0;
}

看下结果，fork前后打印一致

# ./a.out
father pid:23246
father pid:23246
father pid:23246
father pid:23246
father pid old:23246
father pid:23246
son pid:96

strace看下，果然和我们猜的一样，除了第一次getpid调用了syscall后，后边所有的返回值均是从缓存中获取的，所以这就能解释为什么不执行getpid并fork后执行getpid获取到的返回值是0，而执行了getpid并fork后再执行getpid获取到的返回值不变。

# strace ./a.out
execve("./a.out", ["./a.out"], [/* 29 vars */]) = 0
brk(0)                                  = 0x18f4000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa04d79c000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=105026, ...}) = 0
mmap(NULL, 105026, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fa04d782000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/libc.so.6", O_RDONLY)        = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\356\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1478056, ...}) = 0
mmap(NULL, 3586120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa04d215000
mprotect(0x7fa04d377000, 2097152, PROT_NONE) = 0
mmap(0x7fa04d577000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x162000) = 0x7fa04d577000
mmap(0x7fa04d57c000, 18504, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa04d57c000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa04d781000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa04d780000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa04d77f000
arch_prctl(ARCH_SET_FS, 0x7fa04d780700) = 0
mprotect(0x7fa04d577000, 16384, PROT_READ) = 0
mprotect(0x7fa04d79e000, 4096, PROT_READ) = 0
munmap(0x7fa04d782000, 105026)          = 0
getpid()                                = 23253
fstat(1, {st_mode=S_IFCHR|0600, st_rdev=makedev(136, 4), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa04d79b000
write(1, "father pid:23253\n", 17father pid:23253
)      = 17
write(1, "father pid:23253\n", 17father pid:23253
)      = 17
write(1, "father pid:23253\n", 17father pid:23253
)      = 17
write(1, "father pid:23253\n", 17father pid:23253
)      = 17
open("/dev/ns", O_RDWR)                 = 3
ioctl(3, 0x603b, 0x7fa04d57cdf0)        = 0
write(1, "father pid old:23253\n", 21father pid old:23253
)  = 21
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7fa04d7809d0) = 97
write(1, "father pid:23253\n", 17father pid:23253
)      = 17
wait4(-1, son pid:97
0xffffffff, 0, NULL)          = -1 EFAULT (Bad address)
--- SIGCHLD (Child exited) @ 0 (0) ---

3）为什么子进程实现了pid ns的切换，父进程却没有实现？

先来看看setns的实现，在2.6.32中要支持setns很简单首先通过不创建新NS的方式调用copy_namespaces并传入dst ns，这会增加dst ns的引用，之后通过switch_task_namespaces传入current task与dst ns，在函数中会首先进行nsproxy指针的交换，将当前task切换到dst ns中，之后src ns减引用，这样就能保证引用数量的正确。因此这里实际上切换了current task的nsproxy。

copy_namespaces->switch_task_namespaces

int copy_namespaces(unsigned long flags, struct task_struct *tsk)
{struct nsproxy *old_ns = tsk->nsproxy;struct nsproxy *new_ns;int err = 0;if (!old_ns)return 0;get_nsproxy(old_ns);if (!(flags & (CLONE_NEWNS | CLONE_NEWUTS | CLONE_NEWIPC |CLONE_NEWPID | CLONE_NEWNET)))return 0;
。。。
}void switch_task_namespaces(struct task_struct *p, struct nsproxy *new)
{struct nsproxy *ns;might_sleep();ns = p->nsproxy;rcu_assign_pointer(p->nsproxy, new);if (ns && atomic_dec_and_test(&ns->count)) {/** wait for others to get what they want from this nsproxy.** cannot release this nsproxy via the call_rcu() since* put_mnt_ns() will want to sleep*/synchronize_rcu();free_nsproxy(ns);}
}

再看看getpid的调用流程(sys_getpid–>task_tgid_vnr->pid_vnr->pid_nr_ns)，这里考虑线程问题通过task_tgid获取该PID的tgid，因为应用层和内核对PID的定义不同，内核中进程与线程都拥有相同的结构体描述struct task_struct，因此进程与线程在内核中均拥有自己独立的PID，因此这时候找到“用户态PID”的关键是找到根进程，因为父根进程的pid与tgid是一致如下，因此在这里传入tgid来代替父根进程的PID。在pid_nr_ns中会进行两个判断，一是该pid ns的level应大于指定pid ns的level，这里的pid ns level如下图所示是一个树状结构，这里default pid ns中的pid level为0，而基于某进程创建的CLONE_NEWPID的进程，其pid ns level + 1，因此这里的判断会出现问题，因为setns只修改了current->nsproxy，如果是在default pid ns中执行的demo程序以及lxc程序，那么current->nsproxy->pid_ns->level = 1，而pid->level = 0，这会导致判断失败直接返回0，这也是我们在demo程序中通过getpid()得到0的原因。

 <-- PID 43 --> <----------------- PID 42 ----------------->+---------+| process |_| pid=42  |__/ | tgid=42 | \_ (new thread) __ (fork) _/   +---------+                  \/                                        +---------+
+---------+                                    | process |
| process |                                    | pid=44  |
| pid=43  |                                    | tgid=42 |
| tgid=43 |                                    +---------+
+---------+<-- PID 43 --> <--------- PID 42 --------> <--- PID 44 --->

SYSCALL_DEFINE0(getpid)
{return task_tgid_vnr(current);
}static inline pid_t task_tgid_vnr(struct task_struct *tsk)
{return pid_vnr(task_tgid(tsk));
}pid_t pid_vnr(struct pid *pid)
{return pid_nr_ns(pid, current->nsproxy->pid_ns);
}pid_t pid_nr_ns(struct pid *pid, struct pid_namespace *ns)
{struct upid *upid;pid_t nr = 0;if (pid && ns->level <= pid->level) {upid = &pid->numbers[ns->level];if (upid->ns == ns)nr = upid->nr;}return nr;
}

如下，这里有对获取到的upid->nr的一个很好的解释，也就是说upid是pid ns有效的，另外这个实验是在2.6.32.5中实现的，而2.6.32.5中不支持setns(我对它进行了移植)，或许在3.x的某些支持setns的版本中也能复现，但4.9.0中pid_nr_ns的不同实现导致了实验效果的不同，在4.9.0中demo程序会返回default ns中的pid。

/** struct upid is used to get the id of the struct pid, as it is* seen in particular namespace. Later the struct pid is found with* find_pid_ns() using the int nr and struct pid_namespace *ns.*/
struct upid {/* Try to keep pid_chain in the same cacheline as nr for find_vpid */int nr;struct pid_namespace *ns;struct hlist_node pid_chain;
};

最后再补充下，前面可以看到setns实现其实只修改了task_struct中的nsproxy，因此fork会通过调用链do_fork->copy_process->alloc_pid(p->nsproxy->pid_ns)也就是利用父进程的nsproxy->pid_ns来创建自己的pid，这样就说明了为什么lxc中必须fork出子进程来执行execve。