Any zfs / zpool command hangs. I can't even cancel it
I don't know. I have 2 pools:
I noticed that a drive is causing issues the last months. I validated the health of the disk disk, replaced it and validated that drive s health. All good SMART-wise (and WD diagnostic tool - Kitfox - didn't say anything after full test). So I assumed it was the backplane in my server and replaced it with a new part. Still the same issue. I noticed the drive worked fine when connected without the backplane (direct sata + power). Today I reinstalled it via the backplane and wanted to do some diagnosing. Then this issue occured and the whole pool is unavailable for me now. Even after a complete reboot (with power loss and without) the pool is still "broken"?
root@pve-01:~# strace zpool status
execve("/usr/sbin/zpool", ["zpool", "status"], 0x7ffdfe252dc8 /* 19 vars */) = 0
brk(NULL) = 0x557b4e302000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x737ec2be1000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=43419, ...}) = 0
mmap(NULL, 43419, PROT_READ, MAP_PRIVATE, 3, 0) = 0x737ec2bd6000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libzfs.so.7", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=532528, ...}) = 0
mmap(NULL, 554256, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec2b4e000
mmap(0x737ec2b62000, 327680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x737ec2b62000
mmap(0x737ec2bb2000, 102400, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x64000) = 0x737ec2bb2000
mmap(0x737ec2bcb000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7d000) = 0x737ec2bcb000
mmap(0x737ec2bd0000, 21776, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x737ec2bd0000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libnvpair.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=104400, ...}) = 0
mmap(NULL, 106640, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec2b33000
mmap(0x737ec2b38000, 57344, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x737ec2b38000
mmap(0x737ec2b46000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13000) = 0x737ec2b46000
mmap(0x737ec2b4c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18000) = 0x737ec2b4c000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libuutil.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=64056, ...}) = 0
mmap(NULL, 70832, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec2b21000
mmap(0x737ec2b26000, 24576, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x737ec2b26000
mmap(0x737ec2b2c000, 16384, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x737ec2b2c000
mmap(0x737ec2b30000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe000) = 0x737ec2b30000
mmap(0x737ec2b32000, 1200, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x737ec2b32000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libudev.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=280936, ...}) = 0
mmap(NULL, 280904, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec2adc000
mmap(0x737ec2ae4000, 147456, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x737ec2ae4000
mmap(0x737ec2b08000, 77824, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2c000) = 0x737ec2b08000
mmap(0x737ec2b1b000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3f000) = 0x737ec2b1b000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=977112, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x737ec2ada000
mmap(NULL, 978968, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec29ea000
mmap(0x737ec29fb000, 512000, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x11000) = 0x737ec29fb000
mmap(0x737ec2a78000, 393216, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8e000) = 0x737ec2a78000
mmap(0x737ec2ad8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xed000) = 0x737ec2ad8000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libblkid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=392160, ...}) = 0
mmap(NULL, 390016, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec298a000
mmap(0x737ec2994000, 245760, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x737ec2994000
mmap(0x737ec29d0000, 77824, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x46000) = 0x737ec29d0000
mmap(0x737ec29e3000, 28672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x59000) = 0x737ec29e3000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libuuid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=39048, ...}) = 0
mmap(NULL, 36936, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec2980000
mmap(0x737ec2982000, 20480, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x737ec2982000
mmap(0x737ec2987000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x737ec2987000
mmap(0x737ec2988000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x737ec2988000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\236\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 840, 64) = 840
fstat(3, {st_mode=S_IFREG|0755, st_size=1995216, ...}) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 840, 64) = 840
mmap(NULL, 2047568, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec278c000
mmap(0x737ec27b4000, 1454080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x28000) = 0x737ec27b4000
mmap(0x737ec2917000, 352256, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18b000) = 0x737ec2917000
mmap(0x737ec296d000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e0000) = 0x737ec296d000
mmap(0x737ec2973000, 52816, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x737ec2973000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libzfs_core.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=59552, ...}) = 0
mmap(NULL, 65936, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec277b000
mmap(0x737ec2780000, 24576, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x737ec2780000
mmap(0x737ec2786000, 12288, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x737ec2786000
mmap(0x737ec2789000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x737ec2789000
mmap(0x737ec278b000, 400, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x737ec278b000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libcrypto.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=6517312, ...}) = 0
mmap(NULL, 6528072, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec2141000
mmap(0x737ec2238000, 3682304, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf7000) = 0x737ec2238000
mmap(0x737ec25bb000, 1273856, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x47a000) = 0x737ec25bb000
mmap(0x737ec26f2000, 548864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5b1000) = 0x737ec26f2000
mmap(0x737ec2778000, 11336, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x737ec2778000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=125376, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x737ec213f000
mmap(NULL, 127376, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec211f000
mmap(0x737ec2122000, 81920, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x737ec2122000
mmap(0x737ec2136000, 28672, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x737ec2136000
mmap(0x737ec213d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1d000) = 0x737ec213d000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libtirpc.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=195024, ...}) = 0
mmap(NULL, 200336, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec20ee000
mmap(0x737ec20f6000, 114688, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x737ec20f6000
mmap(0x737ec2112000, 40960, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x24000) = 0x737ec2112000
mmap(0x737ec211c000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2d000) = 0x737ec211c000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libcap.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360z\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=47288, ...}) = 0
mmap(NULL, 45128, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec20e2000
mmap(0x737ec20e5000, 20480, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x737ec20e5000
mmap(0x737ec20ea000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x737ec20ea000
mmap(0x737ec20ec000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x737ec20ec000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libzstd.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=825336, ...}) = 0
mmap(NULL, 823352, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec2018000
mmap(0x737ec201d000, 716800, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x737ec201d000
mmap(0x737ec20cc000, 81920, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb4000) = 0x737ec20cc000
mmap(0x737ec20e0000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xc8000) = 0x737ec20e0000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=346904, ...}) = 0
mmap(NULL, 349472, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec1fc2000
mmap(0x737ec1fce000, 229376, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xc000) = 0x737ec1fce000
mmap(0x737ec2006000, 57344, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x44000) = 0x737ec2006000
mmap(0x737ec2014000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x51000) = 0x737ec2014000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libkrb5.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=883984, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x737ec1fc0000
mmap(NULL, 882592, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec1ee8000
mmap(0x737ec1f0b000, 385024, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x737ec1f0b000
mmap(0x737ec1f69000, 290816, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x81000) = 0x737ec1f69000
mmap(0x737ec1fb0000, 65536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xc8000) = 0x737ec1fb0000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libk5crypto.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=187144, ...}) = 0
mmap(NULL, 185048, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec1eba000
mmap(0x737ec1ebe000, 110592, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x737ec1ebe000
mmap(0x737ec1ed9000, 53248, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f000) = 0x737ec1ed9000
mmap(0x737ec1ee6000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2c000) = 0x737ec1ee6000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libcom_err.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18344, ...}) = 0
mmap(NULL, 20552, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec1eb4000
mmap(0x737ec1eb6000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x737ec1eb6000
mmap(0x737ec1eb7000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x737ec1eb7000
mmap(0x737ec1eb8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x737ec1eb8000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libkrb5support.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=52208, ...}) = 0
mmap(NULL, 54600, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec1ea6000
mmap(0x737ec1ea9000, 24576, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x737ec1ea9000
mmap(0x737ec1eaf000, 12288, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x9000) = 0x737ec1eaf000
mmap(0x737ec1eb2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x737ec1eb2000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libkeyutils.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=22448, ...}) = 0
mmap(NULL, 24592, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec1e9f000
mmap(0x737ec1ea1000, 8192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x737ec1ea1000
mmap(0x737ec1ea3000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x737ec1ea3000
mmap(0x737ec1ea4000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x737ec1ea4000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libresolv.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=63936, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x737ec1e9d000
mmap(NULL, 71816, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x737ec1e8b000
mmap(0x737ec1e8e000, 36864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x737ec1e8e000
mmap(0x737ec1e97000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xc000) = 0x737ec1e97000
mmap(0x737ec1e99000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe000) = 0x737ec1e99000
mmap(0x737ec1e9b000, 6280, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x737ec1e9b000
close(3) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x737ec1e89000
mmap(NULL, 32768, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x737ec1e81000
arch_prctl(ARCH_SET_FS, 0x737ec1e86880) = 0
set_tid_address(0x737ec1e86b50) = 26215
set_robust_list(0x737ec1e86b60, 24) = 0
rseq(0x737ec1e81680, 0x21, 0, 0x53053053) = 0
mprotect(0x737ec296d000, 16384, PROT_READ) = 0
mprotect(0x737ec1e99000, 4096, PROT_READ) = 0
mprotect(0x737ec1ea4000, 4096, PROT_READ) = 0
mprotect(0x737ec1eb2000, 4096, PROT_READ) = 0
mprotect(0x737ec1eb8000, 4096, PROT_READ) = 0
mprotect(0x737ec1ee6000, 4096, PROT_READ) = 0
mprotect(0x737ec1fb0000, 53248, PROT_READ) = 0
mprotect(0x737ec2014000, 8192, PROT_READ) = 0
mprotect(0x737ec20e0000, 4096, PROT_READ) = 0
mprotect(0x737ec20ec000, 4096, PROT_READ) = 0
mprotect(0x737ec211c000, 8192, PROT_READ) = 0
mprotect(0x737ec213d000, 4096, PROT_READ) = 0
mprotect(0x737ec26f2000, 536576, PROT_READ) = 0
mprotect(0x737ec2b4c000, 4096, PROT_READ) = 0
mprotect(0x737ec2789000, 4096, PROT_READ) = 0
mprotect(0x737ec2988000, 4096, PROT_READ) = 0
mprotect(0x737ec29e3000, 24576, PROT_READ) = 0
mprotect(0x737ec2ad8000, 4096, PROT_READ) = 0
mprotect(0x737ec2b1b000, 20480, PROT_READ) = 0
mprotect(0x737ec2b30000, 4096, PROT_READ) = 0
mprotect(0x737ec2bcb000, 16384, PROT_READ) = 0
mprotect(0x557b4436a000, 8192, PROT_READ) = 0
mprotect(0x737ec2c1f000, 8192, PROT_READ) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
munmap(0x737ec2bd6000, 43419) = 0
prctl(PR_CAPBSET_READ, CAP_MAC_OVERRIDE) = 1
prctl(PR_CAPBSET_READ, 0x30 /* CAP_??? */) = -1 EINVAL (Invalid argument)
prctl(PR_CAPBSET_READ, CAP_CHECKPOINT_RESTORE) = 1
prctl(PR_CAPBSET_READ, 0x2c /* CAP_??? */) = -1 EINVAL (Invalid argument)
prctl(PR_CAPBSET_READ, 0x2a /* CAP_??? */) = -1 EINVAL (Invalid argument)
prctl(PR_CAPBSET_READ, 0x29 /* CAP_??? */) = -1 EINVAL (Invalid argument)
getrandom("\x0a\xc3\x7e\x0d\x5a\xfe\x42\xfe", 8, GRND_NONBLOCK) = 8
brk(NULL) = 0x557b4e302000
brk(0x557b4e323000) = 0x557b4e323000
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=3063024, ...}) = 0
mmap(NULL, 3063024, PROT_READ, MAP_PRIVATE, 3, 0) = 0x737ec1b95000
close(3) = 0
access("/dev/zfs", F_OK) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/gconv/gconv-modules.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=27028, ...}) = 0
mmap(NULL, 27028, PROT_READ, MAP_SHARED, 3, 0) = 0x737ec2bda000
close(3) = 0
futex(0x737ec297272c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
openat(AT_FDCWD, "/dev/zfs", O_RDWR|O_EXCL|O_CLOEXEC) = 3
openat(AT_FDCWD, "/dev/zfs", O_RDWR|O_CLOEXEC) = 4
openat(AT_FDCWD, "/sys/module/zfs/properties.dataset", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 5
fstat(5, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
getdents64(5, 0x557b4e305f70 /* 108 entries */, 32768) = 3624
getdents64(5, 0x557b4e305f70 /* 0 entries */, 32768) = 0
close(5) = 0
openat(AT_FDCWD, "/sys/module/zfs/properties.pool", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 5
fstat(5, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
getdents64(5, 0x557b4e305f70 /* 42 entries */, 32768) = 1344
getdents64(5, 0x557b4e305f70 /* 0 entries */, 32768) = 0
close(5) = 0
openat(AT_FDCWD, "/sys/module/zfs/features.pool", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 5
fstat(5, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
getdents64(5, 0x557b4e305f70 /* 49 entries */, 32768) = 2352
getdents64(5, 0x557b4e305f70 /* 0 entries */, 32768) = 0
close(5) = 0
openat(AT_FDCWD, "/sys/module/zfs/properties.vdev", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 5
fstat(5, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
getdents64(5, 0x557b4e305f70 /* 57 entries */, 32768) = 1784
getdents64(5, 0x557b4e305f70 /* 0 entries */, 32768) = 0
close(5) = 0
mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x737ec1b54000
ioctl(3, ZFS_IOC_POOL_CONFIGS
System information
Describe the problem you're observing
Any zfs / zpool command hangs. I can't even cancel it
Describe how to reproduce the problem
I don't know. I have 2 pools:
I noticed that a drive is causing issues the last months. I validated the health of the disk disk, replaced it and validated that drive s health. All good SMART-wise (and WD diagnostic tool - Kitfox - didn't say anything after full test). So I assumed it was the backplane in my server and replaced it with a new part. Still the same issue. I noticed the drive worked fine when connected without the backplane (direct sata + power). Today I reinstalled it via the backplane and wanted to do some diagnosing. Then this issue occured and the whole pool is unavailable for me now. Even after a complete reboot (with power loss and without) the pool is still "broken"?
Include any warning/errors/backtraces from the system logs
dmesg
strace zpool status
Additional notes
rpool