On Wed, 2021-11-24 at 10:32 +0000, Mel Gorman wrote:

On Tue, Nov 23, 2021 at 05:19:12PM -0800, Darrick J. Wong wrote:

quoted

AFAICT the system is mostly idle, but it's difficult to tell because ps
and top also get stuck waiting for this cgroup for whatever reason.

But this is surprising because I expect that ps and top are not running
within the cgroup. Was /proc/PID/stack readable?

Probably this.

crash> ps | grep UN
   4418   4417   4  ffff8881cae66e40  UN   0.0    7620    980  memcg_test_1 <== the bad guy
   4419   4417   6  ffff8881cae62f40  UN   0.0    7620    980  memcg_test_1
   4420   4417   5  ffff8881cae65e80  UN   0.0    7620    980  memcg_test_1
   4421   4417   7  ffff8881cae63f00  UN   0.0    7620    980  memcg_test_1
   4422   4417   4  ffff8881cae60000  UN   0.0    7620    980  memcg_test_1
   4423   4417   3  ffff888128985e80  UN   0.0    7620    980  memcg_test_1
   4424   4417   7  ffff888117f79f80  UN   0.0    7620    980  memcg_test_1
   4425   4417   2  ffff888117f7af40  UN   0.0    7620    980  memcg_test_1
   4428   2791   6  ffff8881a8253f00  UN   0.0   38868   3568  ps
   4429   2808   4  ffff888100c90000  UN   0.0   38868   3600  ps
crash> bt -sx 4429
PID: 4429   TASK: ffff888100c90000  CPU: 4   COMMAND: "ps"
 #0 [ffff8881af1c3ce0] __schedule+0x285 at ffffffff817ae6c5
 #1 [ffff8881af1c3d68] schedule+0x3a at ffffffff817aed4a
 #2 [ffff8881af1c3d78] rwsem_down_read_slowpath+0x197 at ffffffff817b11a7
 #3 [ffff8881af1c3e08] down_read_killable+0x5c at ffffffff817b142c
 #4 [ffff8881af1c3e18] down_read_killable+0x5c at ffffffff817b142c
 #5 [ffff8881af1c3e28] __access_remote_vm+0x3f at ffffffff8120131f
 #6 [ffff8881af1c3e90] proc_pid_cmdline_read+0x148 at ffffffff812fc9a8
 #7 [ffff8881af1c3ee8] vfs_read+0x92 at ffffffff8126a302
 #8 [ffff8881af1c3f00] ksys_read+0x7d at ffffffff8126a72d
 #9 [ffff8881af1c3f38] do_syscall_64+0x37 at ffffffff817a3f57
#10 [ffff8881af1c3f50] entry_SYSCALL_64_after_hwframe+0x44 at ffffffff8180007c
    RIP: 00007f4b50fe8b5e  RSP: 00007ffdd7f6fe38  RFLAGS: 00000246
    RAX: ffffffffffffffda  RBX: 00007f4b5186a010  RCX: 00007f4b50fe8b5e
    RDX: 0000000000020000  RSI: 00007f4b5186a010  RDI: 0000000000000006
    RBP: 0000000000020000   R8: 0000000000000007   R9: 00000000ffffffff
    R10: 0000000000000000  R11: 0000000000000246  R12: 00007f4b5186a010
    R13: 0000000000000000  R14: 0000000000000006  R15: 0000000000000000
    ORIG_RAX: 0000000000000000  CS: 0033  SS: 002b
crash> mm_struct -x ffff8881021b4800
struct mm_struct {
  {
    mmap = 0xffff8881ccfe6a80,
    mm_rb = {
      rb_node = 0xffff8881ccfe61a0
    },
...
    mmap_lock = {
      count = {
        counter = 0x3
      },
      owner = {
        counter = 0xffff8881cae66e40
...
crash> bt 0xffff8881cae66e40
PID: 4418   TASK: ffff8881cae66e40  CPU: 4   COMMAND: "memcg_test_1"
 #0 [ffff888154097a88] __schedule at ffffffff817ae6c5
 #1 [ffff888154097b10] schedule at ffffffff817aed4a
 #2 [ffff888154097b20] schedule_timeout at ffffffff817b311f
 #3 [ffff888154097b90] reclaim_throttle at ffffffff811d802b
 #4 [ffff888154097bf0] do_try_to_free_pages at ffffffff811da206
 #5 [ffff888154097c40] try_to_free_mem_cgroup_pages at ffffffff811db522
 #6 [ffff888154097cd0] try_charge_memcg at ffffffff81256440
 #7 [ffff888154097d60] obj_cgroup_charge_pages at ffffffff81256c97
 #8 [ffff888154097d88] obj_cgroup_charge at ffffffff8125898c
 #9 [ffff888154097da8] kmem_cache_alloc at ffffffff81242099
#10 [ffff888154097de0] vm_area_alloc at ffffffff8106c87a
#11 [ffff888154097df0] mmap_region at ffffffff812082b2
#12 [ffff888154097e58] do_mmap at ffffffff81208922
#13 [ffff888154097eb0] vm_mmap_pgoff at ffffffff811e259f
#14 [ffff888154097f38] do_syscall_64 at ffffffff817a3f57
#15 [ffff888154097f50] entry_SYSCALL_64_after_hwframe at
ffffffff8180007c
    RIP: 00007f211c36b743  RSP: 00007ffeaac1bd58  RFLAGS: 00000246
    RAX: ffffffffffffffda  RBX: 0000000000000000  RCX: 00007f211c36b743
    RDX: 0000000000000003  RSI: 0000000000001000  RDI: 0000000000000000
    RBP: 0000000000000000   R8: 0000000000000000   R9: 0000000000000000
    R10: 0000000000002022  R11: 0000000000000246  R12: 0000000000000003
    R13: 0000000000001000  R14: 0000000000002022  R15: 0000000000000000
    ORIG_RAX: 0000000000000009  CS: 0033  SS: 002b
crash>