Crash during sustained writing

[strainu]

Hi fellow Synology lovers,

I setup an xpenology VM on Proxmox using ARPL. The machine has a J6413 CPU and 2x12TB Toshiba N300 disks in RAID 1 plus another NVME drive I'm not using for now. Everything seemed alright until I started to copy my data from the old Xpenology server (DSM6). I tried using HyperBackup with an rsync  AND with a Synology (i.e. HyperBackup Vault) endpoints, as well as SMB transfer via a Windows machine. In all scenarios, after about 10 minutes of sustained transfer, the destination crashes - you can see an example backtrace below.

 

It's pretty obvious to me this is a BTRFS issue, but I can't find anything relevant on Google. What can I try to prevent this crash, except for rebuilding my volume with ext4? Thanks!

 

[ 1253.907004] invalid opcode: 0000 [#1] SMP
[ 1253.907004] Modules linked in: fuse 8021q vfat fat vhost_scsi(O) vhost(O) tcm_loop(O) iscsi_target_mod(O) target_core_user(O) target_core_ep(O) target_core_multi_file(O) target_core_file(O) target_core_iblock(O) target_core_mod(O) syno_extent_pool(PO) rodsp_ep(O) udf isofs synoacl_vfs(O) btrfs ecryptfs zstd_decompress zstd_compress xxhash xor raid6_pq iptable_nat nf_nat_ipv4 nf_nat xt_conntrack xt_mac xt_geoip(O) compat_xtables(O) nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables xt_recent xt_iprange xt_limit xt_state xt_tcpudp xt_multiport xt_LOG nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack iptable_filter ip_tables x_tables zram leds_lp3943 geminilake_synobios(PO) usblp i915 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops cfbfillrect cfbcopyarea cfbimgblt drm drm_panel_orientation_quirks
[ 1253.907004]  iosf_mbi fb fbdev video backlight i2c_algo_bit r8168(O) sg dm_snapshot dm_bufio crc_itu_t crc_ccitt psnap p8022 llc hfsplus md4 hmac sit tunnel4 ipv6 flashcache_syno(O) flashcache(O) syno_flashcache_control(O) dm_mod arc4 cryptd ecb aes_x86_64 authenc des_generic ansi_cprng cts md5 cbc cpufreq_powersave cpufreq_performance cpufreq_stats vxlan ip6_udp_tunnel udp_tunnel ip_tunnel loop sha256_generic synorbd(O) synofsbd(O) usb_storage hid_generic usbhid hid etxhci_hcd e1000(OE) 8139cp(OE) mii(OE) virtio_pci(OE) virtio(OE) virtio_ring(OE) rtc_cmos(OE) processor button xhci_pci xhci_hcd uhci_hcd ehci_pci ehci_hcd usbcore usb_common [last unloaded: geminilake_synobios]
[ 1253.907004] CPU: 2 PID: 17382 Comm: rsync Tainted: P           OE   4.4.180+ #42962
[ 1253.907004] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.1-0-g3208b098f51a-prebuilt.qemu.org 04/01/2014
[ 1253.907004] task: ffff8800b93aa280 ti: ffff8803fce9c000 task.ti: ffff8803fce9c000
[ 1253.907004] RIP: 0010:[<ffffffff8156d7c1>]  [<ffffffff8156d7c1>] apic_timer_interrupt+0x1/0xa0
[ 1253.907004] RSP: 0018:ffff8803fce9fa50  EFLAGS: 00010016
[ 1253.907004] RAX: ffffffff812e91c3 RBX: ffff8800b93aa280 RCX: 000000000000003f
[ 1253.907004] RDX: 0000000000001000 RSI: 0000000000000000 RDI: ffff8803d6255000
[ 1253.907004] RBP: ffff8803fce9fb78 R08: ffff8803fce9fc10 R09: 0000000000000000
[ 1253.907004] R10: ffff8803d6255000 R11: 0000000000000000 R12: 0000000000001000
[ 1253.907004] R13: ffff88042578c320 R14: ffffea000f589540 R15: 000000000006b000
[ 1253.907004] FS:  00007effad62db80(0000) GS:ffff88043dd00000(0000) knlGS:0000000000000000
[ 1253.907004] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 1253.907004] CR2: 00007fa055491c70 CR3: 00000003fe77a000 CR4: 00000000000006f0
[ 1253.907004] Stack:
[ 1253.907004]  0000000000000010 0000000000010216 ffff8803fce9fa70 0000000000000018
[ 1253.907004]  ffffffffa0843455 ffff8803fce9fb20 ffff88042578c33c 000000000006bfff
[ 1253.907004]  ffff88042578c320 0000100800000008 ffff880402d1ac98 ffff880402d1ac90
[ 1253.907004] Call Trace:
[ 1253.907004]  [<ffffffffa0843455>] ? __do_readpage+0x6d5/0x870 [btrfs]
[ 1253.907004]  [<ffffffffa08223c0>] ? btrfs_syno_getattr+0x2a0/0x2a0 [btrfs]
[ 1253.907004]  [<ffffffffa08436d1>] __extent_read_full_page+0xe1/0xf0 [btrfs]
[ 1253.907004]  [<ffffffffa08223c0>] ? btrfs_syno_getattr+0x2a0/0x2a0 [btrfs]
[ 1253.907004]  [<ffffffffa0845582>] extent_read_full_page+0x32/0x50 [btrfs]
[ 1253.907004]  [<ffffffffa081ea20>] btrfs_readpage+0x20/0x30 [btrfs]
[ 1253.907004]  [<ffffffffa0831e8e>] prepare_uptodate_page+0x2e/0x80 [btrfs]
[ 1253.907004]  [<ffffffffa0831f83>] prepare_pages+0xa3/0x270 [btrfs]
[ 1253.907004]  [<ffffffffa08330fb>] __btrfs_buffered_write+0x1fb/0x6a0 [btrfs]
[ 1253.907004]  [<ffffffffa08384e0>] btrfs_file_write_iter+0x250/0x430 [btrfs]
[ 1253.907004]  [<ffffffff81180be7>] new_sync_write+0x87/0xb0
[ 1253.907004]  [<ffffffff81180c34>] __vfs_write+0x24/0x30
[ 1253.907004]  [<ffffffff81182178>] vfs_write+0xa8/0x350
[ 1253.907004]  [<ffffffff81183b3a>] SyS_write+0x5a/0xb0
[ 1253.907004]  [<ffffffff8156caca>] entry_SYSCALL_64_fastpath+0x1e/0x8e
[ 1253.907004] Code: 00 0f 22 d8 58 eb 03 0f 1f 00 48 89 e7 65 ff 04 25 54 d5 00 00 65 48 0f 44 24 25 58 d5 00 00 57 e8 05 23 ac ff e9 1b fe ff ff 0f <1f> 00 68 10 ff ff ff fc 48 83 c4 88 4c 89 5c 24 30 4c 89 54 24
[ 1253.907004] RIP  [<ffffffff8156d7c1>] apic_timer_interrupt+0x1/0xa0
[ 1253.907004]  RSP <ffff8803fce9fa50>
[ 1253.907004] ---[ end trace c2a0de88b5635be1 ]---
[ 1253.974020] ------------[ cut here ]------------
[ 1253.974020] WARNING: CPU: 2 PID: 17382 at kernel/softirq.c:150 __local_bh_enable_ip+0x61/0x90()
[ 1253.974020] Modules linked in: fuse 8021q vfat fat vhost_scsi(O) vhost(O) tcm_loop(O) iscsi_target_mod(O) target_core_user(O) target_core_ep(O) target_core_multi_file(O) target_core_file(O) target_core_iblock(O) target_core_mod(O) syno_extent_pool(PO) rodsp_ep(O) udf isofs synoacl_vfs(O) btrfs ecryptfs zstd_decompress zstd_compress xxhash xor raid6_pq iptable_nat nf_nat_ipv4 nf_nat xt_conntrack xt_mac xt_geoip(O) compat_xtables(O) nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables xt_recent xt_iprange xt_limit xt_state xt_tcpudp xt_multiport xt_LOG nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack iptable_filter ip_tables x_tables zram leds_lp3943 geminilake_synobios(PO) usblp i915 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops cfbfillrect cfbcopyarea cfbimgblt drm drm_panel_orientation_quirks
[ 1253.974020]  iosf_mbi fb fbdev video backlight i2c_algo_bit r8168(O) sg dm_snapshot dm_bufio crc_itu_t crc_ccitt psnap p8022 llc hfsplus md4 hmac sit tunnel4 ipv6 flashcache_syno(O) flashcache(O) syno_flashcache_control(O) dm_mod arc4 cryptd ecb aes_x86_64 authenc des_generic ansi_cprng cts md5 cbc cpufreq_powersave cpufreq_performance cpufreq_stats vxlan ip6_udp_tunnel udp_tunnel ip_tunnel loop sha256_generic synorbd(O) synofsbd(O) usb_storage hid_generic usbhid hid etxhci_hcd e1000(OE) 8139cp(OE) mii(OE) virtio_pci(OE) virtio(OE) virtio_ring(OE) rtc_cmos(OE) processor button xhci_pci xhci_hcd uhci_hcd ehci_pci ehci_hcd usbcore usb_common [last unloaded: geminilake_synobios]
[ 1253.974020] CPU: 2 PID: 17382 Comm: rsync Tainted: P      D    OE   4.4.180+ #42962
[ 1253.974020] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.1-0-g3208b098f51a-prebuilt.qemu.org 04/01/2014
[ 1253.974020]  0000000000000000 ffff8803fce9f710 ffffffff812dc35b 0000000000000009
[ 1253.974020]  0000000000000000 ffff8803fce9f748 ffffffff8104fdd2 0000000000000200
[ 1253.974020]  ffff8800b93aa280 0000000000000001 0000000000000000 ffff88041f043168
[ 1253.974020] Call Trace:
[ 1253.974020]  [<ffffffff812dc35b>] dump_stack+0x4d/0x72
[ 1253.974020]  [<ffffffff8104fdd2>] warn_slowpath_common+0x82/0xa0
[ 1253.974020]  [<ffffffff8104fea5>] warn_slowpath_null+0x15/0x20
[ 1253.974020]  [<ffffffff810536f1>] __local_bh_enable_ip+0x61/0x90
[ 1253.974020]  [<ffffffff8156c665>] _raw_spin_unlock_bh+0x15/0x20
[ 1253.974020]  [<ffffffff810cbc70>] cgroup_exit+0x50/0xc0
[ 1253.974020]  [<ffffffff810525a5>] do_exit+0x2a5/0xa10
[ 1253.974020]  [<ffffffff81007b95>] oops_end+0x85/0xa0
[ 1253.974020]  [<ffffffff81049c3d>] die.cold.6+0x21/0x2a
[ 1253.974020]  [<ffffffff81005806>] do_trap+0x66/0xf0
[ 1253.974020]  [<ffffffff810059f3>] do_error_trap+0x73/0xd0
[ 1253.974020]  [<ffffffff8156d7c1>] ? apic_timer_interrupt+0x1/0xa0
[ 1253.974020]  [<ffffffffa083c051>] ? btrfs_get_token_16+0xf1/0x120 [btrfs]
[ 1253.974020]  [<ffffffff81005f5b>] do_invalid_op+0x1b/0x20
[ 1253.974020]  [<ffffffff8156e5f8>] invalid_op+0x18/0x20
[ 1253.974020]  [<ffffffff812e91c3>] ? memset_orig+0x33/0xb0
[ 1253.974020]  [<ffffffff8156d7c1>] ? apic_timer_interrupt+0x1/0xa0
[ 1253.974020]  [<ffffffffa0843455>] ? __do_readpage+0x6d5/0x870 [btrfs]
[ 1253.974020]  [<ffffffffa08223c0>] ? btrfs_syno_getattr+0x2a0/0x2a0 [btrfs]
[ 1253.974020]  [<ffffffffa08436d1>] __extent_read_full_page+0xe1/0xf0 [btrfs]
[ 1253.974020]  [<ffffffffa08223c0>] ? btrfs_syno_getattr+0x2a0/0x2a0 [btrfs]
[ 1253.974020]  [<ffffffffa0845582>] extent_read_full_page+0x32/0x50 [btrfs]
[ 1253.974020]  [<ffffffffa081ea20>] btrfs_readpage+0x20/0x30 [btrfs]
[ 1253.974020]  [<ffffffffa0831e8e>] prepare_uptodate_page+0x2e/0x80 [btrfs]
[ 1253.974020]  [<ffffffffa0831f83>] prepare_pages+0xa3/0x270 [btrfs]
[ 1253.974020]  [<ffffffffa08330fb>] __btrfs_buffered_write+0x1fb/0x6a0 [btrfs]
[ 1253.974020]  [<ffffffffa08384e0>] btrfs_file_write_iter+0x250/0x430 [btrfs]
[ 1253.974020]  [<ffffffff81180be7>] new_sync_write+0x87/0xb0
[ 1253.974020]  [<ffffffff81180c34>] __vfs_write+0x24/0x30
[ 1253.974020]  [<ffffffff81182178>] vfs_write+0xa8/0x350
[ 1253.974020]  [<ffffffff81183b3a>] SyS_write+0x5a/0xb0
[ 1253.974020]  [<ffffffff8156caca>] entry_SYSCALL_64_fastpath+0x1e/0x8e
[ 1253.974020] ---[ end trace c2a0de88b5635be2 ]---
[ 1265.250018] synobios get empty ttyS current
[ 1320.831441] xhci_hcd 0000:02:1b.0: xHCI host not responding to stop endpoint command.
[ 1320.832006] xhci_hcd 0000:02:1b.0: Assuming host is dying, halting host.
[ 1320.832945] xhci_hcd 0000:02:1b.0: HC died; cleaning up
[ 1320.833941] usb usb2-port1: logical disconnect on port 1 [usb-storage]
[ 1320.835541] usb 2-1: USB disconnect, device number 2