The next SEGV. Looks like this time it crashed while rotating the log. I was not around. I also found a zombie killall, but i couldn't find a hint where it was comming from. Maybe the logrotate script?
Jul 01 06:34:28 Storage-Blue systemd[1]: Got message type=signal sender=org.freedesktop.DBus destination=n/a object=/org/freedesktop/DBus interface=org.freedesktop.DBus member=NameOwnerChanged cookie=23 reply_cookie=0 error=n/a Jul 01 06:34:28 Storage-Blue systemd[1]: Got message type=signal sender=org.freedesktop.DBus destination=n/a object=/org/freedesktop/DBus interface=org.freedesktop.DBus member=NameOwnerChanged cookie=24 reply_cookie=0 error=n/a Jul 01 06:34:56 Storage-Blue systemd[1]: Accepted new private connection. Jul 01 06:34:56 Storage-Blue systemd[1]: Got notification message for unit systemd-journald.service Jul 01 06:34:56 Storage-Blue systemd[1]: systemd-journald.service: Got notification message from PID 220 (WATCHDOG=1...) Jul 01 06:34:56 Storage-Blue systemd[1]: systemd-journald.service: got WATCHDOG=1 Jul 01 06:34:56 Storage-Blue systemd[1]: Got notification message for unit systemd-journald.service Jul 01 06:34:56 Storage-Blue systemd[1]: systemd-journald.service: Got notification message from PID 220 (WATCHDOG=1...) Jul 01 06:34:56 Storage-Blue systemd[1]: systemd-journald.service: got WATCHDOG=1 Jul 01 06:34:56 Storage-Blue systemd[1]: Got message type=method_call sender=n/a destination=org.freedesktop.systemd1 object=/org/freedesktop/systemd1/unit/rsyslog_2eservice interface=org.freedesktop.DBus.Properties member=GetAll cookie=1 reply_cookie=0 error=n/a Jul 01 06:34:56 Storage-Blue systemd[1]: Caught <SEGV>, dumped core as pid 2934. Jul 01 06:34:56 Storage-Blue systemd[1]: Freezing execution.
$ ls -la /core
-rw------- 1 root root 1888256 Jul 1 06:34 /core
$ sudo gdb /lib/systemd/systemd /core
GNU gdb (Debian 7.7.1+dfsg-5) 7.7.1
[...]
[New LWP 2934]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/arm-linux-gnueabi/libthread_db.so.1".
Core was generated by `/sbin/init'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0xb6db9198 in raise (sig=11) at
../nptl/sysdeps/unix/sysv/linux/pt-raise.c:37
37 ../nptl/sysdeps/unix/sysv/linux/pt-raise.c: No such file or directory.
(gdb) bt
#0 0xb6db9198 in raise (sig=11) at
../nptl/sysdeps/unix/sysv/linux/pt-raise.c:37
#1 0xb6e188f0 in crash.lto_priv.245 (sig=11) at ../src/core/main.c:158
#2 <signal handler called>
#3 hashmap_iterate (h=h@entry=0xb810c82c, i=i@entry=0xbef6c200,
key=key@entry=0x0) at ../src/shared/hashmap.c:704
#4 0xb6e774b4 in set_iterate (s=s@entry=0xb810c82c, i=i@entry=0xbef6c200) at
../src/shared/set.c:91
#5 0xb6e820dc in property_get_dependencies.lto_priv.799 (bus=<optimized out>,
path=<optimized out>, interface=<optimized out>, property=<optimized out>,
reply=0xb81689e8,
userdata=0xb80fd37c, error=0xbef6c30c) at ../src/core/dbus-unit.c:108
#6 0xb6e8bde4 in invoke_property_get (bus=bus@entry=0xb8173888,
slot=slot@entry=0xb8152b30, v=v@entry=0xb6f28720 <bus_unit_vtable+256>,
path=path@entry=0xb816d080
"/org/freedesktop/systemd1/unit/rsyslog_2eservice", interface=0xb8152b70
"org.freedesktop.systemd1.Unit", property=0xb6ef7a1c "Wants",
reply=reply@entry=0xb81689e8, userdata=0xb80fd37c, error=0xbef6c30c,
error@entry=0xb6f28720 <bus_unit_vtable+256>) at
../src/libsystemd/sd-bus/bus-objects.c:447
#7 0xb6e8c19c in vtable_append_one_property (bus=bus@entry=0xb8173888,
reply=reply@entry=0xb81689e8, path=path@entry=0xb816d080
"/org/freedesktop/systemd1/unit/rsyslog_2eservice",
c=c@entry=0xb8152b48, v=v@entry=0xb6f28720 <bus_unit_vtable+256>,
userdata=userdata@entry=0xb80fd350, error=error@entry=0xbef6c30c) at
../src/libsystemd/sd-bus/bus-objects.c:702
#8 0xb6e8c39c in vtable_append_all_properties (bus=bus@entry=0xb8173888,
reply=0xb81689e8, path=0xb816d080
"/org/freedesktop/systemd1/unit/rsyslog_2eservice",
c=c@entry=0xb8152b48, userdata=0xb80fd350, error=error@entry=0xbef6c30c) at
../src/libsystemd/sd-bus/bus-objects.c:745
#9 0xb6e908c8 in property_get_all_callbacks_run (found_object=0xbef6c40c,
iface=0x0, require_fallback=true, first=<optimized out>, m=0xb8168800,
bus=0xb8173888)
at ../src/libsystemd/sd-bus/bus-objects.c:806
#10 object_find_and_run.lto_priv.263 (bus=bus@entry=0xb8173888,
m=m@entry=0xb8168800, p=0x1 <error: Cannot access memory at address 0x1>,
p@entry=0xbef6c398 "/org/freedesktop/systemd1/unit",
require_fallback=require_fallback@entry=true,
found_object=found_object@entry=0xbef6c40c)
at ../src/libsystemd/sd-bus/bus-objects.c:1314
#11 0xb6e4b350 in bus_process_object (m=0xb8168800, bus=0xb8173888) at
../src/libsystemd/sd-bus/bus-objects.c:1392
#12 process_message (m=0xb8168800, bus=0xb8173888) at
../src/libsystemd/sd-bus/sd-bus.c:2394
#13 process_running (hint_priority=false, priority=0, ret=0x0, bus=0xb8173888)
at ../src/libsystemd/sd-bus/sd-bus.c:2436
#14 bus_process_internal.constprop.176 (bus=0xb8173888, ret=0x0, priority=0,
hint_priority=false) at ../src/libsystemd/sd-bus/sd-bus.c:2619
#15 0xb6e3a640 in sd_bus_process (ret=0x0, bus=<optimized out>) at
../src/libsystemd/sd-bus/sd-bus.c:2638
#16 io_callback (s=<optimized out>, fd=<optimized out>, revents=<optimized
out>, userdata=<optimized out>) at ../src/libsystemd/sd-bus/sd-bus.c:2897
#17 0xb6e26d10 in source_dispatch (s=0xb8157cb0) at
../src/libsystemd/sd-event/sd-event.c:2016
#18 0xb6e2c400 in sd_event_run (e=0xb80c3f50, timeout=<optimized out>) at
../src/libsystemd/sd-event/sd-event.c:2314
#19 0xb6e15394 in manager_loop (m=0xb80c3c08) at ../src/core/manager.c:2009
#20 main (argc=1, argv=0xbef6ce84) at ../src/core/main.c:1743
$ ps faux
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 2 0.0 0.0 0 0 ? S Jun30 0:00 [kthreadd]
root 3 0.0 0.0 0 0 ? S Jun30 0:09 \_
[ksoftirqd/0]
root 5 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[kworker/0:0H]
root 7 0.0 0.0 0 0 ? S Jun30 0:00 \_
[watchdog/0]
root 8 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [khelper]
root 9 0.0 0.0 0 0 ? S Jun30 0:00 \_ [kdevtmpfs]
root 10 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [netns]
root 11 0.0 0.0 0 0 ? S Jun30 0:00 \_
[khungtaskd]
root 12 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [writeback]
root 13 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [crypto]
root 14 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[kintegrityd]
root 15 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 16 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kblockd]
root 18 0.0 0.0 0 0 ? S Jun30 0:05 \_ [kswapd0]
root 19 0.0 0.0 0 0 ? S Jun30 0:00 \_
[fsnotify_mark]
root 25 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kthrotld]
root 26 0.0 0.0 0 0 ? S Jun30 0:00 \_ [spi0]
root 27 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [deferwq]
root 64 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [ata_sff]
root 65 0.0 0.0 0 0 ? S Jun30 0:00 \_ [scsi_eh_0]
root 66 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[scsi_tmf_0]
root 67 0.0 0.0 0 0 ? S Jun30 0:00 \_ [scsi_eh_1]
root 68 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[scsi_tmf_1]
root 69 0.0 0.0 0 0 ? S Jun30 0:00 \_ [scsi_eh_2]
root 70 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[scsi_tmf_2]
root 71 0.0 0.0 0 0 ? S Jun30 0:00 \_ [scsi_eh_3]
root 72 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[scsi_tmf_3]
root 73 0.0 0.0 0 0 ? S Jun30 0:00 \_ [scsi_eh_4]
root 75 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[scsi_tmf_4]
root 77 0.0 0.0 0 0 ? S Jun30 0:00 \_ [scsi_eh_5]
root 79 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[scsi_tmf_5]
root 88 0.0 0.0 0 0 ? S< Jun30 0:45 \_
[kworker/0:1H]
root 95 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [md]
root 100 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 101 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 132 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 150 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-worker]
root 151 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[kworker/u3:0]
root 152 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-worker-hi]
root 153 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-delalloc]
root 154 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-flush_del]
root 155 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-cache]
root 156 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-submit]
root 157 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-fixup]
root 158 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-endio]
root 159 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-endio-met]
root 160 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-endio-met]
root 161 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-endio-rai]
root 162 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [btrfs-rmw]
root 163 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-endio-wri]
root 164 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-freespace]
root 165 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-delayed-m]
root 166 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-readahead]
root 167 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-qgroup-re]
root 168 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-extent-re]
root 169 0.0 0.0 0 0 ? S Jun30 0:00 \_
[btrfs-cleaner]
root 170 0.0 0.0 0 0 ? S Jun30 0:13 \_
[btrfs-transacti]
root 190 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[ipv6_addrconf]
root 211 0.0 0.0 0 0 ? S Jun30 0:00 \_ [kauditd]
root 289 0.0 0.0 0 0 ? S Jun30 0:00 \_ [khubd]
root 290 0.0 0.0 0 0 ? S Jun30 0:00 \_ [mv_crypto]
root 637 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kdmflush]
root 638 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kdmflush]
root 644 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kdmflush]
root 682 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 683 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 684 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[kcryptd_io]
root 685 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[kcryptd_io]
root 686 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 688 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kcryptd]
root 689 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kcryptd]
root 690 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[kcryptd_io]
root 691 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 692 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 693 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kcryptd]
root 694 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 704 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kdmflush]
root 705 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 706 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[kcryptd_io]
root 707 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kcryptd]
root 708 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 840 0.0 0.0 0 0 ? S< Jun30 0:32 \_
[kworker/u3:1]
root 934 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [rpciod]
root 936 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [nfsiod]
root 1073 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kdmflush]
root 1074 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 1075 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[kcryptd_io]
root 1076 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kcryptd]
root 1077 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 1086 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kdmflush]
root 1087 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 1088 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[kcryptd_io]
root 1089 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [kcryptd]
root 1090 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [bioset]
root 1102 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-worker]
root 1103 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-worker-hi]
root 1104 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-delalloc]
root 1105 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-flush_del]
root 1106 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-cache]
root 1107 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-submit]
root 1108 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-fixup]
root 1109 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-endio]
root 1110 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-endio-met]
root 1111 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-endio-met]
root 1112 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-endio-rai]
root 1113 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [btrfs-rmw]
root 1114 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-endio-wri]
root 1115 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-freespace]
root 1116 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-delayed-m]
root 1117 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-readahead]
root 1118 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-qgroup-re]
root 1119 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[btrfs-extent-re]
root 1120 0.0 0.0 0 0 ? S Jun30 0:00 \_
[btrfs-cleaner]
root 1121 0.1 0.0 0 0 ? S Jun30 2:33 \_
[btrfs-transacti]
root 1154 0.0 0.0 0 0 ? S< Jun30 0:00 \_ [nfsd4]
root 1155 0.0 0.0 0 0 ? S< Jun30 0:00 \_
[nfsd4_callbacks]
root 1156 0.0 0.0 0 0 ? S Jun30 0:00 \_ [lockd]
root 1159 0.0 0.0 0 0 ? S Jun30 1:32 \_ [nfsd]
root 3349 0.2 0.0 0 0 ? S 10:17 0:51 \_
[kworker/0:3]
root 3364 0.2 0.0 0 0 ? S 10:29 0:40 \_
[kworker/0:1]
root 3466 0.2 0.0 0 0 ? S 11:47 0:35 \_
[kworker/0:2]
root 3634 0.0 0.0 0 0 ? S 13:55 0:00 \_
[kworker/u2:0]
root 3734 0.0 0.0 0 0 ? S 15:12 0:00 \_
[kworker/u2:4]
root 3743 0.0 0.0 0 0 ? S 15:20 0:00 \_
[kworker/u2:1]
root 3748 0.0 0.0 0 0 ? S 15:22 0:00 \_
[kworker/u2:2]
root 3761 0.3 0.0 0 0 ? S 15:27 0:03 \_
[kworker/0:4]
root 3769 0.5 0.0 0 0 ? S 15:32 0:02 \_
[kworker/0:5]
root 3771 0.0 0.0 0 0 ? S 15:33 0:00 \_
[kworker/u2:5]
root 3772 0.0 0.0 0 0 ? S 15:33 0:00 \_
[kworker/u2:6]
root 3773 0.0 0.0 0 0 ? S 15:33 0:00 \_
[kworker/u2:7]
root 1 0.0 0.3 5668 2024 ? Ss Jun30 0:03 /sbin/init
root 220 0.0 0.8 8068 4356 ? Ss Jun30 0:28
/lib/systemd/systemd-journald
root 225 0.0 0.2 11624 1132 ? Ss Jun30 0:00
/lib/systemd/systemd-udevd
root 802 0.0 0.9 8596 5092 ? Ss Jun30 0:00 dhclient -v
-pf /run/dhclient.eth0.pid -lf /var/lib/dhcp/dhclient.eth0.leases eth0
root 846 0.0 1.1 8596 5832 ? Ss Jun30 0:00 dhclient -v
-pf /run/dhclient.eth1.pid -lf /var/lib/dhcp/dhclient.eth1.leases eth1
root 919 0.0 0.2 4356 1484 ? Ss Jun30 0:00 /sbin/rpcbind
-w
statd 928 0.0 0.3 4544 1616 ? Ss Jun30 0:00 /sbin/rpc.statd
root 943 0.0 0.2 2812 1136 ? Ss Jun30 0:00
/usr/sbin/rpc.idmapd
root 944 0.0 0.3 5340 1928 ? Ss Jun30 0:00
/usr/sbin/smartd -n --interval=31536000
root 945 0.0 0.4 7832 2452 ? Ss Jun30 0:00 /usr/sbin/sshd
-D
root 3776 0.0 0.9 11568 5008 ? Ss 15:34 0:00 \_ sshd: tino
[priv]
tino 3778 0.8 0.8 11568 4260 ? S 15:34 0:03 \_ sshd:
tino@pts/0
tino 3779 0.1 0.8 6096 4292 pts/0 Ss 15:34 0:00 \_
-bash
tino 3815 0.0 0.4 4760 2240 pts/0 R+ 15:41 0:00
\_ ps faux
root 946 0.0 0.2 4980 1492 ? Ss Jun30 0:00 /usr/sbin/cron
-f
root 947 0.0 0.4 31120 2324 ? Ssl Jun30 0:04
/usr/sbin/rsyslogd -n
daemon 948 0.0 0.2 2540 1200 ? Ss Jun30 0:00 /usr/sbin/atd
-f
root 949 0.0 0.2 4044 1240 ? Ss Jun30 0:00 /usr/bin/rsync
--daemon --no-detach
message+ 952 0.0 0.3 5588 1988 ? Ss Jun30 0:00
/usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile
--systemd-activation
root 956 0.0 0.2 3284 1460 ? Ss Jun30 0:00
/lib/systemd/systemd-logind
mail 978 0.0 0.2 3488 1144 ? S Jun30 0:00
/usr/sbin/nullmailer-send -s
root 994 0.0 0.2 28452 1280 ? Sl Jun30 0:02 qcontrol -f
root 1017 0.0 0.7 22556 4088 ? Ss Jun30 0:01 /usr/sbin/nmbd
-D
root 1027 0.0 1.1 35356 5656 ? Ss Jun30 0:03 /usr/sbin/smbd
-D
root 1136 0.0 0.1 2468 748 ? Ss Jun30 0:00
/usr/sbin/in.tftpd --listen --user tftp --address 0.0.0.0:69 --secure /srv/tftp
root 1162 0.0 0.3 5376 1876 ? Ss Jun30 0:00
/usr/sbin/rpc.mountd --manage-gids
root 3001 0.0 0.0 0 0 ? Z 06:37 0:00 [killall]
<defunct>
$ cat /proc/3001/cmdline
$ sudo cat /proc/3001/stack
[<c001bff0>] do_exit+0x838/0x884
[<c001c0cc>] do_group_exit+0x54/0xb4
[<c001c144>] __wake_up_parent+0x0/0x20
[<ffffffff>] 0xffffffff
signature.asc
Description: OpenPGP digital signature
