Hi Emeric,
since 8d85aa4 ("BUG/MAJOR: map: fix segfault during 'show
map/acl' on cli") my setup crashes when a request comes in
going through SSL termination.
memory corruption, invalid pointers, double free is what haproxy
randomly crashes with.
Here 2 crashes with full backtrace:
*** Error in `/usr/sbin/haproxy': double free or corruption (!prev): 0x0000000000a42590 ***
======= Backtrace: =========
/lib/x86_64-linux-gnu/libc.so.6(+0x777e5)[0x7ffff71bb7e5]
/lib/x86_64-linux-gnu/libc.so.6(+0x8037a)[0x7ffff71c437a]
/lib/x86_64-linux-gnu/libc.so.6(cfree+0x4c)[0x7ffff71c853c]
/usr/sbin/haproxy[0x53a64e]
/usr/sbin/haproxy[0x53630b]
/usr/sbin/haproxy[0x4124de]
/usr/sbin/haproxy[0x48103a]
/usr/sbin/haproxy[0x482f09]
/usr/sbin/haproxy[0x4891af]
/usr/sbin/haproxy[0x50910f]
/usr/sbin/haproxy[0x4d5159]
/usr/sbin/haproxy[0x4d64ba]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf0)[0x7ffff7164830]
/usr/sbin/haproxy[0x4055a9]
======= Memory map: ========
00400000-007af000 r-xp 00000000 ca:02 40972 /usr/sbin/haproxy
009af000-009cf000 r--p 003af000 ca:02 40972 /usr/sbin/haproxy
009cf000-009eb000 rw-p 003cf000 ca:02 40972 /usr/sbin/haproxy
009eb000-00ac5000 rw-p 00000000 00:00 0 [heap]
7ffff0000000-7ffff0021000 rw-p 00000000 00:00 0
7ffff0021000-7ffff4000000 ---p 00000000 00:00 0
7ffff6b0e000-7ffff6b24000 r-xp 00000000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6b24000-7ffff6d23000 ---p 00016000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6d23000-7ffff6d24000 rw-p 00015000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6d24000-7ffff7144000 rw-p 00000000 00:00 0
7ffff7144000-7ffff7304000 r-xp 00000000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7304000-7ffff7504000 ---p 001c0000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7504000-7ffff7508000 r--p 001c0000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7508000-7ffff750a000 rw-p 001c4000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff750a000-7ffff750e000 rw-p 00000000 00:00 0
7ffff750e000-7ffff7526000 r-xp 00000000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7526000-7ffff7725000 ---p 00018000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7725000-7ffff7726000 r--p 00017000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7726000-7ffff7727000 rw-p 00018000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7727000-7ffff772b000 rw-p 00000000 00:00 0
7ffff772b000-7ffff7799000 r-xp 00000000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff7799000-7ffff7999000 ---p 0006e000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff7999000-7ffff799a000 r--p 0006e000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff799a000-7ffff799b000 rw-p 0006f000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff799b000-7ffff799e000 r-xp 00000000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff799e000-7ffff7b9d000 ---p 00003000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9d000-7ffff7b9e000 r--p 00002000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9e000-7ffff7b9f000 rw-p 00003000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9f000-7ffff7ba8000 r-xp 00000000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7ba8000-7ffff7da7000 ---p 00009000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da7000-7ffff7da8000 r--p 00008000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da8000-7ffff7da9000 rw-p 00009000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da9000-7ffff7dd7000 rw-p 00000000 00:00 0
7ffff7dd7000-7ffff7dfd000 r-xp 00000000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7fec000-7ffff7ff0000 rw-p 00000000 00:00 0
7ffff7ff5000-7ffff7ff8000 rw-p 00000000 00:00 0
7ffff7ff8000-7ffff7ffa000 r--p 00000000 00:00 0 [vvar]
7ffff7ffa000-7ffff7ffc000 r-xp 00000000 00:00 0 [vdso]
7ffff7ffc000-7ffff7ffd000 r--p 00025000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7ffd000-7ffff7ffe000 rw-p 00026000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7ffe000-7ffff7fff000 rw-p 00000000 00:00 0
7fffffede000-7ffffffff000 rw-p 00000000 00:00 0 [stack]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
Program received signal SIGABRT, Aborted.
0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
54 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt
#0 0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
#1 0x00007ffff717b02a in __GI_abort () at abort.c:89
#2 0x00007ffff71bb7ea in __libc_message (do_abort=do_abort@entry=2, fmt=fmt@entry=0x7ffff72d4e98 "*** Error in `%s': %s: 0x%s ***\n") at ../sysdeps/posix/libc_fatal.c:175
#3 0x00007ffff71c437a in malloc_printerr (ar_ptr=<optimized out>, ptr=<optimized out>, str=0x7ffff72d4fc8 "double free or corruption (!prev)", action=3) at malloc.c:5006
#4 _int_free (av=<optimized out>, p=<optimized out>, have_lock=0) at malloc.c:3867
#5 0x00007ffff71c853c in __GI___libc_free (mem=<optimized out>) at malloc.c:2968
#6 0x000000000053a64e in SSL_SESSION_free ()
#7 0x000000000053630b in SSL_free ()
#8 0x00000000004124de in ssl_sock_close (conn=0xa45080) at src/ssl_sock.c:5086
#9 0x000000000048103a in conn_force_close (conn=0xa45080) at include/proto/connection.h:151
#10 0x0000000000482f09 in stream_free (s=0xaaf230) at src/stream.c:312
#11 0x00000000004891af in process_stream (t=0xa52a40) at src/stream.c:2419
#12 0x000000000050910f in process_runnable_tasks () at src/task.c:238
#13 0x00000000004d5159 in run_poll_loop () at src/haproxy.c:2168
#14 0x00000000004d64ba in main (argc=4, argv=0x7fffffffe658) at src/haproxy.c:2701
(gdb) bt full
#0 0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
resultvar = 0
pid = 29988
selftid = 29988
#1 0x00007ffff717b02a in __GI_abort () at abort.c:89
save_stage = 2
act = {__sigaction_handler = {sa_handler = 0x20302030303a3030, sa_sigaction = 0x20302030303a3030}, sa_mask = {__val = {2314885530818453536, 2314885530818453536, 7017579609838738208, 4206752516204751980, 3545519503966220848, 2314885530818453536, 2314885530818453536, 7795484802351636512, 3917909816998060649,
3276497845987585332, 7161402270846119527, 3615882721633532274, 7378645557452156467, 3472337303646987878, 3991990709698112816, 8223625903104156004}}, sa_flags = 544222583, sa_restorer = 0x5c}
sigs = {__val = {32, 0 <repeats 15 times>}}
#2 0x00007ffff71bb7ea in __libc_message (do_abort=do_abort@entry=2, fmt=fmt@entry=0x7ffff72d4e98 "*** Error in `%s': %s: 0x%s ***\n") at ../sysdeps/posix/libc_fatal.c:175
ap = <error reading variable ap (Attempt to dereference a generic pointer.)>
fd = 7
on_2 = <optimized out>
list = <optimized out>
nlist = <optimized out>
cp = <optimized out>
written = <optimized out>
#3 0x00007ffff71c437a in malloc_printerr (ar_ptr=<optimized out>, ptr=<optimized out>, str=0x7ffff72d4fc8 "double free or corruption (!prev)", action=3) at malloc.c:5006
buf = "0000000000a42590"
cp = <optimized out>
ar_ptr = <optimized out>
str = 0x7ffff72d4fc8 "double free or corruption (!prev)"
action = 3
#4 _int_free (av=<optimized out>, p=<optimized out>, have_lock=0) at malloc.c:3867
size = <optimized out>
fb = <optimized out>
nextchunk = <optimized out>
nextsize = <optimized out>
nextinuse = <optimized out>
prevsize = <optimized out>
bck = <optimized out>
fwd = <optimized out>
errstr = <optimized out>
locked = <optimized out>
#5 0x00007ffff71c853c in __GI___libc_free (mem=<optimized out>) at malloc.c:2968
ar_ptr = <optimized out>
p = <optimized out>
hook = <optimized out>
#6 0x000000000053a64e in SSL_SESSION_free ()
No symbol table info available.
#7 0x000000000053630b in SSL_free ()
No symbol table info available.
#8 0x00000000004124de in ssl_sock_close (conn=0xa45080) at src/ssl_sock.c:5086
No locals.
#9 0x000000000048103a in conn_force_close (conn=0xa45080) at include/proto/connection.h:151
No locals.
#10 0x0000000000482f09 in stream_free (s=0xaaf230) at src/stream.c:312
sess = 0xa529b0
fe = 0xa429f0
bref = 0xa9f850
back = 0xaaf230
cli_conn = 0xa45080
i = 0
#11 0x00000000004891af in process_stream (t=0xa52a40) at src/stream.c:2419
srv = 0x0
s = 0xaaf230
sess = 0xa529b0
rqf_last = 75554848
rpf_last = 2147787360
rq_prod_last = 9
rq_cons_last = 9
rp_cons_last = 9
rp_prod_last = 9
req_ana_back = 0
req = 0xaaf240
res = 0xaaf280
si_f = 0xaaf468
si_b = 0xaaf490
#12 0x000000000050910f in process_runnable_tasks () at src/task.c:238
t = 0xa52a40
i = 0
max_processed = 1
rq_next = 0x0
rewind = 1
local_tasks = {0xa52a40, 0xa45080, 0x7fffffffe3a0, 0x802a130000a45080, 0x7fffffffe3a0, 0x4fa8aa <conn_cond_update_polling+89>, 0xaaf240, 0x9dc3f0 <applet_active_queue>, 0x7fffffffe3d0, 0x4facd8 <conn_fd_handler+802>, 0xf240107009eceb0, 0x500a14ce0, 0x7fffffffe3c0, 0x7fffffffe3c0, 0x7fffffffe3f0,
0xc2769aa5f730bf00}
local_tasks_count = 1
#13 0x00000000004d5159 in run_poll_loop () at src/haproxy.c:2168
next = 254017799
#14 0x00000000004d64ba in main (argc=4, argv=0x7fffffffe658) at src/haproxy.c:2701
err = 0
retry = 200
limit = {rlim_cur = 4011, rlim_max = 4011}
errmsg = "\000@\243\000\000\000\000\000X\346\377\377\377\177\000\000\004\000\000\000\000\000\000\000ʍ\034\367\377\177\000\000\260=\243\000\000\000\000\000\"\000\000\000\000\000\000\000\000\345\377\377\377\177\000\000\370\364\232\000\000\000\000\000\200\346\377\377\377\177\000\000*\373L\000\000\000\000\000\001\000\000\000\001\000\000\000\060?\243\000\000\000\000\000\"\000\000"
pidfd = -1
(gdb)
Here's another one:
*** Error in `/usr/sbin/haproxy': malloc(): memory corruption: 0x0000000000a41ee0 ***
======= Backtrace: =========
/lib/x86_64-linux-gnu/libc.so.6(+0x777e5)[0x7ffff71bb7e5]
/lib/x86_64-linux-gnu/libc.so.6(+0x8213e)[0x7ffff71c613e]
/lib/x86_64-linux-gnu/libc.so.6(__libc_malloc+0x54)[0x7ffff71c8184]
/usr/sbin/haproxy[0x524a36]
/usr/sbin/haproxy[0x522e24]
/usr/sbin/haproxy[0x523402]
/usr/sbin/haproxy[0x533150]
/usr/sbin/haproxy[0x4120b6]
/usr/sbin/haproxy[0x4d9195]
/usr/sbin/haproxy[0x4da24d]
/usr/sbin/haproxy[0x4fab7d]
/usr/sbin/haproxy[0x51202c]
/usr/sbin/haproxy[0x4d51c8]
/usr/sbin/haproxy[0x4d64ba]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf0)[0x7ffff7164830]
/usr/sbin/haproxy[0x4055a9]
======= Memory map: ========
00400000-007af000 r-xp 00000000 ca:02 40972 /usr/sbin/haproxy
009af000-009cf000 r--p 003af000 ca:02 40972 /usr/sbin/haproxy
009cf000-009eb000 rw-p 003cf000 ca:02 40972 /usr/sbin/haproxy
009eb000-00ac5000 rw-p 00000000 00:00 0 [heap]
7ffff0000000-7ffff0021000 rw-p 00000000 00:00 0
7ffff0021000-7ffff4000000 ---p 00000000 00:00 0
7ffff6b0e000-7ffff6b24000 r-xp 00000000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6b24000-7ffff6d23000 ---p 00016000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6d23000-7ffff6d24000 rw-p 00015000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6d24000-7ffff7144000 rw-p 00000000 00:00 0
7ffff7144000-7ffff7304000 r-xp 00000000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7304000-7ffff7504000 ---p 001c0000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7504000-7ffff7508000 r--p 001c0000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7508000-7ffff750a000 rw-p 001c4000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff750a000-7ffff750e000 rw-p 00000000 00:00 0
7ffff750e000-7ffff7526000 r-xp 00000000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7526000-7ffff7725000 ---p 00018000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7725000-7ffff7726000 r--p 00017000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7726000-7ffff7727000 rw-p 00018000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7727000-7ffff772b000 rw-p 00000000 00:00 0
7ffff772b000-7ffff7799000 r-xp 00000000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff7799000-7ffff7999000 ---p 0006e000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff7999000-7ffff799a000 r--p 0006e000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff799a000-7ffff799b000 rw-p 0006f000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff799b000-7ffff799e000 r-xp 00000000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff799e000-7ffff7b9d000 ---p 00003000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9d000-7ffff7b9e000 r--p 00002000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9e000-7ffff7b9f000 rw-p 00003000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9f000-7ffff7ba8000 r-xp 00000000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7ba8000-7ffff7da7000 ---p 00009000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da7000-7ffff7da8000 r--p 00008000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da8000-7ffff7da9000 rw-p 00009000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da9000-7ffff7dd7000 rw-p 00000000 00:00 0
7ffff7dd7000-7ffff7dfd000 r-xp 00000000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7fec000-7ffff7ff0000 rw-p 00000000 00:00 0
7ffff7ff5000-7ffff7ff8000 rw-p 00000000 00:00 0
7ffff7ff8000-7ffff7ffa000 r--p 00000000 00:00 0 [vvar]
7ffff7ffa000-7ffff7ffc000 r-xp 00000000 00:00 0 [vdso]
7ffff7ffc000-7ffff7ffd000 r--p 00025000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7ffd000-7ffff7ffe000 rw-p 00026000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7ffe000-7ffff7fff000 rw-p 00000000 00:00 0
7fffffede000-7ffffffff000 rw-p 00000000 00:00 0 [stack]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
Program received signal SIGABRT, Aborted.
0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
54 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt
#0 0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
#1 0x00007ffff717b02a in __GI_abort () at abort.c:89
#2 0x00007ffff71bb7ea in __libc_message (do_abort=2, fmt=fmt@entry=0x7ffff72d4e98 "*** Error in `%s': %s: 0x%s ***\n") at ../sysdeps/posix/libc_fatal.c:175
#3 0x00007ffff71c613e in malloc_printerr (ar_ptr=0x7ffff7508b20 <main_arena>, ptr=0xa41ee0, str=0x7ffff72d1cff "malloc(): memory corruption", action=<optimized out>) at malloc.c:5006
#4 _int_malloc (av=av@entry=0x7ffff7508b20 <main_arena>, bytes=bytes@entry=16472) at malloc.c:3474
#5 0x00007ffff71c8184 in __GI___libc_malloc (bytes=16472) at malloc.c:2913
#6 0x0000000000524a36 in ssl3_setup_write_buffer ()
#7 0x0000000000522e24 in do_ssl3_write ()
#8 0x0000000000523402 in ssl3_write_bytes ()
#9 0x0000000000533150 in SSL_write ()
#10 0x00000000004120b6 in ssl_sock_from_buf (conn=0xa45080, buf=0xa9f850, flags=1) at src/ssl_sock.c:4974
#11 0x00000000004d9195 in si_conn_send (conn=0xa45080) at src/stream_interface.c:658
#12 0x00000000004da24d in si_conn_send_cb (conn=0xa45080) at src/stream_interface.c:1295
#13 0x00000000004fab7d in conn_fd_handler (fd=5) at src/connection.c:104
#14 0x000000000051202c in fd_process_cached_events () at src/fd.c:240
#15 0x00000000004d51c8 in run_poll_loop () at src/haproxy.c:2186
#16 0x00000000004d64ba in main (argc=4, argv=0x7fffffffe658) at src/haproxy.c:2701
(gdb) bt full
#0 0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
resultvar = 0
pid = 29977
selftid = 29977
#1 0x00007ffff717b02a in __GI_abort () at abort.c:89
save_stage = 2
act = {__sigaction_handler = {sa_handler = 0x20302030303a3030, sa_sigaction = 0x20302030303a3030}, sa_mask = {__val = {2314885530818453536, 2314885530818453536, 7017579609838738208, 4206752516204751980, 3545519503966220848, 2314885530818453536, 2314885530818453536, 7795484802351636512, 3917909816998060649,
3276497845987585332, 7161402270846119527, 3615882721633532274, 7378645557452156467, 3472337303646987878, 3991990709698112816, 8223625903104156004}}, sa_flags = 544222583, sa_restorer = 0x56}
sigs = {__val = {32, 0 <repeats 15 times>}}
#2 0x00007ffff71bb7ea in __libc_message (do_abort=2, fmt=fmt@entry=0x7ffff72d4e98 "*** Error in `%s': %s: 0x%s ***\n") at ../sysdeps/posix/libc_fatal.c:175
ap = <error reading variable ap (Attempt to dereference a generic pointer.)>
fd = 6
on_2 = <optimized out>
list = <optimized out>
nlist = <optimized out>
cp = <optimized out>
written = <optimized out>
#3 0x00007ffff71c613e in malloc_printerr (ar_ptr=0x7ffff7508b20 <main_arena>, ptr=0xa41ee0, str=0x7ffff72d1cff "malloc(): memory corruption", action=<optimized out>) at malloc.c:5006
buf = "0000000000a41ee0"
cp = <optimized out>
ar_ptr = 0x7ffff7508b20 <main_arena>
ptr = 0xa41ee0
str = 0x7ffff72d1cff "malloc(): memory corruption"
action = <optimized out>
#4 _int_malloc (av=av@entry=0x7ffff7508b20 <main_arena>, bytes=bytes@entry=16472) at malloc.c:3474
iters = <optimized out>
nb = 16480
idx = 114
bin = <optimized out>
victim = 0xa41ed0
size = <optimized out>
victim_index = <optimized out>
remainder = <optimized out>
remainder_size = <optimized out>
block = <optimized out>
bit = <optimized out>
map = <optimized out>
fwd = <optimized out>
bck = 0x7ffff7508b78 <main_arena+88>
errstr = 0x0
__func__ = "_int_malloc"
#5 0x00007ffff71c8184 in __GI___libc_malloc (bytes=16472) at malloc.c:2913
ar_ptr = 0x7ffff7508b20 <main_arena>
victim = <optimized out>
hook = <optimized out>
#6 0x0000000000524a36 in ssl3_setup_write_buffer ()
No symbol table info available.
#7 0x0000000000522e24 in do_ssl3_write ()
No symbol table info available.
#8 0x0000000000523402 in ssl3_write_bytes ()
No symbol table info available.
#9 0x0000000000533150 in SSL_write ()
No symbol table info available.
#10 0x00000000004120b6 in ssl_sock_from_buf (conn=0xa45080, buf=0xa9f850, flags=1) at src/ssl_sock.c:4974
ret = 253949018
try = 212
done = 0
#11 0x00000000004d9195 in si_conn_send (conn=0xa45080) at src/stream_interface.c:658
send_flag = 1
si = 0xaa5b18
oc = 0xaa5930
ret = 0
#12 0x00000000004da24d in si_conn_send_cb (conn=0xa45080) at src/stream_interface.c:1295
si = 0xaa5b18
#13 0x00000000004fab7d in conn_fd_handler (fd=5) at src/connection.c:104
conn = 0xa45080
flags = 0
#14 0x000000000051202c in fd_process_cached_events () at src/fd.c:240
fd = 5
entry = 0
e = 50
#15 0x00000000004d51c8 in run_poll_loop () at src/haproxy.c:2186
next = 253899019
#16 0x00000000004d64ba in main (argc=4, argv=0x7fffffffe658) at src/haproxy.c:2701
err = 0
retry = 200
limit = {rlim_cur = 4011, rlim_max = 4011}
errmsg = "\000@\243\000\000\000\000\000X\346\377\377\377\177\000\000\004\000\000\000\000\000\000\000ʍ\034\367\377\177\000\000\260=\243\000\000\000\000\000\"\000\000\000\000\000\000\000\000\345\377\377\377\177\000\000\370\364\232\000\000\000\000\000\200\346\377\377\377\177\000\000*\373L\000\000\000\000\000\001\000\000\000\001\000\000\000\060?\243\000\000\000\000\000\"\000\000"
pidfd = -1
(gdb)
Repro config (fire requests to /robots.txt from curl or browsers):
global
ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:!DSS
ssl-default-bind-options no-tls-tickets no-tlsv10 no-tlsv11 force-tlsv12 prefer-client-ciphers
defaults
log global
option httplog
option dontlognull
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-keep-alive 60s
option http-keep-alive
option forwardfor
frontend tls-termination
mode http
bind :443 ssl crt /etc/ssl/private/temp.example.com.ecdsa crt /etc/ssl/private/ npn http/1.1 alpn http/1.1 curves X25519:P-256 #strict-sni
use_backend robots if { path /robots.txt }
#use_backend temp if { ssl_fc_sni -i temp.example.com }
backend temp
mode http
server local-nginx 127.0.0.1:80 maxconn 200
backend robots
mode http
errorfile 403 /etc/haproxy/errors/robotstxt.http
http-request deny
root@www:/usr/sbin# haproxy -vv
HA-Proxy version 1.8-dev2-8d85aa-52 2017/06/30
Copyright 2000-2017 Willy Tarreau <willy@haproxy.org>
Build options :
TARGET = linux2628
CPU = generic
CC = gcc
CFLAGS = -O0 -g -fno-strict-aliasing -Wdeclaration-after-statement -fwrapv
OPTIONS = USE_GETADDRINFO=1 USE_SLZ=1 USE_OPENSSL=1 USE_PCRE=1 USE_PCRE_JIT=1
Default settings :
maxconn = 2000, bufsize = 16384, maxrewrite = 1024, maxpollevents = 200
Built with OpenSSL version : OpenSSL 1.1.0g-dev xx XXX xxxx
Running on OpenSSL version : OpenSSL 1.1.0g-dev xx XXX xxxx
OpenSSL library supports TLS extensions : yes
OpenSSL library supports SNI : yes
OpenSSL library supports : SSLv3 TLSv1.0 TLSv1.1 TLSv1.2
Built with transparent proxy support using: IP_TRANSPARENT IPV6_TRANSPARENT IP_FREEBIND
Built with network namespace support.
Built with libslz for stateless compression.
Compression algorithms supported : identity("identity"), deflate("deflate"), raw-deflate("deflate"), gzip("gzip")
Encrypted password support via crypt(3): yes
Built with PCRE version : 8.38 2015-11-23
Running on PCRE version : 8.38 2015-11-23
PCRE library supports JIT : yes
Available polling systems :
epoll : pref=300, test result OK
poll : pref=200, test result OK
select : pref=150, test result OK
Total: 3 (3 usable), will use epoll.
Available filters :
[SPOE] spoe
[COMP] compression
[TRACE] trace
Hope this helps,
Lukas
since 8d85aa4 ("BUG/MAJOR: map: fix segfault during 'show
map/acl' on cli") my setup crashes when a request comes in
going through SSL termination.
memory corruption, invalid pointers, double free is what haproxy
randomly crashes with.
Here 2 crashes with full backtrace:
*** Error in `/usr/sbin/haproxy': double free or corruption (!prev): 0x0000000000a42590 ***
======= Backtrace: =========
/lib/x86_64-linux-gnu/libc.so.6(+0x777e5)[0x7ffff71bb7e5]
/lib/x86_64-linux-gnu/libc.so.6(+0x8037a)[0x7ffff71c437a]
/lib/x86_64-linux-gnu/libc.so.6(cfree+0x4c)[0x7ffff71c853c]
/usr/sbin/haproxy[0x53a64e]
/usr/sbin/haproxy[0x53630b]
/usr/sbin/haproxy[0x4124de]
/usr/sbin/haproxy[0x48103a]
/usr/sbin/haproxy[0x482f09]
/usr/sbin/haproxy[0x4891af]
/usr/sbin/haproxy[0x50910f]
/usr/sbin/haproxy[0x4d5159]
/usr/sbin/haproxy[0x4d64ba]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf0)[0x7ffff7164830]
/usr/sbin/haproxy[0x4055a9]
======= Memory map: ========
00400000-007af000 r-xp 00000000 ca:02 40972 /usr/sbin/haproxy
009af000-009cf000 r--p 003af000 ca:02 40972 /usr/sbin/haproxy
009cf000-009eb000 rw-p 003cf000 ca:02 40972 /usr/sbin/haproxy
009eb000-00ac5000 rw-p 00000000 00:00 0 [heap]
7ffff0000000-7ffff0021000 rw-p 00000000 00:00 0
7ffff0021000-7ffff4000000 ---p 00000000 00:00 0
7ffff6b0e000-7ffff6b24000 r-xp 00000000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6b24000-7ffff6d23000 ---p 00016000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6d23000-7ffff6d24000 rw-p 00015000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6d24000-7ffff7144000 rw-p 00000000 00:00 0
7ffff7144000-7ffff7304000 r-xp 00000000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7304000-7ffff7504000 ---p 001c0000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7504000-7ffff7508000 r--p 001c0000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7508000-7ffff750a000 rw-p 001c4000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff750a000-7ffff750e000 rw-p 00000000 00:00 0
7ffff750e000-7ffff7526000 r-xp 00000000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7526000-7ffff7725000 ---p 00018000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7725000-7ffff7726000 r--p 00017000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7726000-7ffff7727000 rw-p 00018000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7727000-7ffff772b000 rw-p 00000000 00:00 0
7ffff772b000-7ffff7799000 r-xp 00000000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff7799000-7ffff7999000 ---p 0006e000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff7999000-7ffff799a000 r--p 0006e000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff799a000-7ffff799b000 rw-p 0006f000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff799b000-7ffff799e000 r-xp 00000000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff799e000-7ffff7b9d000 ---p 00003000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9d000-7ffff7b9e000 r--p 00002000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9e000-7ffff7b9f000 rw-p 00003000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9f000-7ffff7ba8000 r-xp 00000000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7ba8000-7ffff7da7000 ---p 00009000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da7000-7ffff7da8000 r--p 00008000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da8000-7ffff7da9000 rw-p 00009000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da9000-7ffff7dd7000 rw-p 00000000 00:00 0
7ffff7dd7000-7ffff7dfd000 r-xp 00000000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7fec000-7ffff7ff0000 rw-p 00000000 00:00 0
7ffff7ff5000-7ffff7ff8000 rw-p 00000000 00:00 0
7ffff7ff8000-7ffff7ffa000 r--p 00000000 00:00 0 [vvar]
7ffff7ffa000-7ffff7ffc000 r-xp 00000000 00:00 0 [vdso]
7ffff7ffc000-7ffff7ffd000 r--p 00025000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7ffd000-7ffff7ffe000 rw-p 00026000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7ffe000-7ffff7fff000 rw-p 00000000 00:00 0
7fffffede000-7ffffffff000 rw-p 00000000 00:00 0 [stack]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
Program received signal SIGABRT, Aborted.
0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
54 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt
#0 0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
#1 0x00007ffff717b02a in __GI_abort () at abort.c:89
#2 0x00007ffff71bb7ea in __libc_message (do_abort=do_abort@entry=2, fmt=fmt@entry=0x7ffff72d4e98 "*** Error in `%s': %s: 0x%s ***\n") at ../sysdeps/posix/libc_fatal.c:175
#3 0x00007ffff71c437a in malloc_printerr (ar_ptr=<optimized out>, ptr=<optimized out>, str=0x7ffff72d4fc8 "double free or corruption (!prev)", action=3) at malloc.c:5006
#4 _int_free (av=<optimized out>, p=<optimized out>, have_lock=0) at malloc.c:3867
#5 0x00007ffff71c853c in __GI___libc_free (mem=<optimized out>) at malloc.c:2968
#6 0x000000000053a64e in SSL_SESSION_free ()
#7 0x000000000053630b in SSL_free ()
#8 0x00000000004124de in ssl_sock_close (conn=0xa45080) at src/ssl_sock.c:5086
#9 0x000000000048103a in conn_force_close (conn=0xa45080) at include/proto/connection.h:151
#10 0x0000000000482f09 in stream_free (s=0xaaf230) at src/stream.c:312
#11 0x00000000004891af in process_stream (t=0xa52a40) at src/stream.c:2419
#12 0x000000000050910f in process_runnable_tasks () at src/task.c:238
#13 0x00000000004d5159 in run_poll_loop () at src/haproxy.c:2168
#14 0x00000000004d64ba in main (argc=4, argv=0x7fffffffe658) at src/haproxy.c:2701
(gdb) bt full
#0 0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
resultvar = 0
pid = 29988
selftid = 29988
#1 0x00007ffff717b02a in __GI_abort () at abort.c:89
save_stage = 2
act = {__sigaction_handler = {sa_handler = 0x20302030303a3030, sa_sigaction = 0x20302030303a3030}, sa_mask = {__val = {2314885530818453536, 2314885530818453536, 7017579609838738208, 4206752516204751980, 3545519503966220848, 2314885530818453536, 2314885530818453536, 7795484802351636512, 3917909816998060649,
3276497845987585332, 7161402270846119527, 3615882721633532274, 7378645557452156467, 3472337303646987878, 3991990709698112816, 8223625903104156004}}, sa_flags = 544222583, sa_restorer = 0x5c}
sigs = {__val = {32, 0 <repeats 15 times>}}
#2 0x00007ffff71bb7ea in __libc_message (do_abort=do_abort@entry=2, fmt=fmt@entry=0x7ffff72d4e98 "*** Error in `%s': %s: 0x%s ***\n") at ../sysdeps/posix/libc_fatal.c:175
ap = <error reading variable ap (Attempt to dereference a generic pointer.)>
fd = 7
on_2 = <optimized out>
list = <optimized out>
nlist = <optimized out>
cp = <optimized out>
written = <optimized out>
#3 0x00007ffff71c437a in malloc_printerr (ar_ptr=<optimized out>, ptr=<optimized out>, str=0x7ffff72d4fc8 "double free or corruption (!prev)", action=3) at malloc.c:5006
buf = "0000000000a42590"
cp = <optimized out>
ar_ptr = <optimized out>
str = 0x7ffff72d4fc8 "double free or corruption (!prev)"
action = 3
#4 _int_free (av=<optimized out>, p=<optimized out>, have_lock=0) at malloc.c:3867
size = <optimized out>
fb = <optimized out>
nextchunk = <optimized out>
nextsize = <optimized out>
nextinuse = <optimized out>
prevsize = <optimized out>
bck = <optimized out>
fwd = <optimized out>
errstr = <optimized out>
locked = <optimized out>
#5 0x00007ffff71c853c in __GI___libc_free (mem=<optimized out>) at malloc.c:2968
ar_ptr = <optimized out>
p = <optimized out>
hook = <optimized out>
#6 0x000000000053a64e in SSL_SESSION_free ()
No symbol table info available.
#7 0x000000000053630b in SSL_free ()
No symbol table info available.
#8 0x00000000004124de in ssl_sock_close (conn=0xa45080) at src/ssl_sock.c:5086
No locals.
#9 0x000000000048103a in conn_force_close (conn=0xa45080) at include/proto/connection.h:151
No locals.
#10 0x0000000000482f09 in stream_free (s=0xaaf230) at src/stream.c:312
sess = 0xa529b0
fe = 0xa429f0
bref = 0xa9f850
back = 0xaaf230
cli_conn = 0xa45080
i = 0
#11 0x00000000004891af in process_stream (t=0xa52a40) at src/stream.c:2419
srv = 0x0
s = 0xaaf230
sess = 0xa529b0
rqf_last = 75554848
rpf_last = 2147787360
rq_prod_last = 9
rq_cons_last = 9
rp_cons_last = 9
rp_prod_last = 9
req_ana_back = 0
req = 0xaaf240
res = 0xaaf280
si_f = 0xaaf468
si_b = 0xaaf490
#12 0x000000000050910f in process_runnable_tasks () at src/task.c:238
t = 0xa52a40
i = 0
max_processed = 1
rq_next = 0x0
rewind = 1
local_tasks = {0xa52a40, 0xa45080, 0x7fffffffe3a0, 0x802a130000a45080, 0x7fffffffe3a0, 0x4fa8aa <conn_cond_update_polling+89>, 0xaaf240, 0x9dc3f0 <applet_active_queue>, 0x7fffffffe3d0, 0x4facd8 <conn_fd_handler+802>, 0xf240107009eceb0, 0x500a14ce0, 0x7fffffffe3c0, 0x7fffffffe3c0, 0x7fffffffe3f0,
0xc2769aa5f730bf00}
local_tasks_count = 1
#13 0x00000000004d5159 in run_poll_loop () at src/haproxy.c:2168
next = 254017799
#14 0x00000000004d64ba in main (argc=4, argv=0x7fffffffe658) at src/haproxy.c:2701
err = 0
retry = 200
limit = {rlim_cur = 4011, rlim_max = 4011}
errmsg = "\000@\243\000\000\000\000\000X\346\377\377\377\177\000\000\004\000\000\000\000\000\000\000ʍ\034\367\377\177\000\000\260=\243\000\000\000\000\000\"\000\000\000\000\000\000\000\000\345\377\377\377\177\000\000\370\364\232\000\000\000\000\000\200\346\377\377\377\177\000\000*\373L\000\000\000\000\000\001\000\000\000\001\000\000\000\060?\243\000\000\000\000\000\"\000\000"
pidfd = -1
(gdb)
Here's another one:
*** Error in `/usr/sbin/haproxy': malloc(): memory corruption: 0x0000000000a41ee0 ***
======= Backtrace: =========
/lib/x86_64-linux-gnu/libc.so.6(+0x777e5)[0x7ffff71bb7e5]
/lib/x86_64-linux-gnu/libc.so.6(+0x8213e)[0x7ffff71c613e]
/lib/x86_64-linux-gnu/libc.so.6(__libc_malloc+0x54)[0x7ffff71c8184]
/usr/sbin/haproxy[0x524a36]
/usr/sbin/haproxy[0x522e24]
/usr/sbin/haproxy[0x523402]
/usr/sbin/haproxy[0x533150]
/usr/sbin/haproxy[0x4120b6]
/usr/sbin/haproxy[0x4d9195]
/usr/sbin/haproxy[0x4da24d]
/usr/sbin/haproxy[0x4fab7d]
/usr/sbin/haproxy[0x51202c]
/usr/sbin/haproxy[0x4d51c8]
/usr/sbin/haproxy[0x4d64ba]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf0)[0x7ffff7164830]
/usr/sbin/haproxy[0x4055a9]
======= Memory map: ========
00400000-007af000 r-xp 00000000 ca:02 40972 /usr/sbin/haproxy
009af000-009cf000 r--p 003af000 ca:02 40972 /usr/sbin/haproxy
009cf000-009eb000 rw-p 003cf000 ca:02 40972 /usr/sbin/haproxy
009eb000-00ac5000 rw-p 00000000 00:00 0 [heap]
7ffff0000000-7ffff0021000 rw-p 00000000 00:00 0
7ffff0021000-7ffff4000000 ---p 00000000 00:00 0
7ffff6b0e000-7ffff6b24000 r-xp 00000000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6b24000-7ffff6d23000 ---p 00016000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6d23000-7ffff6d24000 rw-p 00015000 ca:02 24641 /lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6d24000-7ffff7144000 rw-p 00000000 00:00 0
7ffff7144000-7ffff7304000 r-xp 00000000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7304000-7ffff7504000 ---p 001c0000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7504000-7ffff7508000 r--p 001c0000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff7508000-7ffff750a000 rw-p 001c4000 ca:02 26350 /lib/x86_64-linux-gnu/libc-2.23.so
7ffff750a000-7ffff750e000 rw-p 00000000 00:00 0
7ffff750e000-7ffff7526000 r-xp 00000000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7526000-7ffff7725000 ---p 00018000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7725000-7ffff7726000 r--p 00017000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7726000-7ffff7727000 rw-p 00018000 ca:02 24805 /lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7727000-7ffff772b000 rw-p 00000000 00:00 0
7ffff772b000-7ffff7799000 r-xp 00000000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff7799000-7ffff7999000 ---p 0006e000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff7999000-7ffff799a000 r--p 0006e000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff799a000-7ffff799b000 rw-p 0006f000 ca:02 24672 /lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff799b000-7ffff799e000 r-xp 00000000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff799e000-7ffff7b9d000 ---p 00003000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9d000-7ffff7b9e000 r--p 00002000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9e000-7ffff7b9f000 rw-p 00003000 ca:02 26330 /lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9f000-7ffff7ba8000 r-xp 00000000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7ba8000-7ffff7da7000 ---p 00009000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da7000-7ffff7da8000 r--p 00008000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da8000-7ffff7da9000 rw-p 00009000 ca:02 24741 /lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da9000-7ffff7dd7000 rw-p 00000000 00:00 0
7ffff7dd7000-7ffff7dfd000 r-xp 00000000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7fec000-7ffff7ff0000 rw-p 00000000 00:00 0
7ffff7ff5000-7ffff7ff8000 rw-p 00000000 00:00 0
7ffff7ff8000-7ffff7ffa000 r--p 00000000 00:00 0 [vvar]
7ffff7ffa000-7ffff7ffc000 r-xp 00000000 00:00 0 [vdso]
7ffff7ffc000-7ffff7ffd000 r--p 00025000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7ffd000-7ffff7ffe000 rw-p 00026000 ca:02 24651 /lib/x86_64-linux-gnu/ld-2.23.so
7ffff7ffe000-7ffff7fff000 rw-p 00000000 00:00 0
7fffffede000-7ffffffff000 rw-p 00000000 00:00 0 [stack]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
Program received signal SIGABRT, Aborted.
0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
54 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt
#0 0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
#1 0x00007ffff717b02a in __GI_abort () at abort.c:89
#2 0x00007ffff71bb7ea in __libc_message (do_abort=2, fmt=fmt@entry=0x7ffff72d4e98 "*** Error in `%s': %s: 0x%s ***\n") at ../sysdeps/posix/libc_fatal.c:175
#3 0x00007ffff71c613e in malloc_printerr (ar_ptr=0x7ffff7508b20 <main_arena>, ptr=0xa41ee0, str=0x7ffff72d1cff "malloc(): memory corruption", action=<optimized out>) at malloc.c:5006
#4 _int_malloc (av=av@entry=0x7ffff7508b20 <main_arena>, bytes=bytes@entry=16472) at malloc.c:3474
#5 0x00007ffff71c8184 in __GI___libc_malloc (bytes=16472) at malloc.c:2913
#6 0x0000000000524a36 in ssl3_setup_write_buffer ()
#7 0x0000000000522e24 in do_ssl3_write ()
#8 0x0000000000523402 in ssl3_write_bytes ()
#9 0x0000000000533150 in SSL_write ()
#10 0x00000000004120b6 in ssl_sock_from_buf (conn=0xa45080, buf=0xa9f850, flags=1) at src/ssl_sock.c:4974
#11 0x00000000004d9195 in si_conn_send (conn=0xa45080) at src/stream_interface.c:658
#12 0x00000000004da24d in si_conn_send_cb (conn=0xa45080) at src/stream_interface.c:1295
#13 0x00000000004fab7d in conn_fd_handler (fd=5) at src/connection.c:104
#14 0x000000000051202c in fd_process_cached_events () at src/fd.c:240
#15 0x00000000004d51c8 in run_poll_loop () at src/haproxy.c:2186
#16 0x00000000004d64ba in main (argc=4, argv=0x7fffffffe658) at src/haproxy.c:2701
(gdb) bt full
#0 0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
resultvar = 0
pid = 29977
selftid = 29977
#1 0x00007ffff717b02a in __GI_abort () at abort.c:89
save_stage = 2
act = {__sigaction_handler = {sa_handler = 0x20302030303a3030, sa_sigaction = 0x20302030303a3030}, sa_mask = {__val = {2314885530818453536, 2314885530818453536, 7017579609838738208, 4206752516204751980, 3545519503966220848, 2314885530818453536, 2314885530818453536, 7795484802351636512, 3917909816998060649,
3276497845987585332, 7161402270846119527, 3615882721633532274, 7378645557452156467, 3472337303646987878, 3991990709698112816, 8223625903104156004}}, sa_flags = 544222583, sa_restorer = 0x56}
sigs = {__val = {32, 0 <repeats 15 times>}}
#2 0x00007ffff71bb7ea in __libc_message (do_abort=2, fmt=fmt@entry=0x7ffff72d4e98 "*** Error in `%s': %s: 0x%s ***\n") at ../sysdeps/posix/libc_fatal.c:175
ap = <error reading variable ap (Attempt to dereference a generic pointer.)>
fd = 6
on_2 = <optimized out>
list = <optimized out>
nlist = <optimized out>
cp = <optimized out>
written = <optimized out>
#3 0x00007ffff71c613e in malloc_printerr (ar_ptr=0x7ffff7508b20 <main_arena>, ptr=0xa41ee0, str=0x7ffff72d1cff "malloc(): memory corruption", action=<optimized out>) at malloc.c:5006
buf = "0000000000a41ee0"
cp = <optimized out>
ar_ptr = 0x7ffff7508b20 <main_arena>
ptr = 0xa41ee0
str = 0x7ffff72d1cff "malloc(): memory corruption"
action = <optimized out>
#4 _int_malloc (av=av@entry=0x7ffff7508b20 <main_arena>, bytes=bytes@entry=16472) at malloc.c:3474
iters = <optimized out>
nb = 16480
idx = 114
bin = <optimized out>
victim = 0xa41ed0
size = <optimized out>
victim_index = <optimized out>
remainder = <optimized out>
remainder_size = <optimized out>
block = <optimized out>
bit = <optimized out>
map = <optimized out>
fwd = <optimized out>
bck = 0x7ffff7508b78 <main_arena+88>
errstr = 0x0
__func__ = "_int_malloc"
#5 0x00007ffff71c8184 in __GI___libc_malloc (bytes=16472) at malloc.c:2913
ar_ptr = 0x7ffff7508b20 <main_arena>
victim = <optimized out>
hook = <optimized out>
#6 0x0000000000524a36 in ssl3_setup_write_buffer ()
No symbol table info available.
#7 0x0000000000522e24 in do_ssl3_write ()
No symbol table info available.
#8 0x0000000000523402 in ssl3_write_bytes ()
No symbol table info available.
#9 0x0000000000533150 in SSL_write ()
No symbol table info available.
#10 0x00000000004120b6 in ssl_sock_from_buf (conn=0xa45080, buf=0xa9f850, flags=1) at src/ssl_sock.c:4974
ret = 253949018
try = 212
done = 0
#11 0x00000000004d9195 in si_conn_send (conn=0xa45080) at src/stream_interface.c:658
send_flag = 1
si = 0xaa5b18
oc = 0xaa5930
ret = 0
#12 0x00000000004da24d in si_conn_send_cb (conn=0xa45080) at src/stream_interface.c:1295
si = 0xaa5b18
#13 0x00000000004fab7d in conn_fd_handler (fd=5) at src/connection.c:104
conn = 0xa45080
flags = 0
#14 0x000000000051202c in fd_process_cached_events () at src/fd.c:240
fd = 5
entry = 0
e = 50
#15 0x00000000004d51c8 in run_poll_loop () at src/haproxy.c:2186
next = 253899019
#16 0x00000000004d64ba in main (argc=4, argv=0x7fffffffe658) at src/haproxy.c:2701
err = 0
retry = 200
limit = {rlim_cur = 4011, rlim_max = 4011}
errmsg = "\000@\243\000\000\000\000\000X\346\377\377\377\177\000\000\004\000\000\000\000\000\000\000ʍ\034\367\377\177\000\000\260=\243\000\000\000\000\000\"\000\000\000\000\000\000\000\000\345\377\377\377\177\000\000\370\364\232\000\000\000\000\000\200\346\377\377\377\177\000\000*\373L\000\000\000\000\000\001\000\000\000\001\000\000\000\060?\243\000\000\000\000\000\"\000\000"
pidfd = -1
(gdb)
Repro config (fire requests to /robots.txt from curl or browsers):
global
ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:!DSS
ssl-default-bind-options no-tls-tickets no-tlsv10 no-tlsv11 force-tlsv12 prefer-client-ciphers
defaults
log global
option httplog
option dontlognull
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-keep-alive 60s
option http-keep-alive
option forwardfor
frontend tls-termination
mode http
bind :443 ssl crt /etc/ssl/private/temp.example.com.ecdsa crt /etc/ssl/private/ npn http/1.1 alpn http/1.1 curves X25519:P-256 #strict-sni
use_backend robots if { path /robots.txt }
#use_backend temp if { ssl_fc_sni -i temp.example.com }
backend temp
mode http
server local-nginx 127.0.0.1:80 maxconn 200
backend robots
mode http
errorfile 403 /etc/haproxy/errors/robotstxt.http
http-request deny
root@www:/usr/sbin# haproxy -vv
HA-Proxy version 1.8-dev2-8d85aa-52 2017/06/30
Copyright 2000-2017 Willy Tarreau <willy@haproxy.org>
Build options :
TARGET = linux2628
CPU = generic
CC = gcc
CFLAGS = -O0 -g -fno-strict-aliasing -Wdeclaration-after-statement -fwrapv
OPTIONS = USE_GETADDRINFO=1 USE_SLZ=1 USE_OPENSSL=1 USE_PCRE=1 USE_PCRE_JIT=1
Default settings :
maxconn = 2000, bufsize = 16384, maxrewrite = 1024, maxpollevents = 200
Built with OpenSSL version : OpenSSL 1.1.0g-dev xx XXX xxxx
Running on OpenSSL version : OpenSSL 1.1.0g-dev xx XXX xxxx
OpenSSL library supports TLS extensions : yes
OpenSSL library supports SNI : yes
OpenSSL library supports : SSLv3 TLSv1.0 TLSv1.1 TLSv1.2
Built with transparent proxy support using: IP_TRANSPARENT IPV6_TRANSPARENT IP_FREEBIND
Built with network namespace support.
Built with libslz for stateless compression.
Compression algorithms supported : identity("identity"), deflate("deflate"), raw-deflate("deflate"), gzip("gzip")
Encrypted password support via crypt(3): yes
Built with PCRE version : 8.38 2015-11-23
Running on PCRE version : 8.38 2015-11-23
PCRE library supports JIT : yes
Available polling systems :
epoll : pref=300, test result OK
poll : pref=200, test result OK
select : pref=150, test result OK
Total: 3 (3 usable), will use epoll.
Available filters :
[SPOE] spoe
[COMP] compression
[TRACE] trace
Hope this helps,
Lukas
