apache2 core dump

[Igor Cicimov <icicimov@xxxxxxxxx>]

Hi all,

I get the following core dumps from the apache:


core 'core.httpd.15844.wenpweb1.500.500.1253648224' of 15844:   ./httpd -k start
-----------------  lwp# 5 / thread# 5  --------------------
 ff29da3c apr_table_add (6c0348, 453800, 453809, 5, fffc0000, b) + b4
 fec521d8 deep_table_copy (6c0348, 0, 5, 4582d4, 38, 1) + 60
 fec522f0 store_headers (135c78, 4528b8, 135c80, 4469c8, 135c80, fec522d4) + 1c
 fec92860 cache_save_filter (0, 0, 4742f, b8f21380, 4742f, b90155c0) + 538
 000454b8 ???????? (44e888, 450838, 7, 0, 0, fea31494)
 000454b8 ???????? (45d2a8, 450838, 7, 0, 4488d0, 45083c)
 fe8d4d18 ap_proxy_http_process_response (ffffdfec, 4528b8, 3511c8, 456898, fd0f9a20, 0) + 6c0
 fe8d5880 proxy_http_handler (0, 1ba088, 1b9f18, 45d4d8, 0, fd0fbab0) + 290
 fe938b4c proxy_run_scheme_handler (4528b8, 1ba088, 1b9f18, 44ebbe, 0, 0) + 64
 fe935cf8 proxy_handler (4528b8, 70, 4528b8, 0, 1b9fa0, c9bd8) + 41c
 0003b994 ???????? (4528b8, 0, 1b, 1bad40, 1ba7b8, 0)
 0003bf38 ???????? (4528b8, 5cc00, 4528b8, fd0fbcdc, fd400028, 1)
 00048444 ???????? (4528b8, 77400, 4, 4528b8, 0, 1e8480)
 00045b00 ???????? (446980, 446890, 446890, 42, 78088, 1bb820)
 00042184 ???????? (446980, 446890, 446890, 42, 446978, 448850)
 0004cb3c ???????? (446848, 446890, 1, 2, 448850, 187e48)
 0004d28c ???????? (187ef0, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187ef0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 6 / thread# 6  --------------------
 fef9dbb8 _lwp_makecontext (fcffb418, 1, 1b7740, 3e8, 0, 4ada4) + 14
 ff2ac204 apr_wait_for_io_or_timeout (11, fcffb418, 0, 0, fcffb674, 3208) + 54
 ff2a7114 apr_socket_sendv (1ee378, fcffb658, 3, fcffb504, 1f9712, b96) + 50
 0003a424 ???????? (1ee378, fcffb650, 4, 1f49, fcffb5a8, fcffb6d0)
 0003ada8 ???????? (0, 6fd7e8, 78348, 5d2b4, 0, 0)
 000454b8 ???????? (1ee9e8, 6fd7e8, 611de, fcffb818, 0, fead0c24)
 000454b8 ???????? (1ee9a8, 6fd7e8, 2, 0, 0, 1f0358)
 0004b004 ???????? (6fd9e0, 6fd7e8, 0, 0, 780e8, 1ee468)
 000454b8 ???????? (6fd9e0, 6fd7e8, 0, 0, 0, 4ada4)
 000454b8 ???????? (1f31c8, 6fd7e8, 0, 0, 0, 0)
 00030b94 ???????? (1f3198, 6fd7e8, 0, 0, 0, 30abc)
 000454b8 ???????? (1f3198, 6fd7e8, 1, 0, 0, 1f06b8)
 ff3270f8 apr_brigade_write (6fd7e8, 45648, 1f3198, fcffbaef, 1, 1ee468) + 168
 00030f88 ???????? (ffffffff, 2e, 0, 0, 6eddc8, fe8249f8)
 fe811f98 status_handler (20c, 0, 0, 1f2380, fe8249fc, fe8249f8) + 710
 0003b994 ???????? (1f2380, 0, f, 1f4160, 1f4060, 0)
 0003bf38 ???????? (1f2380, 5cc00, 1f2380, fcffbcdc, fd400028, 1)
 00048444 ???????? (1f2380, 77400, 4, 1f2380, 78088, 1baf38)
 00045b00 ???????? (1ee468, 1ee378, 1ee378, 43, 78088, 1bb820)
 00042184 ???????? (1ee468, 1ee378, 1ee378, 43, 1ee460, 1f0338)
 0004cb3c ???????? (1ee330, 1ee378, 1, 3, 1f0338, 187e48)
 0004d28c ???????? (187f10, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187f10, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 7 / thread# 7  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee50c00, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bed8d0b0, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fcefbf1c, fcefbf18, 0, 3a48a0, 187e48)
 0004d1e4 ???????? (187f30, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187f30, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 8 / thread# 8  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee50e00, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bed81995, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fcdfbf1c, fcdfbf18, 0, 309068, 187e48)
 0004d1e4 ???????? (187f50, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187f50, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 9 / thread# 9  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51000, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bed6e868, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fccfbf1c, fccfbf18, 0, 1f0338, 187e48)
 0004d1e4 ???????? (187f70, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187f70, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 10 / thread# 10  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51200, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bee29721, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fcbfbf1c, fcbfbf18, 0, 3a48a0, 187e48)
 0004d1e4 ???????? (187f90, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187f90, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 11 / thread# 11  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51400, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bee29721, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fcafbf1c, fcafbf18, 0, 3230d0, 187e48)
 0004d1e4 ???????? (187fb0, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187fb0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 12 / thread# 12  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51600, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bee165f4, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fc9fbf1c, fc9fbf18, 0, 1f0338, 187e48)
 0004d1e4 ???????? (187fd0, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187fd0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 13 / thread# 13  --------------------
 fef9fc00 _lwp_suspend (4d7080, 4dcfa8, 4, 4dcfa8, 0, 1) + 18
 00045af0 ???????? (4d7088, 4d6f98, 4d6f98, 4a, 78088, 1bb820)
 00042184 ???????? (4d7088, 4d6f98, 4d6f98, 4a, 4d7080, 4d8f58)
 0004cb3c ???????? (4d6f50, 4d6f98, 1, a, 4d8f58, 187e48)
 0004d28c ???????? (187ff0, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187ff0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 14 / thread# 14  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51a00, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bed5f444, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fc7fbf1c, fc7fbf18, 0, 448850, 187e48)
 0004d1e4 ???????? (188010, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (188010, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 15 / thread# 15  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51c00, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bedd9564, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fc6fbf1c, fc6fbf18, 0, 341148, 187e48)
 0004d1e4 ???????? (188030, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (188030, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 16 / thread# 16  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51e00, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bedf40a3, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fc5fbf1c, fc5fbf18, 0, 1f0338, 187e48)
 0004d1e4 ???????? (188050, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (188050, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 17 / thread# 17  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52000, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bee31133, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fc4fbf1c, fc4fbf18, 0, 3a48a0, 187e48)
 0004d1e4 ???????? (188070, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (188070, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 18 / thread# 18  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52200, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bedca140, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fc3fbf1c, fc3fbf18, 0, 3230d0, 187e48)
 0004d1e4 ???????? (188090, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (188090, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 19 / thread# 19  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52400, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bed4c317, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fc2fbf1c, fc2fbf18, 0, 309068, 187e48)
 0004d1e4 ???????? (1880b0, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (1880b0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 20 / thread# 20  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52600, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bee034c7, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fc1fbf1c, fc1fbf18, 0, 1f0338, 187e48)
 0004d1e4 ???????? (1880d0, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (1880d0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 21 / thread# 21  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52800, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bed50020, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fc0fbf1c, fc0fbf18, 0, 245620, 187e48)
 0004d1e4 ???????? (1880f0, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (1880f0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 22 / thread# 22  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52a00, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bed7dc8c, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fbffbf1c, fbffbf18, 0, 3a48a0, 187e48)
 0004d1e4 ???????? (188110, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (188110, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 23 / thread# 23  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52c00, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bedb7013, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fbefbf1c, fbefbf18, 0, 3a48a0, 187e48)
 0004d1e4 ???????? (188130, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (188130, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 24 / thread# 24  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52e00, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bee21d0f, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fbdfbf1c, fbdfbf18, 0, 4e8f98, 187e48)
 0004d1e4 ???????? (188150, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (188150, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 25 / thread# 25  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee53000, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bedaf601, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fbcfbf1c, fbcfbf18, 0, 448850, 187e48)
 0004d1e4 ???????? (188170, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (188170, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 26 / thread# 26  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee53200, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, beda01dd, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fbbfbf1c, fbbfbf18, 0, 3230d0, 187e48)
 0004d1e4 ???????? (188190, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (188190, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 27 / thread# 27  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee53400, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bedec691, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fbafbf1c, fbafbf18, 0, 3a48a0, 187e48)
 0004d1e4 ???????? (1881b0, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (1881b0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 28 / thread# 28  --------------------
 fef9c330 lseek64  (16, 7, ff2c1bb0, 0, 0, 0) + 24
 fee6df28 fcntl    (16, 7, ff2c1bb0, 0, fee51400, ff2c1cf4) + 74
 ff2a29d0 proc_mutex_fcntl_acquire (186638, 33f180, 3a28d8, 1, 187cf0, ff2c1b5c) + 24
 ff2a2cf4 apr_proc_mutex_lock (186638, fb9fbf00, 1ee330, e4bb0, fb9fbef8, 81df8) + c
 0004d01c ???????? (81b68, 10, 1881d0, 78000, 1, 77400)
 ff2ad898 dummy_worker (1881d0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 1 / thread# 1  --------------------
 fef9ee28 ProcUpdate (a, ffbff873, 1, 187c60, 187c60, 328) + 5c
 0004fa64 ???????? (15fc58, 4cbc4, 4d3c0, 187c38, 0, 1)
 0004dab8 ???????? (1, 4c6f8, 0, 0, 76c00, 0)
 0004dd58 ???????? (77400, 1, 0, 0, 11, 78000)
 0004e0cc ???????? (1, 1, 11, 11, 28, 1)
 0004e28c ???????? (0, 6cc, 7, 0, 78400, 78000)
 0004e5c8 ???????? (837a8, b1860, 0, 856a0, 78000, 78000)
 000291f0 ???????? (75c00, 856a0, 77800, 77800, 0, 0)
 000282d0 ???????? (0, 0, 0, 0, 0, 0)
-----------------  lwp# 3 / thread# 3  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee50400, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bede4c7f, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fd2fbf1c, fd2fbf18, 0, 3a48a0, 187e48)
 0004d1e4 ???????? (187eb0, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187eb0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 4 / thread# 4  --------------------
 fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee50600, fee88000) + 2c
 fee73358 cond_wait (187cd0, 187ca0, 11d0, bee29721, 23a, 2f) + 14
 fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
 0004f96c ???????? (187c80, fd1fbf1c, fd1fbf18, 0, 245620, 187e48)
 0004d1e4 ???????? (187ed0, 1, 0, 0, 0, 0)
 ff2ad898 dummy_worker (187ed0, 0, 0, 0, 0, 0) + c
 fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
-----------------  lwp# 2 / thread# 2  --------------------
 ff2ad88c dummy_worker(), exit value = 0x00000000
        ** zombie (exited, not detached, not yet joined) **


It's apache 2.2.12 running as reverse proxy on Solaris 9 box with mpm_worker module. Looks like the threads get in some conditional wait state and the process becomes zombie. I can see individual child processes dieing in the log file with segmentation fault.

Any idea what might be causing these core dumps?

Thanks in advance for any suggestion/idea. Please tell me if anything else needed for troubleshooting.

Igor