So here's the core dump for our ETL test with --nightly=stable. We have not yet tried to restart mserver5. Here's some stats on the data dirs:
$ du -sh * 10G bat 4.0K box 5.6G core.6950
$ find bat|wc -l 777933
(box is an empty dir)
Core was generated by `//mochi/bin/mserver5 --dbname=mochiads_stats --dbinit=include sql; --dbfarm=/da'. Program terminated with signal 6, Aborted. #0 0x00002b5e2eb77cab in raise () from /lib/libc.so.6 (gdb) bt #0 0x00002b5e2eb77cab in raise () from /lib/libc.so.6 #1 0x00002b5e2eb79660 in abort () from /lib/libc.so.6 #2 0x00002b5e2ebaf66b in ?? () from /lib/libc.so.6 #3 0x00002b5e2ebb6b23 in ?? () from /lib/libc.so.6 #4 0x00002b5e2ebba26c in free () from /lib/libc.so.6 #5 0x00002b5e2d18f63e in GDKfree (blk=<value optimized out>) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_utils.mx:1140 #6 0x00002b5e2d12bd2d in HEAPfree (h=0x1e7270d0) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_heap.mx:265 #7 0x00002b5e2d1d3a50 in BATfree (b=0x1e726f98) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_bat.mx:522 #8 0x00002b5e2d129d3e in BBPfree (b=0x1e726f98, calledFrom=0x2b5e2d2eb338 "BBPtrim") at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_bbp.mx:1748 #9 0x00002b5e2d12a841 in BBPtrim (memtarget=0, vmtarget=0) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_bbp.mx:2081 #10 0x00002b5e2d190d7d in GDKmemchk (memchk=1, vmchk=761245848) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_utils.mx:688 #11 0x00002b5e2d19166d in GDKvmalloc (size=707393416, maxsize=0x42801688, emergency=<value optimized out>) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_utils.mx:1306 #12 0x00002b5e2d191b30 in GDKmallocmax (size=<value optimized out>, maxsize=0x3439f438, emergency=0) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_utils.mx:1051 #13 0x00002b5e2d12c630 in HEAPalloc (h=0x3439f438, nitems=88424175, itemsize=<value optimized out>) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_heap.mx:127 #14 0x00002b5e2d115a02 in HASHnew (hp=0x1b26, tpe=6, size=21315311, mask=18446744073709551615) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_search.mx:289 #15 0x00002b5e2d116fc4 in BAThash (b=0x2aab2d943118, masksize=<value optimized out>) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_search.mx:383 #16 0x00002b5e2d253853 in BAThashjoin (l=0x190, r=0x2aab2d943118, estimate=<value optimized out>) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_relop.mx:426 #17 0x00002b5e2d2501aa in batjoin (l=0x2aab9c1f6128, r=0x2aabb04f49f8, estimate=9223372036854775808, swap=<value optimized out>) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_relop.mx:1157 #18 0x00002b5e2d25050b in BATjoin (l=0x1b26, r=0x2356, estimate=6) at /scratch/monet.GNU.64.64.d.29168/MonetDB/src/gdk/gdk_relop.mx:1168 #19 0x00002aaab14a5302 in ALGjoinPath (mb=<value optimized out>, stk=0x2aab56174f18, pci=0x245b2378) ---Type <return> to continue, or q <return> to quit--- at /scratch/monet.GNU.64.64.d.29168/MonetDB5/src/optimizer/opt_joinpath.mx:223 #20 0x00002b5e2c880504 in runMALsequence (cntxt=0x604980, mb=0x2aab697741b8, startpc=1, stoppc=0, stk=0x2aab56174f18, env=0x0, pcicaller=0x0) at /scratch/monet.GNU.64.64.d.29168/MonetDB5/src/mal/mal_interpreter.mx:1159 #21 0x00002b5e2c8821cb in callMAL (cntxt=0x604980, mb=0x2aab697741b8, env=0x42803fd8, argv=0x42803fb0, debug=0 '\0') at /scratch/monet.GNU.64.64.d.29168/MonetDB5/src/mal/mal_interpreter.mx:377 #22 0x00002aaab266d19c in SQLexecutePrepared (cntxt=0x604980, be=<value optimized out>, q=0x2aab1c3ca488) at /scratch/monet.GNU.64.64.d.29168/sql/src/backends/monet5/sql_scenario.mx:1188 #23 0x00002aaab266ed0d in SQLengineIntern (c=0x604980, be=0x2aab140554d8) at /scratch/monet.GNU.64.64.d.29168/sql/src/backends/monet5/sql_scenario.mx:1273 #24 0x00002b5e2c8a105d in runScenarioBody (c=0x604980) at /scratch/monet.GNU.64.64.d.29168/MonetDB5/src/mal/mal_scenario.mx:558 #25 0x00002b5e2c8a113a in runScenario (c=0x1b26) at /scratch/monet.GNU.64.64.d.29168/MonetDB5/src/mal/mal_scenario.mx:622 #26 0x00002b5e2c87c1d8 in MSserveClient (dummy=<value optimized out>) at /scratch/monet.GNU.64.64.d.29168/MonetDB5/src/mal/mal_session.mx:471 #27 0x00002b5e2df362a5 in start_thread () from /lib/libpthread.so.0 #28 0x00002b5e2ec1661d in clone () from /lib/libc.so.6 #29 0x0000000000000000 in ?? () (gdb)
And this is the log of everything printed by mserver5:
!WARNING: GDKlockHome: created directory /data/db/monetdb/mochiads_stats/ !WARNING: GDKlockHome: ignoring empty or invalid .gdk_lock. !WARNING: BBPdir: initializing BBP. # MonetDB server v5.2.3, based on kernel v1.20.1 # Serving database 'mochiads_stats' # Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs dynamically linked # Copyright (c) 1993-2007 CWI, all rights reserved # Visit http://monetdb.cwi.nl/ for further information #warning: please don't forget to set your vault key! #(see /mochi/etc/monetdb5.conf) # Listening for connection requests on mapi:monetdb://127.0.0.1:50000/ # MonetDB/SQL module v2.20.3 loaded
*** glibc detected *** //mochi/bin/mserver5: free(): invalid pointer:
0x00002aae5199e008 *** ======= Backtrace: ========= /lib/libc.so.6[0x2b5e2ebb6b23] /lib/libc.so.6(cfree+0x8c)[0x2b5e2ebba26c] /mochi/lib/libbat.so.0(GDKfree+0x2e)[0x2b5e2d18f63e] /mochi/lib/libbat.so.0(HEAPfree+0xad)[0x2b5e2d12bd2d] /mochi/lib/libbat.so.0(BATfree+0x1f0)[0x2b5e2d1d3a50] /mochi/lib/libbat.so.0[0x2b5e2d129d3e] /mochi/lib/libbat.so.0(BBPtrim+0x851)[0x2b5e2d12a841] /mochi/lib/libbat.so.0[0x2b5e2d190d7d] /mochi/lib/libbat.so.0(GDKvmalloc+0xdd)[0x2b5e2d19166d] /mochi/lib/libbat.so.0(GDKmallocmax+0xf0)[0x2b5e2d191b30] /mochi/lib/libbat.so.0(HEAPalloc+0x170)[0x2b5e2d12c630] /mochi/lib/libbat.so.0(HASHnew+0x32)[0x2b5e2d115a02] /mochi/lib/libbat.so.0(BAThash+0x334)[0x2b5e2d116fc4] /mochi/lib/libbat.so.0(BAThashjoin+0x3193)[0x2b5e2d253853] /mochi/lib/libbat.so.0[0x2b5e2d2501aa] /mochi/lib/libbat.so.0(BATjoin+0xb)[0x2b5e2d25050b] /mochi/lib/MonetDB5/lib/lib_optimizer.so(ALGjoinPath+0x222)[0x2aaab14a5302] /mochi/lib/libmal.so.0[0x2b5e2c880504] /mochi/lib/libmal.so.0(callMAL+0x25b)[0x2b5e2c8821cb] /mochi/lib/MonetDB5/lib/lib_sql.so(SQLexecutePrepared+0x25c)[0x2aaab266d19c] /mochi/lib/MonetDB5/lib/lib_sql.so(SQLengineIntern+0x21d)[0x2aaab266ed0d] /mochi/lib/libmal.so.0(runScenarioBody+0x14d)[0x2b5e2c8a105d] /mochi/lib/libmal.so.0(runScenario+0x3a)[0x2b5e2c8a113a] /mochi/lib/libmal.so.0(MSserveClient+0x58)[0x2b5e2c87c1d8] /lib/libpthread.so.0[0x2b5e2df362a5] /lib/libc.so.6(clone+0x6d)[0x2b5e2ec1661d] ======= Memory map: ======== 00400000-00404000 r-xp 00000000 fe:00 2115695 /mochi/bin/mserver5 00604000-00605000 rw-p 00004000 fe:00 2115695 /mochi/bin/mserver5 00605000-3ffec000 rw-p 00605000 00:00 0 [heap] 40000000-40001000 ---p 40000000 00:00 0 40001000-40801000 rw-p 40001000 00:00 0 40801000-40802000 ---p 40801000 00:00 0 40802000-41002000 rw-p 40802000 00:00 0 41002000-41003000 ---p 41002000 00:00 0 41003000-41803000 rw-p 41003000 00:00 0 41803000-41804000 ---p 41803000 00:00 0 41804000-42004000 rw-p 41804000 00:00 0 42004000-42005000 ---p 42004000 00:00 0 42005000-42805000 rw-p 42005000 00:00 0 2aaaaabaf000-2aaaaabb3000 r-xp 00000000 fe:00 2085181 /mochi/lib/MonetDB5/lib/lib_mdb.so.0.0.0 2aaaaabb3000-2aaaaadb3000 ---p 00004000 fe:00 2085181 /mochi/lib/MonetDB5/lib/lib_mdb.so.0.0.0 2aaaaadb3000-2aaaaadb4000 rw-p 00004000 fe:00 2085181 /mochi/lib/MonetDB5/lib/lib_mdb.so.0.0.0 2aaaaadb4000-2aaaaadb6000 r-xp 00000000 fe:00 2084480 /mochi/lib/MonetDB5/lib/lib_xml.so.0.0.0 2aaaaadb6000-2aaaaafb5000 ---p 00002000 fe:00 2084480 /mochi/lib/MonetDB5/lib/lib_xml.so.0.0.0 2aaaaafb5000-2aaaaafb6000 rw-p 00001000 fe:00 2084480 /mochi/lib/MonetDB5/lib/lib_xml.so.0.0.0 2aaaaafbd000-2aaaab03e000 r-xp 00000000 fe:00 540697 /lib/libm-2.5.so 2aaaab03e000-2aaaab23d000 ---p 00081000 fe:00 540697 /lib/libm-2.5.so 2aaaab23d000-2aaaab23f000 rw-p 00080000 fe:00 540697 /lib/libm-2.5.so 2aaaab23f000-2aaaab24d000 r-xp 00000000 fe:00 2016109 /mochi/lib/libMapi.so.0.0.0 2aaaab24d000-2aaaab44c000 ---p 0000e000 fe:00 2016109 /mochi/lib/libMapi.so.0.0.0 2aaaab44c000-2aaaab44d000 rw-p 0000d000 fe:00 2016109 /mochi/lib/libMapi.so.0.0.0 2aaaab44d000-2aaaab44e000 r-xp 00000000 fe:00 2085120 /mochi/lib/MonetDB5/lib/lib_unix.so.0.0.0 2aaaab44e000-2aaaab64d000 ---p 00001000 fe:00 2085120 /mochi/lib/MonetDB5/lib/lib_unix.so.0.0.0 2aaaab64d000-2aaaab64e000 rw-p 00000000 fe:00 2085120 /mochi/lib/MonetDB5/lib/lib_unix.so.0.0.0 2aaaab64e000-2aaaab650000 r-xp 00000000 fe:00 2085099 /mochi/lib/MonetDB5/lib/lib_lock.so.0.0.0 2aaaab650000-2aaaab84f000 ---p 00002000 fe:00 2085099 /mochi/lib/MonetDB5/lib/lib_lock.so.0.0.0 2aaaab84f000-2aaaab850000 rw-p 00001000 fe:00 2085099 /mochi/lib/MonetDB5/lib/lib_lock.so.0.0.0 2aaaab850000-2aaaab851000 r-xp 00000000 fe:00 2085118 /mochi/lib/MonetDB5/lib/lib_sema.so.0.0.0 2aaaab851000-2aaaaba51000 ---p 00001000 fe:00 2085118 /mochi/lib/MonetDB5/lib/lib_sema.so.0.0.0 2aaaaba51000-2aaaaba52000 rw-p 00001000 fe:00 2085118 /mochi/lib/MonetDB5/lib/lib_sema.so.0.0.0 2aaaaba52000-2aaaaba57000 r-xp 00000000 fe:00 2085116