Re: [Monetdb-developers] MonetDB: default - Plug memory leak.

(on my 64-bit Fedora 14 desktop) this checkin seems cause several tests to fail or even crash:

testing status with changeset 42821:3967ec4a8aef:

monetdb5: !ERROR: Testing FAILED SIGNIFICANTLY (4 out of 494 tests failed) 1 out of 494 tests ran into timeout 3 out of 494 tests produced SIGNIFICANTLY different output

sql: !ERROR: Testing FAILED SIGNIFICANTLY (4 out of 1161 tests failed) 15 out of 1161 tests could not be executed 1 out of 1161 tests produced slightly different output 3 out of 1161 tests produced SIGNIFICANTLY different output

geom: !ERROR: Testing FAILED SIGNIFICANTLY (1 out of 16 tests failed) 1 out of 16 tests resulted in a crash

testing status with changeset 42825:f15583fdc322

monetdb5: !ERROR: Testing FAILED SIGNIFICANTLY (21 out of 494 tests failed) 1 out of 494 tests ran into timeout 20 out of 494 tests produced SIGNIFICANTLY different output

sql: !ERROR: Testing FAILED SIGNIFICANTLY (909 out of 1161 tests failed) 248 out of 1161 tests could not be executed 6 out of 1161 tests ran into timeout 900 out of 1161 tests resulted in a crash 3 out of 1161 tests produced SIGNIFICANTLY different output

geom: !ERROR: Testing FAILED SIGNIFICANTLY (16 out of 16 tests failed) 16 out of 16 tests resulted in a crash

I have not yet looked into the details.

Stefan

On Fri, Jan 13, 2012 at 06:08:12PM +0100, Sjoerd Mullender wrote:

Changeset: f15583fdc322 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f15583fdc322 Modified Files: clients/Tests/exports.stable.out gdk/gdk.mx gdk/gdk_atoms.mx monetdb5/mal/mal_atom.c monetdb5/mal/mal_interpreter.mx monetdb5/mal/mal_resolve.mx monetdb5/modules/atoms/blob.c monetdb5/modules/atoms/blob.mal monetdb5/modules/mal/Tests/inspect05.stable.out Branch: default Log Message:

Plug memory leak. Variables of types other than the standard built-in types could not be freed since the interpreter didn't know what to do. We now have a new function pointer in BATatoms that the interpreter can use to destroy an atom. Typically the function pointer will be NULL, but for e.g. BLOBs it should be GDKfree. This function pointer can be set using the new command destroy() address XXX; construct in MAL.

diffs (278 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -954,6 +954,7 @@ str BLOBblob_blob(blob **d, blob **s); str BLOBblob_fromstr(blob **b, str *d); void BLOBconvert(blob *b, int direction); void BLOBdel(Heap *h, var_t *index); +void BLOBdestroy(blob *b); str BLOBeoln(char *src, char *end); str BLOBfromblob(str *retval, blob **b); str BLOBfromidx(str *retval, blob **binp, int *index); diff --git a/gdk/gdk.mx b/gdk/gdk.mx --- a/gdk/gdk.mx +++ b/gdk/gdk.mx @@ -2125,6 +2125,7 @@ typedef struct { void (*atomConvert) (ptr v, int direction); int (*atomFix) (ptr atom); int (*atomUnfix) (ptr atom);

• void (*atomDestroy) (ptr atom);

  /* varsized atom-only ADT functions */ var_t (*atomPut) (Heap *, var_t *off, ptr src);

diff --git a/gdk/gdk_atoms.mx b/gdk/gdk_atoms.mx --- a/gdk/gdk_atoms.mx +++ b/gdk/gdk_atoms.mx @@ -2350,64 +2350,44 @@ atomDesc BATatoms[MAXATOMS] = { (int (*)(ptr, ptr)) lngCmp, (BUN (*)(ptr)) lngHash, 0, #endif

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"bit", TYPE_chr, 1, sizeof(bit), sizeof(bit), 0, 0, (ptr) &chr_nil, (int (*)(str, int *, ptr *)) bitFromStr, (int (*)(str *, int *, ptr)) bitToStr, (void *(*)(void *, stream *, size_t)) bitRead, (int (*)(void *, stream *, size_t)) bitWrite, (int (*)(ptr, ptr)) chrCmp, (BUN (*)(ptr)) chrHash, 0,

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"chr", TYPE_chr, 1, sizeof(chr), sizeof(chr), 0, 0, (ptr) &chr_nil, (int (*)(str, int *, ptr *)) chrFromStr, (int (*)(str *, int *, ptr)) chrToStr, (void *(*)(void *, stream *, size_t)) chrRead, (int (*)(void *, stream *, size_t)) chrWrite, (int (*)(ptr, ptr)) chrCmp, (BUN (*)(ptr)) chrHash, 0,

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"bte", TYPE_bte, 1, sizeof(bte), sizeof(bte), 0, 0, (ptr) &bte_nil, (int (*)(str, int *, ptr *)) bteFromStr, (int (*)(str *, int *, ptr)) bteToStr, (void *(*)(void *, stream *, size_t)) bteRead, (int (*)(void *, stream *, size_t)) bteWrite, (int (*)(ptr, ptr)) bteCmp, (BUN (*)(ptr)) bteHash, 0,

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"sht", TYPE_sht, 1, sizeof(sht), sizeof(sht), 0, 0, (ptr) &sht_nil, (int (*)(str, int *, ptr *)) shtFromStr, (int (*)(str *, int *, ptr)) shtToStr, (void *(*)(void *, stream *, size_t)) shtRead, (int (*)(void *, stream *, size_t)) shtWrite, (int (*)(ptr, ptr)) shtCmp, (BUN (*)(ptr)) shtHash, (void (*)(ptr, int)) shtConvert,

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"BAT", TYPE_int, 1, sizeof(bat), sizeof(bat), 0, 0, (ptr) &int_nil, (int (*)(str, int *, ptr *)) batFromStr, (int (*)(str *, int *, ptr)) batToStr, (void *(*)(void *, stream *, size_t)) batRead, (int (*)(void *, stream *, size_t)) batWrite, (int (*)(ptr, ptr)) intCmp, (BUN (*)(ptr)) intHash, (void (*)(ptr, int)) intConvert, (int (*)(ptr)) batFix, (int (*)(ptr)) batUnfix,

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0},
  

  {"int", TYPE_int, 1, sizeof(int), sizeof(int), 0, 0, (ptr) &int_nil, (int (*)(str, int *, ptr *)) intFromStr, (int (*)(str *, int *, ptr)) intToStr, (void *(*)(void *, stream *, size_t)) intRead, (int (*)(void *, stream *, size_t)) intWrite, (int (*)(ptr, ptr)) intCmp, (BUN (*)(ptr)) intHash, (void (*)(ptr, int)) intConvert,

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"oid",

#if SIZEOF_OID == SIZEOF_INT TYPE_int, 1, sizeof(oid), sizeof(oid), 0, 0, (ptr) &int_nil, @@ -2422,10 +2402,7 @@ atomDesc BATatoms[MAXATOMS] = { (int (*)(ptr, ptr)) lngCmp, (BUN (*)(ptr)) lngHash, (void (*)(ptr, int)) lngConvert, #endif

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"wrd",

#if SIZEOF_WRD == SIZEOF_INT TYPE_int, 1, sizeof(wrd), sizeof(wrd), 0, 0, (ptr) &int_nil, @@ -2440,10 +2417,7 @@ atomDesc BATatoms[MAXATOMS] = { (int (*)(ptr, ptr)) lngCmp, (BUN (*)(ptr)) lngHash, (void (*)(ptr, int)) lngConvert, #endif

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"ptr",

#if SIZEOF_VOID_P == SIZEOF_INT TYPE_ptr, 1, sizeof(ptr), sizeof(ptr), 0, 0, (ptr) &ptr_nil, @@ -2458,43 +2432,31 @@ atomDesc BATatoms[MAXATOMS] = { (int (*)(ptr, ptr)) lngCmp, (BUN (*)(ptr)) lngHash, (void (*)(ptr, int)) lngConvert, #endif

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"flt", TYPE_flt, 1, sizeof(flt), sizeof(flt), 0, 0, (ptr) &flt_nil, (int (*)(str, int *, ptr *)) fltFromStr, (int (*)(str *, int *, ptr)) fltToStr, (void *(*)(void *, stream *, size_t)) fltRead, (int (*)(void *, stream *, size_t)) fltWrite, (int (*)(ptr, ptr)) fltCmp, (BUN (*)(ptr)) intHash, (void (*)(ptr, int)) intConvert,

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"dbl", TYPE_dbl, 1, sizeof(dbl), sizeof(dbl), 0, 0, (ptr) &dbl_nil, (int (*)(str, int *, ptr *)) dblFromStr, (int (*)(str *, int *, ptr)) dblToStr, (void *(*)(void *, stream *, size_t)) dblRead, (int (*)(void *, stream *, size_t)) dblWrite, (int (*)(ptr, ptr)) dblCmp, (BUN (*)(ptr)) lngHash, (void (*)(ptr, int)) lngConvert,

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"lng", TYPE_lng, 1, sizeof(lng), sizeof(lng), 0, 0, (ptr) &lng_nil, (int (*)(str, int *, ptr *)) lngFromStr, (int (*)(str *, int *, ptr)) lngToStr, (void *(*)(void *, stream *, size_t)) lngRead, (int (*)(void *, stream *, size_t)) lngWrite, (int (*)(ptr, ptr)) lngCmp, (BUN (*)(ptr)) lngHash, (void (*)(ptr, int)) lngConvert,

• 0, 0,
  

• 0, 0,
  

• 0, 0,
  

• 0, 0},
  

• 0, 0, 0, 0, 0, 0, 0, 0, 0},
  

  {"str", TYPE_str, 1, sizeof(var_t), sizeof(var_t), 0, 1, (ptr) str_nil, (int (*)(str, int *, ptr *)) strFromStr, (int (*)(str *, int *, ptr)) strToStr, (void *(*)(void *, stream *, size_t)) strRead, (int (*)(void *, stream *, size_t)) strWrite, (int (*)(ptr, ptr)) strCmp, (BUN (*)(ptr)) strHash, 0,

• 0, 0,
  

• 0, 0, GDKfree,
  

  (var_t (*)(Heap *, var_t *, ptr)) strPut, 0, (int (*)(ptr)) strLen, strHeap, (void (*)(Heap *, int)) 0, 0},

diff --git a/monetdb5/mal/mal_atom.c b/monetdb5/mal/mal_atom.c --- a/monetdb5/mal/mal_atom.c +++ b/monetdb5/mal/mal_atom.c @@ -118,6 +118,11 @@ int malAtomProperty(MalBlkPtr mb, InstrP setAtomName(pci); return 1; }

• if (idcmp("destroy", name) == 0 && pci->argc == 1) {
  

• 	BATatoms[tpe].atomDestroy = (void (*)(void *))pci->fcn;
  

• 	setAtomName(pci);
  

• 	return 1;
  

• }
  

  break; case 'c': if (idcmp("cmp", name) == 0 && pci->argc == 1) {

diff --git a/monetdb5/mal/mal_interpreter.mx b/monetdb5/mal/mal_interpreter.mx --- a/monetdb5/mal/mal_interpreter.mx +++ b/monetdb5/mal/mal_interpreter.mx @@ -2442,9 +2442,7 @@ void garbageElement(Client cntxt, ValPtr v->val.sval = NULL; } v->len = 0;

• return;
  

• }
• if (v->vtype == TYPE_bat) {

• } else if (v->vtype == TYPE_bat) { /*
  • @-
  • All operations are responsible to properly set the

@@ -2466,6 +2464,8 @@ void garbageElement(Client cntxt, ValPtr if (cntxt && cntxt->flags & bigfootFlag) updateBigFoot(cntxt, bid, FALSE); BBPdecref(bid, TRUE);

• } else if (BATatoms[v->vtype].atomDestroy) {

• (*BATatoms[v->vtype].atomDestroy)(v->val.pval);
  

  }

} /* diff --git a/monetdb5/mal/mal_resolve.mx b/monetdb5/mal/mal_resolve.mx --- a/monetdb5/mal/mal_resolve.mx +++ b/monetdb5/mal/mal_resolve.mx @@ -529,7 +529,8 @@ findFunctionType(Module scope, MalBlkPtr isaBatType(@1) || findGDKtype(@1) == TYPE_str || (!isPolyType(@1) && @1 < TYPE_any &&

•  @1 >= 0 && ATOMstorage(@1) == TYPE_str)) {
  

•  @1 >= 0 && (ATOMstorage(@1) == TYPE_str ||
  

• 			 BATatoms[@1].atomDestroy))) {
  

  getInstrPtr(@3, 0)->gc |= GARBAGECONTROL; setVarCleanup(mb, getArg(p, @2)); p->gc |= GARBAGECONTROL;

@@ -783,7 +784,9 @@ void typeChecker(Module scope, MalBlkPtr int tpe = getArgType(mb,p,k); if (findGDKtype(tpe) == TYPE_bat || findGDKtype(tpe) == TYPE_str ||

• 	(!isPolyType(tpe) && tpe< TYPE_any && ATOMstorage(tpe) == TYPE_str))
  

• 	(!isPolyType(tpe) && tpe< TYPE_any &&
  

• 	 (ATOMstorage(tpe) == TYPE_str ||
  

• 	  BATatoms[tpe].atomDestroy)))
  setVarCleanup(mb,getArg(p,k));
  

  }

} diff --git a/monetdb5/modules/atoms/blob.c b/monetdb5/modules/atoms/blob.c --- a/monetdb5/modules/atoms/blob.c +++ b/monetdb5/modules/atoms/blob.c @@ -68,6 +68,7 @@ blob_export int BLOBnitems(int *ret, blo blob_export int BLOBget(Heap *h, int *bun, int *l, blob **val); blob_export blob * BLOBread(blob *a, stream *s, size_t cnt); blob_export int BLOBwrite(blob *a, stream *s, size_t cnt); +blob_export void BLOBdestroy(blob *b);

blob_export str BLOBblob_blob(blob **d, blob **s); blob_export str BLOBblob_fromstr(blob **b, str *d); @@ -741,6 +742,12 @@ BLOBheap(Heap *heap, size_t capacity) blob_heap(heap, capacity); }

+void +BLOBdestroy(blob *b) +{

• GDKfree(b);

+}

var_t BLOBput(Heap *h, var_t *bun, blob *val) { diff --git a/monetdb5/modules/atoms/blob.mal b/monetdb5/modules/atoms/blob.mal --- a/monetdb5/modules/atoms/blob.mal +++ b/monetdb5/modules/atoms/blob.mal @@ -31,6 +31,7 @@ command put() address BLOBput; command del() address BLOBdel; command length() address BLOBlength; command heap() address BLOBheap; +command destroy() address BLOBdestroy;

command blob(s:blob):blob address BLOBblob_blob diff --git a/monetdb5/modules/mal/Tests/inspect05.stable.out b/monetdb5/modules/mal/Tests/inspect05.stable.out --- a/monetdb5/modules/mal/Tests/inspect05.stable.out +++ b/monetdb5/modules/mal/Tests/inspect05.stable.out @@ -3399,6 +3399,7 @@ end main; [ "blob", "command", "blob", "(s:str):blob ", "BLOBblob_fromstr;" ] [ "blob", "command", "blob", "(s:blob):blob ", "BLOBblob_blob;" ] [ "#convert", "command", "blob", "():void ", "BLOBconvert;" ] +[ "#destroy", "command", "blob", "():void ", "BLOBdestroy;" ] [ "#del", "command", "blob", "():void ", "BLOBdel;" ] [ "#fromstr", "command", "blob", "():void ", "BLOBfromstr;" ] [ "#heap", "command", "blob", "():void ", "BLOBheap;" ] _______________________________________________ Checkin-list mailing list Checkin-list@monetdb.org http://mail.monetdb.org/mailman/listinfo/checkin-list