[Monetdb-developers] [Monetdb-checkins] MonetDB/src/gdk gdk_posix.mx, , 1.157, 1.158

Stefan Manegold Stefan.Manegold at cwi.nl
Sat Oct 25 14:04:29 CEST 2008


On Sat, Oct 25, 2008 at 07:30:50AM +0000, Martin Kersten wrote:
> Update of /cvsroot/monetdb/MonetDB/src/gdk
> In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv30373
> 
> Modified Files:
> 	gdk_posix.mx 
> Log Message:
> The memory map table maintained for the vmtrim
> should be aligned to the number of mapped files.
> In more recent applications, such as the SkyServer
> this may be around 2000 BATs easily.
> For the time being the limit of 100 is increased to 4096
> 
> The remainder are additions of '#' in error messages
> to prepare for problems in the API.

just an idea (without background knowledge):
shouldn't we introduce a new symbol/chracter in the protocol to indicate
warnings and informative message?
'!' indicates errors (obviously never to be ignored)
'#' indicates (usually to be ignored) comments
'?' to be added to indicate warnings (and "informative messages)?
'?' to be added for ... ??

Stefan

> 
> 
> 
> U gdk_posix.mx
> Index: gdk_posix.mx
> ===================================================================
> RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_posix.mx,v
> retrieving revision 1.157
> retrieving revision 1.158
> diff -u -d -r1.157 -r1.158
> --- gdk_posix.mx	10 Oct 2008 08:50:35 -0000	1.157
> +++ gdk_posix.mx	25 Oct 2008 07:30:48 -0000	1.158
> @@ -389,7 +389,7 @@
>   * hope the VM paging algorithm behaves better than Linux which just runs off
>   * the cliff and if MonetDB does not prevent RSS from being too high, enters coma.
>   *
> - * We will only eb able to sensibly test this on Windows64. On Windows32, mmap sizes
> + * We will only be able to sensibly test this on Windows64. On Windows32, mmap sizes
>   * do not significantly exceed RAM sizes so MonetDB swapping actually will not happen
>   * (of course, you've got this nasty problem of VM fragemntation and failing mmaps instead).
>   *
> @@ -488,6 +488,10 @@
>   * to the cursors (i.e. pointers to b->batBuns->free or the cursors
>   * in radix-cluster), which an enhanced version of this thread might
>   * take into account.
> + *
> + * [Kersten] The memory map table should be aligned to the number of
> + * mapped files. In more recent applications, such as the SkyServer
> + * this may be around 2000 BATs easily.
>   */
>  
>  #ifdef HAVE_PTHREAD_H
> @@ -519,7 +523,7 @@
>  #endif
>  
>  #define MT_MMAP_TILE (1<<27)
> -#define MT_MMAP_BUFSIZE 100
> +#define MT_MMAP_BUFSIZE 4096
>  MT_mmap_t MT_mmap_tab[MT_MMAP_BUFSIZE];
>  int MT_mmap_cur = -1, MT_mmap_first = -1, MT_mmap_free = 0;
>  
> @@ -598,7 +602,7 @@
>  		if (MT_mmap_cur == -1)
>  			MT_mmap_cur = i;
>  #ifdef MMAP_DEBUG
> -		stream_printf(GDKerr, "MT_mmap_new: %s fd=%d\n", path, fd);
> +		stream_printf(GDKerr, "#MT_mmap_new: %s fd=%d\n", path, fd);
>  #endif
>  		strncpy(MT_mmap_tab[i].path, path, 128);
>  		MT_mmap_tab[i].base = base;
> @@ -642,13 +646,13 @@
>  				/* tell the OS quite clearly that you want to drop this */
>  				ret = posix_fadvise(MT_mmap_tab[victim].fd, 0LL, MT_mmap_tab[victim].len, POSIX_FADV_DONTNEED);
>  #ifdef MMAP_DEBUG
> -				stream_printf(GDKerr, "MT_mmap_del: posix_fadvise(%s,fd=%d,%uMB,POSIX_FADV_DONTNEED) = %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, (unsigned int) (MT_mmap_tab[victim].len >> 20), ret);
> +				stream_printf(GDKerr, "#MT_mmap_del: posix_fadvise(%s,fd=%d,%uMB,POSIX_FADV_DONTNEED) = %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, (unsigned int) (MT_mmap_tab[victim].len >> 20), ret);
>  #endif
>  			}
>  #endif
>  			ret = close(MT_mmap_tab[victim].fd);
>  #ifdef MMAP_DEBUG
> -			stream_printf(GDKerr, "MT_mmap_del: close(%s fd=%d) = %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, ret);
> +			stream_printf(GDKerr, "#MT_mmap_del: close(%s fd=%d) = %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, ret);
>  #endif
>  			MT_mmap_tab[victim].next = MT_mmap_free;
>  			MT_mmap_empty(victim);
> @@ -674,7 +678,7 @@
>  			if (MT_mmap_tab[i].fd >= 0) {
>  				ret = posix_fadvise(MT_mmap_tab[i].fd, 0, len, advice);
>  #ifdef MMAP_DEBUG
> -				stream_printf(GDKerr, "MT_fadvise: posix_fadvise(%s,fd=%d,%uMB,%d) = %d\n", MT_mmap_tab[i].path, MT_mmap_tab[i].fd, (unsigned int) (len >> 20), advice, ret);
> +				stream_printf(GDKerr, "#MT_fadvise: posix_fadvise(%s,fd=%d,%uMB,%d) = %d\n", MT_mmap_tab[i].path, MT_mmap_tab[i].fd, (unsigned int) (len >> 20), advice, ret);
>  #endif
>  			}
>  		}
> @@ -697,7 +701,7 @@
>  	int ret = posix_madvise(MT_mmap_tab[i].base + off, MT_MMAP_TILE, POSIX_MADV_DONTNEED);
>  
>  	if (err) {
> -		stream_printf(err, "MT_mmap_unload_tile: posix_madvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n",
> +		stream_printf(err, "#MT_mmap_unload_tile: posix_madvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n",
>  			      MT_mmap_tab[i].path, (unsigned int) (off >> 20),
>  			      (unsigned int) (MT_MMAP_TILE >> 20), MT_mmap_tab[i].fd, ret);
>  	}
> @@ -706,7 +710,7 @@
>  		/* tell the OS quite clearly that you want to drop this */
>  		ret = posix_fadvise(MT_mmap_tab[i].fd, off, MT_MMAP_TILE, POSIX_FADV_DONTNEED);
>  		if (err) {
> -			stream_printf(err, "MT_mmap_unload_tile: posix_fadvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n",
> +			stream_printf(err, "#MT_mmap_unload_tile: posix_fadvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n",
>  				      MT_mmap_tab[i].path, (unsigned int) (off >> 20),
>  				      (unsigned int) (MT_MMAP_TILE >> 20), MT_mmap_tab[i].fd, ret);
>  		}
> @@ -721,7 +725,7 @@
>  
>  	/* save to disk an 128MB tile, and observe how long this takes */
>  	if (err) {
> -		stream_printf(err, "MT_mmap_save_tile: msync(%s,off=%uM,%u,SYNC)...\n",
> +		stream_printf(err, "#MT_mmap_save_tile: msync(%s,off=%uM,%u,SYNC)...\n",
>  			      MT_mmap_tab[i].path, (unsigned int) (tile >> 20),
>  			      (unsigned int) (MT_MMAP_TILE >> 20));
>  	}
> @@ -729,7 +733,7 @@
>  	ret = MT_msync(MT_mmap_tab[i].base, tile, MT_MMAP_TILE, MMAP_SYNC);
>  	t = GDKms() - t;
>  	if (err) {
> -		stream_printf(err, "MT_mmap_save_tile: msync(%s,tile=%uM,%uM,SYNC) = %d (%dms)\n",
> +		stream_printf(err, "#MT_mmap_save_tile: msync(%s,tile=%uM,%uM,SYNC) = %d (%dms)\n",
>  			      MT_mmap_tab[i].path, (unsigned int) (tile >> 20),
>  			      (unsigned int) (MT_MMAP_TILE >> 20), ret, t);
>  	}
> @@ -768,7 +772,7 @@
>  
>  	(void) pthread_mutex_lock(&MT_mmap_lock);
>  	if (err) {
> -		stream_printf(err, "MT_mmap_trim(%u MB): rss = %u MB\n", (unsigned int) ((target) >> 20), (unsigned int) (rss >> 20));
> +		stream_printf(err, "#MT_mmap_trim(%u MB): rss = %u MB\n", (unsigned int) ((target) >> 20), (unsigned int) (rss >> 20));
>  	}
>  	if (rss > target) {
>  		/* try to selectively unload pages from the writable regions */
> @@ -1085,7 +1089,7 @@
>  	int ret = munmap(p, len);
>  
>  #ifdef MMAP_DEBUG
> -	stream_printf(GDKerr, "munmap(" LLFMT "," LLFMT ",%d) = %d\n", (long long) p, (long long) len, ret);
> +	stream_printf(GDKerr, "#munmap(" LLFMT "," LLFMT ",%d) = %d\n", (long long) p, (long long) len, ret);
>  #endif
>  	MT_mmap_del(p, len);
>  	return ret;
> @@ -1097,7 +1101,7 @@
>  	int ret = msync(((char*) p)+off, len, (mode & MMAP_SYNC) ? MS_SYNC : ((mode & MMAP_ASYNC) ? MS_ASYNC : MS_INVALIDATE));
>  
>  #ifdef MMAP_DEBUG
> -	stream_printf(GDKerr, "msync(" LLFMT "," LLFMT ",%s) = %d\n", (long long) p, (long long) len, (mode & MMAP_SYNC) ? "MS_SYNC" : ((mode & MMAP_ASYNC) ? "MS_ASYNC" : "MS_INVALIDATE"), ret);
> +	stream_printf(GDKerr, "#msync(" LLFMT "," LLFMT ",%s) = %d\n", (long long) p, (long long) len, (mode & MMAP_SYNC) ? "MS_SYNC" : ((mode & MMAP_ASYNC) ? "MS_ASYNC" : "MS_INVALIDATE"), ret);
>  #endif
>  	if (ret < 0)
>  		return errno;
> @@ -1110,7 +1114,7 @@
>  	int ret = posix_madvise(p, len, advise);
>  
>  #ifdef MMAP_DEBUG
> -	stream_printf(GDKerr, "posix_madvise(" PTRFMT "," SZFMT ",%d) = %d\n", PTRFMTCAST p, len, advise, ret);
> +	stream_printf(GDKerr, "#posix_madvise(" PTRFMT "," SZFMT ",%d) = %d\n", PTRFMTCAST p, len, advise, ret);
>  #endif
>  	if (MT_fadvise(p, len, advise))
>  		ret = -1;
> @@ -1468,7 +1472,7 @@
>  	}
>  	if (heapstatus == _HEAPBADPTR || heapstatus == _HEAPBADBEGIN || heapstatus == _HEAPBADNODE) {
>  
> -		stream_printf(GDKerr, "mallinfo(): heap is corrupt.");
> +		stream_printf(GDKerr, "#mallinfo(): heap is corrupt.");
>  	}
>  	_heapmin();
>  	return _ret;
> @@ -1604,7 +1608,7 @@
>  		return;
>  
>  	if (!FindClose(dir->find_file_handle))
> -		stream_printf(GDKerr, "rewinddir(): FindClose() failed\n");
> +		stream_printf(GDKerr, "#rewinddir(): FindClose() failed\n");
>  
>  	mask = malloc(strlen(dir->dir_name) + 3);
>  	sprintf(mask, "%s\\*", dir->dir_name);
> @@ -2116,7 +2120,7 @@
>  	p = (void *) VirtualAlloc(a, size, MEM_COMMIT | mode, PAGE_READWRITE);
>  	(void) pthread_mutex_unlock(&MT_mmap_lock);
>  	if (p == NULL) {
> -		stream_printf(GDKerr, "VirtualAlloc(" PTRFMT "," SZFMT ",MEM_COMMIT,PAGE_READWRITE): failed\n", PTRFMTCAST a, size);
> +		stream_printf(GDKerr, "#VirtualAlloc(" PTRFMT "," SZFMT ",MEM_COMMIT,PAGE_READWRITE): failed\n", PTRFMTCAST a, size);
>  	}
>  	return p;
>  }
> @@ -2126,9 +2130,9 @@
>  MT_vmfree(void *p, size_t size)
>  {
>  	if (VirtualFree(p, size, MEM_DECOMMIT) == 0)
> -		stream_printf(GDKerr, "VirtualFree(" PTRFMT "," SZFMT ",MEM_DECOMMIT): failed\n", PTRFMTCAST p, size);
> +		stream_printf(GDKerr, "#VirtualFree(" PTRFMT "," SZFMT ",MEM_DECOMMIT): failed\n", PTRFMTCAST p, size);
>  	if (VirtualFree(p, 0, MEM_RELEASE) == 0)
> -		stream_printf(GDKerr, "VirtualFree(" PTRFMT ",0,MEM_RELEASE): failed\n", PTRFMTCAST p);
> +		stream_printf(GDKerr, "#VirtualFree(" PTRFMT ",0,MEM_RELEASE): failed\n", PTRFMTCAST p);
>  }
>  
>  void *
> @@ -2149,7 +2153,7 @@
>  		size_t ret = VirtualFree(p + newsize, oldsize - newsize, MEM_DECOMMIT);
>  
>  		if (ret == 0)
> -			stream_printf(GDKerr, "VirtualFree(" PTRFMT "," SSZFMT ",MEM_DECOMMIT): failed\n", PTRFMTCAST(p + newsize), (ssize_t) (oldsize - newsize));
> +			stream_printf(GDKerr, "#VirtualFree(" PTRFMT "," SSZFMT ",MEM_DECOMMIT): failed\n", PTRFMTCAST(p + newsize), (ssize_t) (oldsize - newsize));
>  	} else if (oldsize < newsize) {
>  		(void) pthread_mutex_lock(&MT_mmap_lock);
>  		a = (char *) VirtualAlloc(p, newsize, MEM_COMMIT, PAGE_READWRITE);
> 
> 
> -------------------------------------------------------------------------
> This SF.Net email is sponsored by the Moblin Your Move Developer's challenge
> Build the coolest Linux based applications with Moblin SDK & win great prizes
> Grand prize is a trip for two to an Open Source event anywhere in the world
> http://moblin-contest.org/redirect.php?banner_id=100&url=/
> _______________________________________________
> Monetdb-checkins mailing list
> Monetdb-checkins at lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/monetdb-checkins
> 
> 

-- 
| Dr. Stefan Manegold | mailto:Stefan.Manegold at cwi.nl |
| CWI,  P.O.Box 94079 | http://www.cwi.nl/~manegold/  |
| 1090 GB Amsterdam   | Tel.: +31 (20) 592-4212       |
| The Netherlands     | Fax : +31 (20) 592-4312       |




More information about the developers-list mailing list