[Monetdb-developers] MonetDB: default - Remainder of group optimizer additions.

Sjoerd Mullender sjoerd at acm.org
Mon Mar 12 20:23:12 CET 2012


The new opt_group optimizer defines a group.new function that is not
compatible with the existing group.new functions.
In group.mx we have the definitions
command new(b:bat[:any_1,:any_2], start:int, incr:int, grpsize:int)
			:bat[:any_1,:int]
command new(attr:bat[:any_1,:any_2] )
	(histo:bat[:any_1,:wrd], grp:bat[:any_1,:void])
command new(attr:bat[:any_1,:any_2] )
	(histo:bat[:any_1,:wrd], grp:bat[:any_1,:oid])
command new(attr:bat[:any_1,:any_2], N:int, rng:int)
	(histo:bat[:any_1,:wrd],grp:bat[:any_1,:oid])

If called with a single :bat[:oid,:oid] bat as argument (as
sql.zero_or_one does), it returns two bats with types :bat[:oid,:wrd]
and :bat[:oid,:oid]

The group.new in the new opt_group optimizer is defined as

pattern
group.new(l:bat[:oid,:any]...)(grp:bat[:oid,:any],ext:bat[:oid,:any])

so if this one is called with a single :bat[:oid,:oid] bat as argument,
it returns two bats with types :bat[:oid,:oid] and :bat[:oid,:oid].

Of course, we now get the one sql.zero_or_one does *not* expect (since
it is defined later).

This causes a bunch of tests to fail.

On 2012-03-12 12:52, Martin Kersten wrote:
> Changeset: 89489d21d9bb for MonetDB
> URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=89489d21d9bb
> Added Files:
> 	monetdb5/optimizer/opt_groups.mx
> Removed Files:
> 	monetdb5/optimizer/Tests/derivePath00.mal
> 	monetdb5/optimizer/Tests/groupderive.mal
> 	monetdb5/optimizer/Tests/groupderive.stable.err
> 	monetdb5/optimizer/Tests/groupderive.stable.out
> 	monetdb5/optimizer/opt_derivepath.mx
> Modified Files:
> 	monetdb5/optimizer/Tests/All
> 	sql/test/Tests/setoptimizer.stable.err
> 	sql/test/Tests/setoptimizer.stable.out
> Branch: default
> Log Message:
> 
> Remainder of group optimizer additions.
> 
> 
> diffs (truncated from 654 to 300 lines):
> 
> diff --git a/monetdb5/optimizer/Tests/All b/monetdb5/optimizer/Tests/All
> --- a/monetdb5/optimizer/Tests/All
> +++ b/monetdb5/optimizer/Tests/All
> @@ -74,7 +74,7 @@ mitosis00
>  
>  joinpath
>  leftjoinpath
> -groupderive
> +groups00
>  
>  ifthencst
>  CXerror1
> diff --git a/monetdb5/optimizer/Tests/derivePath00.mal b/monetdb5/optimizer/Tests/derivePath00.mal
> deleted file mode 100644
> --- a/monetdb5/optimizer/Tests/derivePath00.mal
> +++ /dev/null
> @@ -1,19 +0,0 @@
> --- select distinct(A) from T
> --- only needs the group extend table, which means we can use a kunique.
> -function tst();
> -b:= bat.new(:oid,:str);
> -
> -bat.append(b,"hello");
> -bat.append(b,"hello");
> -bat.append(b,"world");
> -br:= bat.reverse(b);
> -k:= algebra.kunique(br);
> -kr:= bat.reverse(k);
> -io.print(kr);
> -(e,g):= group.new(b);
> -io.print(e);
> -end tst;
> -
> -user.tst();
> -optimizer.derivePath("user","tst");
> -mdb.List("user","tst");
> diff --git a/monetdb5/optimizer/Tests/groupderive.mal b/monetdb5/optimizer/Tests/groupderive.mal
> deleted file mode 100644
> --- a/monetdb5/optimizer/Tests/groupderive.mal
> +++ /dev/null
> @@ -1,10 +0,0 @@
> -function tst();
> -	b:= bat.new(:oid,:int);
> -	c:= bat.new(:oid,:str);
> -	d:= bat.new(:oid,:flt);
> -    (ext1,grp1) := group.new(b);
> -    (ext2,grp2) := group.derive(ext1,grp1, c);
> -    (ext3,grp3) := group.derive(ext2,grp2, d);
> -end tst;
> -optimizer.derivePath("user","tst");
> -mdb.list("user","tst");
> diff --git a/monetdb5/optimizer/Tests/groupderive.stable.err b/monetdb5/optimizer/Tests/groupderive.stable.err
> deleted file mode 100644
> --- a/monetdb5/optimizer/Tests/groupderive.stable.err
> +++ /dev/null
> @@ -1,80 +0,0 @@
> -stderr of test 'groupderive` in directory 'optimizer` itself:
> -
> -
> -# 09:23:28 >  
> -# 09:23:28 >   mserver5 "--config=/ufs/mk/monet5//Linux/etc/monetdb5.conf" --debug=10 --set gdk_nr_threads=0 --set "monet_mod_path=/ufs/mk/monet5//Linux/lib/MonetDB5:/ufs/mk/monet5//Linux/lib/MonetDB5/lib:/ufs/mk/monet5//Linux/lib/MonetDB5/bin" --set "gdk_dbfarm=/ufs/mk/monet5//Linux/var/MonetDB5/dbfarm" --set "sql_logdir=/ufs/mk/monet5//Linux/var/MonetDB5/sql_logs"   --set mapi_open=true --set xrpc_open=true --set mapi_port=30087 --set xrpc_port=43368 --set xrpc_docroot=/ufs/mk/monet5/package/MonetDB5 --set monet_prompt= --trace  --dbname=mTests_src_optimizer  groupderive.mal
> -# 09:23:28 >  
> -
> -# builtin opt 	gdk_arch = 64bitx86_64-unknown-linux-gnu
> -# builtin opt 	gdk_version = 1.28.1
> -# builtin opt 	monet_pid = 10370
> -# builtin opt 	prefix = /ufs/mk/monet5//Linux
> -# builtin opt 	exec_prefix = ${prefix}
> -# builtin opt 	gdk_dbname = tst
> -# builtin opt 	gdk_dbfarm = ${prefix}/var/MonetDB
> -# builtin opt 	gdk_debug = 8
> -# builtin opt 	gdk_mem_bigsize = 262144
> -# builtin opt 	gdk_alloc_map = yes
> -# builtin opt 	gdk_mem_pagebits = 14
> -# builtin opt 	gdk_vmtrim = yes
> -# builtin opt 	monet_admin = adm
> -# builtin opt 	monet_prompt = >
> -# builtin opt 	monet_welcome = yes
> -# builtin opt 	monet_mod_path = ${exec_prefix}/lib/MonetDB:${exec_prefix}/lib/bin
> -# builtin opt 	monet_daemon = yes
> -# builtin opt 	host = localhost
> -# builtin opt 	mapi_port = 50000
> -# builtin opt 	mapi_noheaders = no
> -# builtin opt 	mapi_debug = 0
> -# builtin opt 	mapi_clients = 2
> -# builtin opt 	sql_debug = 0
> -# builtin opt 	sql_logdir = ${prefix}/var/MonetDB
> -# builtin opt 	xquery_logdir = ${prefix}/var/MonetDB
> -# builtin opt 	standoff_ns = http://monetdb.cwi.nl/standoff
> -# builtin opt 	standoff_start = start
> -# builtin opt 	standoff_end = end
> -# config opt 	prefix = /ufs/mk/monet5//Linux
> -# config opt 	config = ${prefix}/etc/monetdb5.conf
> -# config opt 	prefix = /ufs/mk/monet5//Linux
> -# config opt 	exec_prefix = ${prefix}
> -# config opt 	gdk_dbfarm = ${prefix}/var/MonetDB5/dbfarm
> -# config opt 	gdk_dbname = demo
> -# config opt 	gdk_alloc_map = no
> -# config opt 	gdk_embedded = no
> -# config opt 	gdk_debug = 0
> -# config opt 	monet_mod_path = ${exec_prefix}/lib/MonetDB5:${exec_prefix}/lib/MonetDB5/lib:${exec_prefix}/lib/MonetDB5/bin
> -# config opt 	monet_daemon = no
> -# config opt 	monet_welcome = yes
> -# config opt 	mero_msglog = ${prefix}/var/log/merovingian.log
> -# config opt 	mero_errlog = ${prefix}/var/log/merovingian.log
> -# config opt 	mero_timeinterval = 600
> -# config opt 	mero_pidfile = ${prefix}/var/run/merovingian.pid
> -# config opt 	mero_exittimeout = 7
> -# config opt 	mero_doproxy = yes
> -# config opt 	mero_discoveryttl = 600
> -# config opt 	mal_init = ${exec_prefix}/lib/MonetDB5/mal_init.mal
> -# config opt 	mal_listing = 2
> -# config opt 	checkpoint_dir = ${prefix}/var/MonetDB5/chkpnt
> -# config opt 	mapi_port = 50000
> -# config opt 	mapi_open = false
> -# config opt 	sql_logdir = ${prefix}/var/MonetDB5/sql_logs
> -# config opt 	sql_init = ${exec_prefix}/lib/MonetDB5/sql_init.sql
> -# cmdline opt 	config = /ufs/mk/monet5//Linux/etc/monetdb5.conf
> -# cmdline opt 	gdk_nr_threads = 0
> -# cmdline opt 	monet_mod_path = /ufs/mk/monet5//Linux/lib/MonetDB5:/ufs/mk/monet5//Linux/lib/MonetDB5/lib:/ufs/mk/monet5//Linux/lib/MonetDB5/bin
> -# cmdline opt 	gdk_dbfarm = /ufs/mk/monet5//Linux/var/MonetDB5/dbfarm
> -# cmdline opt 	sql_logdir = /ufs/mk/monet5//Linux/var/MonetDB5/sql_logs
> -# cmdline opt 	mapi_open = true
> -# cmdline opt 	xrpc_open = true
> -# cmdline opt 	mapi_port = 30087
> -# cmdline opt 	xrpc_port = 43368
> -# cmdline opt 	xrpc_docroot = /ufs/mk/monet5/package/MonetDB5
> -# cmdline opt 	monet_prompt = 
> -# cmdline opt 	gdk_dbname = mTests_src_optimizer
> -#warning: please don't forget to set your vault key!
> -#(see /ufs/mk/monet5//Linux/etc/monetdb5.conf)
> -
> -# 09:23:28 >  
> -# 09:23:28 >  Done.
> -# 09:23:28 >  
> -
> diff --git a/monetdb5/optimizer/Tests/groupderive.stable.out b/monetdb5/optimizer/Tests/groupderive.stable.out
> deleted file mode 100644
> --- a/monetdb5/optimizer/Tests/groupderive.stable.out
> +++ /dev/null
> @@ -1,38 +0,0 @@
> -stdout of test 'groupderive` in directory 'optimizer` itself:
> -
> -
> -# 09:23:28 >  
> -# 09:23:28 >   mserver5 "--config=/ufs/mk/monet5//Linux/etc/monetdb5.conf" --debug=10 --set gdk_nr_threads=0 --set "monet_mod_path=/ufs/mk/monet5//Linux/lib/MonetDB5:/ufs/mk/monet5//Linux/lib/MonetDB5/lib:/ufs/mk/monet5//Linux/lib/MonetDB5/bin" --set "gdk_dbfarm=/ufs/mk/monet5//Linux/var/MonetDB5/dbfarm" --set "sql_logdir=/ufs/mk/monet5//Linux/var/MonetDB5/sql_logs"   --set mapi_open=true --set xrpc_open=true --set mapi_port=30087 --set xrpc_port=43368 --set xrpc_docroot=/ufs/mk/monet5/package/MonetDB5 --set monet_prompt= --trace  --dbname=mTests_src_optimizer  groupderive.mal
> -# 09:23:28 >  
> -
> -# MonetDB server v5.10.1, based on kernel v1.28.1
> -# Serving database 'mTests_src_optimizer', using 4 threads
> -# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs dynamically linked
> -# Copyright (c) 1993-July 2008 CWI.
> -# Copyright (c) August 2008- MonetDB B.V., all rights reserved
> -# Visit http://monetdb.cwi.nl/ for further information
> -# Listening for connection requests on mapi:monetdb://eir.ins.cwi.nl:30087/
> -function user.tst():void;
> -    b := bat.new(:oid,:int);
> -    c := bat.new(:oid,:str);
> -    d := bat.new(:oid,:flt);
> -    (ext1,grp1) := group.new(b);
> -    (ext2,grp2) := group.derive(ext1,grp1,c);
> -    (ext3,grp3) := group.derive(ext2,grp2,d);
> -end tst;
> -function user.main():void;
> -    mdb.list("user","tst");
> -end main;
> -function user.tst():void;
> -    b := bat.new(:oid,:int);
> -    c := bat.new(:oid,:str);
> -    d := bat.new(:oid,:flt);
> -    (ext1,grp1) := group.derivePath(b);
> -    (ext2,grp2) := group.derivePath(b,c);
> -    (ext3,grp3) := group.derivePath(b,c,d);
> -end tst;
> -
> -# 09:23:28 >  
> -# 09:23:28 >  Done.
> -# 09:23:28 >  
> -
> diff --git a/monetdb5/optimizer/opt_derivepath.mx b/monetdb5/optimizer/opt_derivepath.mx
> deleted file mode 100644
> --- a/monetdb5/optimizer/opt_derivepath.mx
> +++ /dev/null
> @@ -1,239 +0,0 @@
> -@/
> -The contents of this file are subject to the MonetDB Public License
> -Version 1.1 (the "License"); you may not use this file except in
> -compliance with the License. You may obtain a copy of the License at
> -http://www.monetdb.org/Legal/MonetDBLicense
> -
> -Software distributed under the License is distributed on an "AS IS"
> -basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
> -License for the specific language governing rights and limitations
> -under the License.
> -
> -The Original Code is the MonetDB Database System.
> -
> -The Initial Developer of the Original Code is CWI.
> -Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
> -Copyright August 2008-2012 MonetDB B.V.
> -All Rights Reserved.
> -@
> -
> - at f opt_derivepath
> -
> - at c
> -/*
> - * @- Group derive paths
> - * The routine @sc{optimizer.derivepaths()}
> - * walks through the program looking for grouping operations
> - * and cascades them into a single multiple group paths.
> - * To illustrate, consider
> - * @verbatim
> - * 	(ext1,grp1) := group.new(b);
> - * 	(ext2,grp2) := group.derive(ext1,grp1, c);
> - * 	(ext3,grp3) := group.done(ext2,grp2, d);
> - * @end verbatim
> - * The result becomes.
> - * @verbatim
> - * 	(ext3,grp3) := group.derivepath(b,c,d);
> - * @end verbatim
> - *
> - * The implementation of this operator can freely re-order the BATs
> - * for reduced intermediate results or as a basis for parallel scanning
> - * all BATs involved to derive their group id.
> - *
> - * The collection can be extended to immediately aim for group counting
> - * or summation, avoiding the materialisation of the group id table.
> - */
> - at mal
> -pattern optimizer.derivePath():str
> -address OPTderivePath;
> -pattern optimizer.derivePath(mod:str, fcn:str):str
> -address OPTderivePath
> -comment "Join path constructor";
> -
> -pattern group.derivePath(l:bat[:oid,:any]...)(grp:bat[:oid,:any],ext:bat[:oid,:any])
> -address ALGderivePath
> -comment "Derive a group index.";
> -
> -pattern group.deriveCount(l:bat[:oid,:any]...):bat[:oid,:wrd]
> -address ALGderiveCount
> -comment "Derive a group count.";
> -
> -pattern group.deriveSum(s:bat[:oid,:int],l:bat[:oid,:any]...):bat[:oid,:int]
> -address ALGderiveSum
> -comment "Derive a group sum.";
> -pattern group.deriveSum(s:bat[:oid,:lng],l:bat[:oid,:any]...):bat[:oid,:lng]
> -address ALGderiveSum
> -comment "Derive a group sum.";
> -pattern group.deriveSum(s:bat[:oid,:flt],l:bat[:oid,:any]...):bat[:oid,:flt]
> -address ALGderiveSum
> -comment "Derive a group sum.";
> -pattern group.deriveSum(s:bat[:oid,:dbl],l:bat[:oid,:any]...):bat[:oid,:dbl]
> -address ALGderiveSum
> -comment "Derive a group sum.";
> - at h
> -#ifndef _OPT_DERIVEPATH_
> -#define _OPT_DERIVEPATH_
> -#include "opt_prelude.h"
> -#include "opt_support.h"
> -#include "mal_interpreter.h"
> -
> -opt_export str ALGderivePath(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
> -opt_export str ALGderiveCount(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
> -opt_export str ALGderiveSum(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
> -
> -#define OPTDEBUGderivePath  if ( optDebug & ((lng)1 <<DEBUG_OPT_DERIVEPATH) )
> -
> - at c
> -#include "monetdb_config.h"
> -#include "opt_derivepath.h"
> -#include "group.h"
> -
> -static int
> -OPTderivePathImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
> -{
> -	int i, actions=0;
> -	int *pc;
> -	InstrPtr q;
> -	InstrPtr *old;
> -	int limit,slimit;
> -    Lifespan span;
> -
> -
> -
> -	(void) cntxt;
> -	(void) stk;
> -    span= setLifespan(mb);
> -    if( span == NULL)
> -        return 0;
> -	if (varGetProp(mb, getArg(mb->stmt[0], 0), inlineProp) != NULL)
> -		return 0;
> -
> -	/* beware, new variables and instructions are introduced */
> -	pc= (int*) GDKzalloc(sizeof(int)* mb->vtop * 2); /* to find last assignment */
> -	if ( pc == NULL)
> -		return 0;
> -
> -	old= mb->stmt;
> _______________________________________________
> Checkin-list mailing list
> Checkin-list at monetdb.org
> http://mail.monetdb.org/mailman/listinfo/checkin-list


-- 
Sjoerd Mullender

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 372 bytes
Desc: OpenPGP digital signature
URL: <http://www.monetdb.org/pipermail/developers-list/attachments/20120312/4592e4a8/attachment.sig>


More information about the developers-list mailing list