Problems executing custom aggregation function on large datasets

Sebastian Dorok sebastian.dorok at st.ovgu.de
Wed Jul 3 14:48:22 CEST 2013


Hi all,

I'm developing a custom aggregation function. Everything works fine 
until I use it on large datasets (ca. 270 Mio. rows).

Here is what my function signature looks like:

str AGGSimpleGenotype(int *ret, int *bid, int *gid, int *eid, bit 
ignore_nils);

I assume that the BATs identified by bid and gid have the same length. 
That's why my function checks this circumstance. On my test dataset (ca. 
200 rows) everything works fine. When I use my production dataset my 
aggregation function returns an error complaining that the BATs have not 
the same length and stops aggregating (by design)! How can this happen?

The SQL query is:
select grpgenotype(base) from test.bases group by pos;
I attached the MAL plan generated for the large dataset. When I execute 
it step by step I get an error in line 39 saying:

mal>X_106 := sql.delta(X_56,X_70,X_71,X_8);
MAPI  = (monetdb) /tmp/.s.monetdb.50000
QUERY = X_106 := sql.delta(X_56,X_70,X_71,X_8);
ERROR = !TypeException:user.main[1]:'sql.delta' undefined in: X_106:any 
:= sql.delta(X_56:bat[:oid,:str], X_70:bat[:oid,:oid], 
X_71:bat[:oid,:str], X_8:bat[:oid,:any])

This error occurs before my function is entered but I think it must be 
related with my problem. Is it an internal MonetDB problem?

I'm using the Feb2013-SP2 release.

Any ideas? Thx.
-------------- next part --------------
function user.s1_3{autoCommit=true}():void;                                                   
     X_16:bat[:any,:str]  := nil:bat[:any,:str];                                               
 barrier X_179 := language.dataflow();                                                         
     X_2 := sql.mvc();                                                                         
     X_37:bat[:oid,:oid]  := sql.tid(X_2,"test","bases",0,7);                                  
     X_50:bat[:oid,:str]  := sql.bind(X_2,"test","bases","base",0,0,7);                        
     (X_57:bat[:oid,:oid] ,X_58:bat[:oid,:str] ) := sql.bind(X_2,"test","bases","base",2,0,7); 
     X_100 := sql.delta(X_50,X_57,X_58);                                                       
     X_108 := algebra.leftfetchjoin(X_37,X_100);                                               
     X_39:bat[:oid,:oid]  := sql.tid(X_2,"test","bases",1,7);                                  
     X_51:bat[:oid,:str]  := sql.bind(X_2,"test","bases","base",0,1,7);                        
     (X_59:bat[:oid,:oid] ,X_60:bat[:oid,:str] ) := sql.bind(X_2,"test","bases","base",2,1,7); 
     X_101 := sql.delta(X_51,X_59,X_60);                                                       
     X_110 := algebra.leftfetchjoin(X_39,X_101);                                               
     X_41:bat[:oid,:oid]  := sql.tid(X_2,"test","bases",2,7);                                  
     X_52:bat[:oid,:str]  := sql.bind(X_2,"test","bases","base",0,2,7);                        
     (X_61:bat[:oid,:oid] ,X_62:bat[:oid,:str] ) := sql.bind(X_2,"test","bases","base",2,2,7); 
     X_102 := sql.delta(X_52,X_61,X_62);                                                       
     X_112 := algebra.leftfetchjoin(X_41,X_102);                                               
     X_43:bat[:oid,:oid]  := sql.tid(X_2,"test","bases",3,7);                                  
     X_53:bat[:oid,:str]  := sql.bind(X_2,"test","bases","base",0,3,7);                        
     (X_63:bat[:oid,:oid] ,X_64:bat[:oid,:str] ) := sql.bind(X_2,"test","bases","base",2,3,7); 
     X_103 := sql.delta(X_53,X_63,X_64);                                                       
     X_114 := algebra.leftfetchjoin(X_43,X_103);                                               
     X_45:bat[:oid,:oid]  := sql.tid(X_2,"test","bases",4,7);                                  
     X_54:bat[:oid,:str]  := sql.bind(X_2,"test","bases","base",0,4,7);                        
     (X_65:bat[:oid,:oid] ,X_66:bat[:oid,:str] ) := sql.bind(X_2,"test","bases","base",2,4,7); 
     X_104 := sql.delta(X_54,X_65,X_66);                                                       
     X_116 := algebra.leftfetchjoin(X_45,X_104);                                               
     X_47:bat[:oid,:oid]  := sql.tid(X_2,"test","bases",5,7);                                  
     X_55:bat[:oid,:str]  := sql.bind(X_2,"test","bases","base",0,5,7);                        
     (X_67:bat[:oid,:oid] ,X_68:bat[:oid,:str] ) := sql.bind(X_2,"test","bases","base",2,5,7); 
     X_105 := sql.delta(X_55,X_67,X_68);                                                       
     X_118 := algebra.leftfetchjoin(X_47,X_105);                                               
     X_49:bat[:oid,:oid]  := sql.tid(X_2,"test","bases",6,7);                                  
     X_56:bat[:oid,:str]  := sql.bind(X_2,"test","bases","base",0,6,7);                        
     (X_70:bat[:oid,:oid] ,X_71:bat[:oid,:str] ) := sql.bind(X_2,"test","bases","base",2,6,7); 
     X_8 := sql.bind(X_2,"test","bases","base",1);                                             
     X_106 := sql.delta(X_56,X_70,X_71,X_8);                                                  
     X_120 := algebra.leftfetchjoin(X_49,X_106);                                               
     X_164 := mat.packIncrement(X_108,7);                                                      
     X_166 := mat.packIncrement(X_164,X_110);                                                  
     X_167 := mat.packIncrement(X_166,X_112);                                                  
     X_168 := mat.packIncrement(X_167,X_114);                                                  
     X_169 := mat.packIncrement(X_168,X_116);                                                  
     X_170 := mat.packIncrement(X_169,X_118);                                                  
     X_10 := mat.packIncrement(X_170,X_120);                                                   
     X_73:bat[:oid,:int]  := sql.bind(X_2,"test","bases","pos",0,0,7);                         
     (X_86:bat[:oid,:oid] ,X_87:bat[:oid,:int] ) := sql.bind(X_2,"test","bases","pos",2,0,7);  
     X_121 := sql.delta(X_73,X_86,X_87);                                                       
     X_128 := algebra.leftfetchjoin(X_37,X_121);                                               
     (X_135,X_136,X_137) := group.subgroupdone(X_128);                                         
     X_138 := algebra.leftfetchjoin(X_136,X_128);                                              
     X_75:bat[:oid,:int]  := sql.bind(X_2,"test","bases","pos",0,1,7);                         
     (X_88:bat[:oid,:oid] ,X_89:bat[:oid,:int] ) := sql.bind(X_2,"test","bases","pos",2,1,7);  
     X_122 := sql.delta(X_75,X_88,X_89);                                                       
     X_129 := algebra.leftfetchjoin(X_39,X_122);                                               
     (X_139,X_140,X_141) := group.subgroupdone(X_129);                                         
     X_142 := algebra.leftfetchjoin(X_140,X_129);                                              
     X_77:bat[:oid,:int]  := sql.bind(X_2,"test","bases","pos",0,2,7);                         
     (X_90:bat[:oid,:oid] ,X_91:bat[:oid,:int] ) := sql.bind(X_2,"test","bases","pos",2,2,7);  
     X_123 := sql.delta(X_77,X_90,X_91);                                                       
     X_130 := algebra.leftfetchjoin(X_41,X_123);                                               
     (X_143,X_144,X_145) := group.subgroupdone(X_130);                                         
     X_146 := algebra.leftfetchjoin(X_144,X_130);                                              
     X_79:bat[:oid,:int]  := sql.bind(X_2,"test","bases","pos",0,3,7);                         
     (X_92:bat[:oid,:oid] ,X_93:bat[:oid,:int] ) := sql.bind(X_2,"test","bases","pos",2,3,7);  
     X_124 := sql.delta(X_79,X_92,X_93);                                                       
     X_131 := algebra.leftfetchjoin(X_43,X_124);                                               
     (X_147,X_148,X_149) := group.subgroupdone(X_131);                                         
     X_150 := algebra.leftfetchjoin(X_148,X_131);                                              
     X_81:bat[:oid,:int]  := sql.bind(X_2,"test","bases","pos",0,4,7);                         
     (X_94:bat[:oid,:oid] ,X_95:bat[:oid,:int] ) := sql.bind(X_2,"test","bases","pos",2,4,7);  
     X_125 := sql.delta(X_81,X_94,X_95);                                                       
     X_132 := algebra.leftfetchjoin(X_45,X_125);                                               
     (X_151,X_152,X_153) := group.subgroupdone(X_132);                                         
     X_154 := algebra.leftfetchjoin(X_152,X_132);                                              
     X_83:bat[:oid,:int]  := sql.bind(X_2,"test","bases","pos",0,5,7);                         
     (X_96:bat[:oid,:oid] ,X_97:bat[:oid,:int] ) := sql.bind(X_2,"test","bases","pos",2,5,7);  
     X_126 := sql.delta(X_83,X_96,X_97);                                                       
     X_133 := algebra.leftfetchjoin(X_47,X_126);                                               
     (X_155,X_156,X_157) := group.subgroupdone(X_133);                                         
     X_158 := algebra.leftfetchjoin(X_156,X_133);                                              
     X_85:bat[:oid,:int]  := sql.bind(X_2,"test","bases","pos",0,6,7);                         
     (X_98:bat[:oid,:oid] ,X_99:bat[:oid,:int] ) := sql.bind(X_2,"test","bases","pos",2,6,7);  
     X_12 := sql.bind(X_2,"test","bases","pos",1);                                             
     X_127 := sql.delta(X_85,X_98,X_99,X_12);                                                  
     X_134 := algebra.leftfetchjoin(X_49,X_127);                                               
     (X_159,X_160,X_161) := group.subgroupdone(X_134);                                         
     X_162 := algebra.leftfetchjoin(X_160,X_134);                                              
     X_172 := mat.packIncrement(X_138,7);                                                      
     X_173 := mat.packIncrement(X_172,X_142);                                                  
     X_174 := mat.packIncrement(X_173,X_146);                                                  
     X_175 := mat.packIncrement(X_174,X_150);                                                  
     X_176 := mat.packIncrement(X_175,X_154);                                                  
     X_177 := mat.packIncrement(X_176,X_158);                                                  
     X_13 := mat.packIncrement(X_177,X_162);                                                   
     (X_14,r1_27,X_163) := group.subgroupdone(X_13);                                           
     X_16:bat[:any,:str]  := udf.subgenotype(X_10,X_14,r1_27,true);                            
     language.pass(X_37);                                                                      
     language.pass(X_128);                                                                     
     language.pass(X_39);                                                                      
     language.pass(X_129);                                                                     
     language.pass(X_41);                                                                      
     language.pass(X_130);                                                                     
     language.pass(X_43);                                                                      
     language.pass(X_131);                                                                     
     language.pass(X_45);                                                                      
     language.pass(X_132);                                                                     
     language.pass(X_47);                                                                      
     language.pass(X_133);                                                                     
     language.pass(X_49);                                                                      
     language.pass(X_134);                                                                     
 exit X_179;                                                                                   
     X_171:bat[:oid,:str]  := nil:bat[:oid,:str];                                              
     X_178:bat[:oid,:int]  := nil:bat[:oid,:int];                                              
     X_18 := sql.resultSet(1,1,X_16);                                                          
     sql.rsColumn(X_18,"test.bases","L1","clob",1,0,X_16);                                     
     X_22 := io.stdout();                                                                      
     sql.exportResult(X_22,X_18);                                                              
 end s1_3;                                                                                     


More information about the users-list mailing list