[Monetdb-developers] MonetDB: default - Cleanup of logger code to split head and tail...

Niels Nes Niels.Nes at cwi.nl
Wed Jul 18 12:57:49 CEST 2012


On Wed, Jul 18, 2012 at 12:55:31PM +0200, Niels Nes wrote:
> Changeset: 982fdf69e13b for MonetDB
> URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=982fdf69e13b
> Modified Files:
> 	gdk/gdk_logger.c
> 	gdk/gdk_logger.h
> 	sql/server/rel_optimizer.c
> 	sql/storage/bat/bat_logger.c
> 	sql/storage/restrict/restrict_logger.c
> Branch: default
> Log Message:
> 
> Cleanup of logger code to split head and tail bat into 2 void headed bats

This checkin breaks the current on disk format, ie if your using the
defaul branch a reload is needed.
Before the (september) release we need to add updating functions
for atleast SQL.

Niels 
> 
> 
> diffs (truncated from 723 to 300 lines):
> 
> diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
> --- a/gdk/gdk_logger.c
> +++ b/gdk/gdk_logger.c
> @@ -73,7 +73,10 @@
>  #include "gdk_logger.h"
>  #include <string.h>
>  
> -
> +static BUN BUNfndT( BAT *b, ptr v)
> +{
> +	return BUNfnd(BATmirror(b), v);
> +}
>  /*
>   * @-
>   * The log record encoding is geared at reduced storage space, but
> @@ -218,12 +221,11 @@ la_bat_clear(logger *lg, logaction *la)
>  	BAT *b;
>  
>  	/* do we need to skip these old updates */
> -	if (BATcount(lg->snapshots)) {
> -		BUN p = BUNfnd(lg->snapshots, &bid);
> +	if (BATcount(lg->snapshots_bid)) {
> +		BUN p = BUNfndT(lg->snapshots_bid, &bid);
>  
>  		if (p != BUN_NONE) {
> -			BATiter i = bat_iterator(lg->snapshots);
> -			int tid = *(int *) BUNtloc(i, p);
> +			int tid = *(int *) Tloc(lg->snapshots_tid, p);
>  
>  			if (lg->tid <= tid)
>  				return;
> @@ -244,16 +246,18 @@ static int
>  log_read_seq(logger *lg, logformat *l)
>  {
>  	int seq = l->nr;
> -	lng id;
> +	lng val;
> +	BUN p;
>  
> -	if (!mnstr_readLng(lg->log, &id))
> +	if (!mnstr_readLng(lg->log, &val))
>  		 return LOG_ERR;
>  
> -	if (BUNfnd(lg->seqs, &seq) != BUN_NONE) {
> -		BUNdelHead(lg->seqs, &seq, FALSE);
> +	if ((p = BUNfndT(lg->seqs_id, &seq)) != BUN_NONE) {
> +		BUNdelete(lg->seqs_id, p, FALSE);
> +		BUNdelete(lg->seqs_val, p, FALSE);
>  	}
> -	BUNins(lg->seqs, &seq, &id, FALSE);
> -
> +	BUNappend(lg->seqs_id, &seq, FALSE);
> +	BUNappend(lg->seqs_val, &val, FALSE);
>  	return LOG_OK;
>  }
>  
> @@ -404,12 +408,11 @@ la_bat_updates(logger *lg, logaction *la
>  		return;		/* ignore bats no longer in the catalog */
>  
>  	/* do we need to skip these old updates */
> -	if (BATcount(lg->snapshots)) {
> -		BUN p = BUNfnd(lg->snapshots, &bid);
> +	if (BATcount(lg->snapshots_bid)) {
> +		BUN p = BUNfndT(lg->snapshots_bid, &bid);
>  
>  		if (p != BUN_NONE) {
> -			BATiter i = bat_iterator(lg->snapshots);
> -			int tid = *(int *) BUNtloc(i, p);
> +			int tid = *(int *) Tloc(lg->snapshots_tid, p);
>  
>  			if (lg->tid <= tid)
>  				return;
> @@ -477,10 +480,14 @@ la_bat_destroy(logger *lg, logaction *la
>  	log_bid bid = logger_find_bat(lg, la->name);
>  
>  	if (bid) {
> +		BUN p;
> +
>  		logger_del_bat(lg, bid);
> -		if (BUNfnd(lg->snapshots, &bid) != BUN_NONE) {
> -			BUNdelHead(lg->snapshots, &bid, FALSE);
> -			BUNins(lg->snapshots, &bid, &lg->tid, FALSE);
> +		if ((p = BUNfndT(lg->snapshots_bid, &bid)) != BUN_NONE) {
> +			BUNdelete(lg->snapshots_bid, p, FALSE);
> +			BUNdelete(lg->snapshots_tid, p, FALSE);
> +			BUNappend(lg->snapshots_bid, &bid, FALSE);
> +			BUNappend(lg->snapshots_tid, &lg->tid, FALSE);
>  		}
>  	}
>  }
> @@ -564,15 +571,19 @@ la_bat_use(logger *lg, logaction *la)
>  {
>  	log_bid bid = la->nr;
>  	BAT *b = BATdescriptor(bid);
> +	BUN p;
>  
>  	if (!b) {
>  		GDKerror("logger: could not use bat (%d) for %s\n", (int) bid, la->name);
>  		return;
>  	}
>  	logger_add_bat(lg, b, la->name);
> -	if (BUNfnd(lg->snapshots, &b->batCacheid) != BUN_NONE)
> -		BUNdelHead(lg->snapshots, &b->batCacheid, FALSE);
> -	BUNins(lg->snapshots, &b->batCacheid, &lg->tid, FALSE);
> +	if ((p = BUNfndT(lg->snapshots_bid, &b->batCacheid)) != BUN_NONE) {
> +		BUNdelete(lg->snapshots_bid, p, FALSE);
> +		BUNdelete(lg->snapshots_tid, p, FALSE);
> +	}
> +	BUNappend(lg->snapshots_bid, &b->batCacheid, FALSE);
> +	BUNappend(lg->snapshots_tid, &lg->tid, FALSE);
>  	logbat_destroy(b);
>  }
>  
> @@ -875,17 +886,23 @@ static int
>  logger_commit(logger *lg)
>  {
>  	int id = LOG_SID;
> +	BUN p;
>  
>  	if (lg->debug & 1)
>  		fprintf(stderr, "logger_commit\n");
>  
> -	BUNdelHead(lg->seqs, &id, FALSE);
> -	BUNins(lg->seqs, &id, &lg->id, FALSE);
> +	p = BUNfndT(lg->seqs_id, &id);
> +	BUNdelete(lg->seqs_id, p, FALSE);
> +	BUNdelete(lg->seqs_val, p, FALSE);
> +	BUNappend(lg->seqs_id, &id, FALSE);
> +	BUNappend(lg->seqs_val, &lg->id, FALSE);
>  
>  	/* cleanup old snapshots */
> -	if (BATcount(lg->snapshots)) {
> -		BATclear(lg->snapshots, FALSE);
> -		BATcommit(lg->snapshots);
> +	if (BATcount(lg->snapshots_bid)) {
> +		BATclear(lg->snapshots_bid, FALSE);
> +		BATclear(lg->snapshots_tid, FALSE);
> +		BATcommit(lg->snapshots_bid);
> +		BATcommit(lg->snapshots_tid);
>  	}
>  	return bm_commit(lg);
>  }
> @@ -919,34 +936,34 @@ check_version(logger *lg, FILE *fp)
>  }
>  
>  static int
> -bm_subcommit(BAT *list, BAT *catalog, BAT *extra, int debug)
> +bm_subcommit(BAT *list_bid, BAT *list_nme, BAT *catalog_bid, BAT *catalog_nme, BAT *extra, int debug)
>  {
>  	BUN p, q;
> -	BUN nn = 2 + (list->batFirst > list->batDeleted ? list->batFirst - list->batDeleted : 0) + BATcount(list) + (extra ? BATcount(extra) : 0);
> +	BUN nn = 3 + (list_bid->batFirst > list_bid->batDeleted ? list_bid->batFirst - list_bid->batDeleted : 0) + BATcount(list_bid) + (extra ? BATcount(extra) : 0);
>  	bat *n = (bat*)GDKmalloc(sizeof(bat) * nn);
>  	int i = 0;
> -	BATiter iter = bat_iterator(list);
> +	BATiter iter = (list_nme)?bat_iterator(list_nme):bat_iterator(list_bid);
>  	int res;
>  
>  	n[i++] = 0;		/* n[0] is not used */
>  
>  	/* first loop over deleted then over current and new */
> -	for (p = list->batDeleted; p < list->batFirst; p++) {
> -		bat col = *(log_bid *) BUNhead(iter, p);
> +	for (p = list_bid->batDeleted; p < list_bid->batFirst; p++) {
> +		bat col = *(log_bid *) Tloc(list_bid, p);
>  
>  		if (debug & 1)
>  			fprintf(stderr, "commit deleted %s (%d) %s\n",
>  				BBPname(col), col,
> -				(list == catalog) ? BUNtail(iter, p) : "snapshot");
> +				(list_bid == catalog_bid) ? BUNtail(iter, p) : "snapshot");
>  		n[i++] = ABS(col);
>  	}
> -	BATloop(list, p, q) {
> -		bat col = *(log_bid *) BUNhead(iter, p);
> +	BATloop(list_bid, p, q) {
> +		bat col = *(log_bid *) Tloc(list_bid, p);
>  
>  		if (debug & 1)
>  			fprintf(stderr, "commit new %s (%d) %s\n",
>  				BBPname(col), col,
> -				(list == catalog) ? BUNtail(iter, p) : "snapshot");
> +				(list_bid == catalog_bid) ? BUNtail(iter, p) : "snapshot");
>  		n[i++] = ABS(col);
>  	}
>  	if (extra) {
> @@ -957,14 +974,16 @@ bm_subcommit(BAT *list, BAT *catalog, BA
>  			if (debug & 1)
>  				fprintf(stderr, "commit extra %s %s\n",
>  					name,
> -					(list == catalog) ? BUNtail(iter, p) : "snapshot");
> +					(list_bid == catalog_bid) ? BUNtail(iter, p) : "snapshot");
>  			n[i++] = ABS(BBPindex(name));
>  		}
>  	}
>  	/* now commit catalog, so it's also up to date on disk */
> -	n[i++] = ABS(catalog->batCacheid);
> +	n[i++] = ABS(catalog_bid->batCacheid);
> +	n[i++] = ABS(catalog_nme->batCacheid);
>  	assert((BUN) i <= nn);
> -	BATcommit(catalog);
> +	BATcommit(catalog_bid);
> +	BATcommit(catalog_nme);
>  	res = TMsubcommit_list(n, i);
>  	GDKfree(n);
>  	return res;
> @@ -992,8 +1011,8 @@ logger_new(int debug, char *fn, char *lo
>  	FILE *fp;
>  	char filename[BUFSIZ];
>  	char bak[BUFSIZ];
> -	log_bid seqs = 0;
> -	bat catalog;
> +	log_bid seqs_id = 0;
> +	bat catalog_bid, catalog_nme;
>  
>  	if (lg == NULL)
>  		return NULL;
> @@ -1022,9 +1041,12 @@ logger_new(int debug, char *fn, char *lo
>  	lg->postfuncp = postfuncp;
>  	lg->log = NULL;
>  	lg->end = 0;
> -	lg->catalog = NULL;
> -	lg->snapshots = NULL;
> -	lg->seqs = NULL;
> +	lg->catalog_bid = NULL;
> +	lg->catalog_nme = NULL;
> +	lg->snapshots_bid = NULL;
> +	lg->snapshots_tid = NULL;
> +	lg->seqs_id = NULL;
> +	lg->seqs_val = NULL;
>  
>  	snprintf(filename, BUFSIZ, "%s%s", lg->dir, LOGFILE);
>  	snprintf(bak, BUFSIZ, "%s.bak", filename);
> @@ -1040,10 +1062,10 @@ logger_new(int debug, char *fn, char *lo
>  	}
>  	fp = fopen(filename, "r");
>  
> -	snprintf(bak, BUFSIZ, "%s_catalog", fn);
> -	catalog = BBPindex(bak);
> +	snprintf(bak, BUFSIZ, "%s_catalog_bid", fn);
> +	catalog_bid = BBPindex(bak);
>  
> -	if (catalog == 0) {
> +	if (catalog_bid == 0) {
>  		log_bid bid = 0;
>  
>  		/* catalog does not exist, so the log file also shouldn't exist */
> @@ -1055,16 +1077,24 @@ logger_new(int debug, char *fn, char *lo
>  			goto error;
>  		}
>  
> -		lg->catalog = logbat_new(TYPE_int, TYPE_str, BATSIZE);
> +		lg->catalog_bid = logbat_new(TYPE_void, TYPE_int, BATSIZE);
> +		lg->catalog_nme = logbat_new(TYPE_void, TYPE_str, BATSIZE);
>  		if (debug)
>  			fprintf(stderr, "create %s catalog\n", fn);
>  
> -		bid = lg->catalog->batCacheid;
> +		/* Make persistent */
> +		bid = lg->catalog_bid->batCacheid;
> +		BBPincref(bid, TRUE);
> +		BATmode(lg->catalog_bid, PERSISTENT);
> +		snprintf(bak, BUFSIZ, "%s_catalog_bid", fn);
> +		BBPrename(lg->catalog_bid->batCacheid, bak);
>  
>  		/* Make persistent */
> +		bid = lg->catalog_nme->batCacheid;
>  		BBPincref(bid, TRUE);
> -		BATmode(lg->catalog, PERSISTENT);
> -		BBPrename(lg->catalog->batCacheid, bak);
> +		BATmode(lg->catalog_nme, PERSISTENT);
> +		snprintf(bak, BUFSIZ, "%s_catalog_nme", fn);
> +		BBPrename(lg->catalog_nme->batCacheid, bak);
>  
>  		if (!GDKcreatedir(filename)) {
>  			logger_fatal("logger_new: cannot create directory for log file %s\n",
> @@ -1082,7 +1112,7 @@ logger_new(int debug, char *fn, char *lo
>  		fclose(fp);
>  		fp = NULL;
>  
> -		if (bm_subcommit(lg->catalog, lg->catalog, NULL, lg->debug) != 0) {
> +		if (bm_subcommit(lg->catalog_bid, lg->catalog_nme, lg->catalog_bid, lg->catalog_nme, NULL, lg->debug) != 0) {
>  			/* cannot commit catalog, so remove log */
>  			unlink(filename);
>  			goto error;
> @@ -1092,8 +1122,11 @@ logger_new(int debug, char *fn, char *lo
>  		   require a logical reference we also add a logical
>  		   reference for the persistent bats */
>  		BUN p, q;
> -		BAT *b = BATdescriptor(catalog);
> -		BATiter bi = bat_iterator(b);
> +		BAT *b = BATdescriptor(catalog_bid), *n;
> +
> +		snprintf(bak, BUFSIZ, "%s_catalog_nme", fn);
> +		catalog_nme = BBPindex(bak);
> +		n = BATdescriptor(catalog_nme);
>  
>  		/* the catalog exists, and so should the log file */
>  		if (fp == NULL) {
> @@ -1107,42 +1140,63 @@ logger_new(int debug, char *fn, char *lo
> _______________________________________________
> Checkin-list mailing list
> Checkin-list at monetdb.org
> http://mail.monetdb.org/mailman/listinfo/checkin-list

-- 
Niels Nes, Centrum Wiskunde & Informatica (CWI)
Science Park 123, 1098 XG Amsterdam, The Netherlands
room L3.14,  phone ++31 20 592-4098 	sip:4098 at sip.cwi.nl
url: http://www.cwi.nl/~niels   e-mail: Niels.Nes at cwi.nl
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 198 bytes
Desc: not available
URL: <http://www.monetdb.org/pipermail/developers-list/attachments/20120718/d51281af/attachment.sig>


More information about the developers-list mailing list