MonetDB: Protobuf - Protocol buffer prototype for the record

Stefan Manegold Stefan.Manegold at cwi.nl
Tue Aug 26 09:15:48 CEST 2014


Thanks, Hannes!

Just for the records,
new (and required) files
pbstream.c pbstream.h mapi_pb-c.c mapi_pb-c.h
are missing.

Stefan

----- Original Message -----
> Changeset: af2757009ce3 for MonetDB
> URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=af2757009ce3
> Modified Files:
> 	clients/mapiclient/Makefile.ag
> 	clients/mapiclient/mclient.c
> 	clients/mapilib/Makefile.ag
> 	clients/mapilib/mapi.c
> 	clients/mapilib/mapi.h
> 	common/stream/Makefile.ag
> 	common/stream/stream.c
> 	common/utils/mcrypt.c
> 	common/utils/mcrypt.h
> 	configure.ag
> 	monetdb5/extras/jaql/Makefile.ag
> 	monetdb5/extras/mal_optimizer_template/Makefile.ag
> 	monetdb5/mal/Makefile.ag
> 	monetdb5/mal/mal_authorize.c
> 	monetdb5/mal/mal_client.c
> 	monetdb5/mal/mal_client.h
> 	monetdb5/mal/mal_debugger.c
> 	monetdb5/mal/mal_readline.c
> 	monetdb5/mal/mal_scenario.c
> 	monetdb5/mal/mal_session.c
> 	monetdb5/mal/mal_session.h
> 	monetdb5/modules/atoms/Makefile.ag
> 	monetdb5/modules/kernel/Makefile.ag
> 	monetdb5/modules/mal/Makefile.ag
> 	monetdb5/modules/mal/mal_mapi.c
> 	monetdb5/modules/mal/mal_mapi.h
> 	monetdb5/optimizer/Makefile.ag
> 	monetdb5/scheduler/Makefile.ag
> 	sql/backends/monet5/LSST/Makefile.ag
> 	sql/backends/monet5/Makefile.ag
> 	sql/backends/monet5/UDF/Makefile.ag
> 	sql/backends/monet5/sql_result.c
> 	sql/backends/monet5/sql_scenario.c
> 	sql/backends/monet5/sql_scenario.h
> 	sql/backends/monet5/vaults/Makefile.ag
> 	sql/storage/bat/Makefile.ag
> 	tools/merovingian/daemon/client.c
> 	tools/mserver/Makefile.ag
> 	tools/mserver/mserver5.c
> Branch: Protobuf
> Log Message:
> 
> Protocol buffer prototype for the record
> 
> 
> diffs (truncated from 2665 to 300 lines):
> 
> diff --git a/clients/mapiclient/Makefile.ag b/clients/mapiclient/Makefile.ag
> --- a/clients/mapiclient/Makefile.ag
> +++ b/clients/mapiclient/Makefile.ag
> @@ -16,7 +16,7 @@
>  # All Rights Reserved.
>  
>  MTSAFE
> -INCLUDES = ../mapilib ../../common/options ../../common/stream
> $(READLINE_INCS)
> +INCLUDES = ../mapilib ../../common/options ../../common/stream
> $(READLINE_INCS)  $(PROTOBUF_C_CFLAGS)
>  
>  lib_mcutil = {
>  	NOINST
> @@ -35,8 +35,8 @@ bin_msqldump = {
>  	SOURCES = msqldump.c
>  	LIBS = libmcutil ../mapilib/libmapi \
>  		../../common/stream/libstream \
> -		$(curl_LIBS)
> -}
> +		$(curl_LIBS)
> +	}
>  
>  bin_stethoscope = {
>  	SOURCES = stethoscope.c
> diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
> --- a/clients/mapiclient/mclient.c
> +++ b/clients/mapiclient/mclient.c
> @@ -347,13 +347,9 @@ fetch_line(MapiHdl hdl)
>  static int
>  fetch_row(MapiHdl hdl)
>  {
> -	char *reply;
>  
> -	do {
> -		if ((reply = fetch_line(hdl)) == NULL)
> -			return 0;
> -	} while (*reply != '[' && *reply != '=');
> -	return mapi_split_line(hdl);
> +	// TODO: this is almost a NOP, because we already have everything from the
> PB. fetch_line() just has to pay attention to the export size in the future.
> +	return mapi_fetch_row(hdl);
>  }
>  
>  static void
> @@ -668,11 +664,14 @@ CSVrenderer(MapiHdl hdl)
>  			s = mapi_get_name(hdl, i);
>  			if (s == NULL)
>  				s = "";
> -			mnstr_printf(toConsole, "%s%s",
> +			printf( "%s%s",
>  				     i == 0 ? "" : sep, s ? s : "");
>  		}
> -		mnstr_printf(toConsole, "\n");
> +		printf( "\n");
>  	}
> +
> +	// TODO: switch on type!
> +
>  	while (!mnstr_errnr(toConsole) && (fields = fetch_row(hdl)) != 0) {
>  		for (i = 0; i < fields; i++) {
>  			s = mapi_fetch_field(hdl, i);
> @@ -681,37 +680,38 @@ CSVrenderer(MapiHdl hdl)
>  			if (strchr(s, *sep) != NULL ||
>  			    strchr(s, '\n') != NULL ||
>  			    strchr(s, '"') != NULL) {
> -				mnstr_printf(toConsole, "%s\"",
> +				printf( "%s\"",
>  					      i == 0 ? "" : sep);
> +				printf("%s",s);
>  				while (*s) {
>  					switch (*s) {
>  					case '\n':
> -						mnstr_write(toConsole, "\\n", 1, 2);
> +						printf( "\\n");
>  						break;
>  					case '\t':
> -						mnstr_write(toConsole, "\\t", 1, 2);
> +						printf( "\\t");
>  						break;
>  					case '\r':
> -						mnstr_write(toConsole, "\\r", 1, 2);
> +						printf( "\\r");
>  						break;
>  					case '\\':
> -						mnstr_write(toConsole, "\\\\", 1, 2);
> +						printf( "\\\\");
>  						break;
>  					case '"':
> -						mnstr_write(toConsole, "\"\"", 1, 2);
> +						printf( "\"\"");
>  						break;
>  					default:
> -						mnstr_write(toConsole, s, 1, 1);
> +
>  						break;
>  					}
>  					s++;
>  				}
> -				mnstr_write(toConsole, "\"", 1, 1);
> +				printf("\"");
>  			} else
> -				mnstr_printf(toConsole, "%s%s",
> +				printf( "%s%s",
>  					      i == 0 ? "" : sep, s ? s : "");
>  		}
> -		mnstr_printf(toConsole, "\n");
> +		printf( "\n");
>  	}
>  }
>  
> diff --git a/clients/mapilib/Makefile.ag b/clients/mapilib/Makefile.ag
> --- a/clients/mapilib/Makefile.ag
> +++ b/clients/mapilib/Makefile.ag
> @@ -18,11 +18,11 @@
>  MTSAFE
>  
>  INCLUDES = ../../common/options ../../common/stream ../../common/utils \
> -		   $(MSGCONTROL_FLAGS)
> +		 $(MSGCONTROL_FLAGS) $(PROTOBUF_C_CFLAGS)
>  
>  lib_mapi = {
>  	VERSION = $(MAPI_VERSION)
> -	SOURCES = mapi.c mapi.rc
> +	SOURCES = mapi.c mapi.rc
>  	LIBS = $(SOCKET_LIBS) ../../common/stream/libstream \
>  		../../common/options/libmoptions \
>  		../../common/utils/libmcrypt $(openssl_LIBS)
> diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
> --- a/clients/mapilib/mapi.c
> +++ b/clients/mapilib/mapi.c
> @@ -772,10 +772,13 @@
>  
>  #include "monetdb_config.h"
>  #include <stream.h>		/* include before mapi.h */
> +
>  #include <stream_socket.h>
>  #include "mapi.h"
>  #include "mcrypt.h"
>  
> +#include "pbstream.h"
> +
>  #ifdef HAVE_UNISTD_H
>  # include <unistd.h>
>  #endif
> @@ -946,6 +949,7 @@ struct MapiResultSet {
>  	struct MapiColumn *fields;
>  	struct MapiRowBuf cache;
>  	int commentonly;	/* only comments seen so far */
> +	MapiReply__QueryResult *pb;
>  };
>  
>  struct MapiStatement {
> @@ -1429,7 +1433,6 @@ close_result(MapiHdl hdl)
>  {
>  	struct MapiResultSet *result;
>  	Mapi mid;
> -	int i;
>  
>  	result = hdl->result;
>  	if (result == NULL)
> @@ -1438,125 +1441,13 @@ close_result(MapiHdl hdl)
>  	assert(mid != NULL);
>  	if (mid->trace == MAPI_TRACE)
>  		printf("closing result set\n");
> -	if (result->tableid >= 0 && result->querytype != Q_PREPARE) {
> -		if (mid->active &&
> -		    result->next == NULL &&
> -		    !mid->active->needmore &&
> -		    read_into_cache(mid->active, -1) != MOK)
> -			return MERROR;
> -		assert(hdl->npending_close == 0 ||
> -		       (hdl->npending_close > 0 && hdl->pending_close != NULL));
> -		if (mid->active &&
> -		    (mid->active->active != result ||
> -		     result->cache.tuplecount < result->row_count))
> -		{
> -			/* results for which we got all tuples at the initial
> -			 * response, need not to be closed as the server already
> -			 * did that immediately */
> -			if (result->row_count > result->tuple_count) {
> -				/* can't write "X" commands now, so save for later */
> -				REALLOC(hdl->pending_close, hdl->npending_close + 1);
> -				hdl->pending_close[hdl->npending_close] = result->tableid;
> -				hdl->npending_close++;
> -			}
> -		} else if (mid->to != NULL) {
> -			/* first close saved up to-be-closed tables */
> -			for (i = 0; i < hdl->npending_close; i++) {
> -				char msg[256];
> -
> -				snprintf(msg, sizeof(msg), "Xclose %d\n", hdl->pending_close[i]);
> -				mapi_log_record(mid, msg);
> -				mid->active = hdl;
> -				if (mnstr_printf(mid->to, "%s", msg) < 0 ||
> -				    mnstr_flush(mid->to)) {
> -					close_connection(mid);
> -					mapi_setError(mid, mnstr_error(mid->to), "mapi_close_handle",
> MTIMEOUT);
> -					break;
> -				}
> -				read_into_cache(hdl, 0);
> -			}
> -			hdl->npending_close = 0;
> -			if (hdl->pending_close)
> -				free(hdl->pending_close);
> -			hdl->pending_close = NULL;
> -			if (mid->to != NULL && result->tuple_count < result->row_count) {
> -				char msg[256];
> -
> -				snprintf(msg, sizeof(msg), "Xclose %d\n", result->tableid);
> -				mapi_log_record(mid, msg);
> -				mid->active = hdl;
> -				if (mnstr_printf(mid->to, "%s", msg) < 0 ||
> -				    mnstr_flush(mid->to)) {
> -					close_connection(mid);
> -					mapi_setError(mid, mnstr_error(mid->to), "mapi_close_handle",
> MTIMEOUT);
> -				} else
> -					read_into_cache(hdl, 0);
> -			}
> -		}
> -		result->tableid = -1;
> +	if (hdl->active != NULL) {
> +		// TODO: cleanup
>  	}
> -	if (mid->active == hdl &&
> -	    hdl->active == result &&
> -	    read_into_cache(hdl, -1) != MOK)
> -		return MERROR;
> -	assert(hdl->active != result);
> -	if (result->fields) {
> -		for (i = 0; i < result->maxfields; i++) {
> -			if (result->fields[i].tablename)
> -				free(result->fields[i].tablename);
> -			if (result->fields[i].columnname)
> -				free(result->fields[i].columnname);
> -			if (result->fields[i].columntype)
> -				free(result->fields[i].columntype);
> -		}
> -		free(result->fields);
> -	}
> -	result->fields = NULL;
> -	result->maxfields = result->fieldcnt = 0;
> -	if (result->cache.line) {
> -		for (i = 0; i < result->cache.writer; i++) {
> -			if (result->cache.line[i].rows)
> -				free(result->cache.line[i].rows);
> -			if (result->cache.line[i].anchors) {
> -				int j;
> -
> -				for (j = 0; j < result->cache.line[i].fldcnt; j++)
> -					if (result->cache.line[i].anchors[j]) {
> -						free(result->cache.line[i].anchors[j]);
> -						result->cache.line[i].anchors[j] = NULL;
> -					}
> -				free(result->cache.line[i].anchors);
> -			}
> -			if (result->cache.line[i].lens)
> -				free(result->cache.line[i].lens);
> -		}
> -		free(result->cache.line);
> -		result->cache.line = NULL;
> -		result->cache.tuplecount = 0;
> -	}
> -	if (result->errorstr)
> -		free(result->errorstr);
> -	result->errorstr = NULL;
> -	result->hdl = NULL;
> -	hdl->result = result->next;
> -	if (hdl->result == NULL)
> -		hdl->lastresult = NULL;
> -	result->next = NULL;
> -	free(result);
> +
>  	return MOK;
>  }
>  
> -static void
> -add_error(struct MapiResultSet *result, char *error)
> -{
> -	/* concatenate the error messages */
> -	size_t size = result->errorstr ? strlen(result->errorstr) : 0;
> -
> -	REALLOC(result->errorstr, size + strlen(error) + 2);
> -	strcpy(result->errorstr + size, error);
> -	strcat(result->errorstr + size, "\n");
> -}
> -
>  char *
>  mapi_result_error(MapiHdl hdl)
>  {
> @@ -1571,17 +1462,11 @@ MapiMsg
>  mapi_next_result(MapiHdl hdl)
>  {
>  	mapi_hdl_check(hdl, "mapi_next_result");
> -
> -	while (hdl->result != NULL) {
> +// TODO: we only like to export more
> +	while (hdl->active != NULL) {
>  		if (close_result(hdl) != MOK)
>  			return MERROR;
> -		if (hdl->result &&
> -		    (hdl->result->querytype == -1 ||
> -			 /* basically exclude Q_PARSE and Q_BLOCK */
> _______________________________________________
> checkin-list mailing list
> checkin-list at monetdb.org
> https://www.monetdb.org/mailman/listinfo/checkin-list
> 

-- 
| Stefan.Manegold at CWI.nl | DB Architectures   (DA) |
| www.CWI.nl/~manegold/  | Science Park 123 (L321) |
| +31 (0)20 592-4212     | 1098 XG Amsterdam  (NL) |




More information about the developers-list mailing list