Re: MonetDB: Protobuf - Protocol buffer prototype for the record

Thanks, Hannes!

Just for the records, new (and required) files pbstream.c pbstream.h mapi_pb-c.c mapi_pb-c.h are missing.

Stefan

----- Original Message -----

Changeset: af2757009ce3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=af2757009ce3 Modified Files: clients/mapiclient/Makefile.ag clients/mapiclient/mclient.c clients/mapilib/Makefile.ag clients/mapilib/mapi.c clients/mapilib/mapi.h common/stream/Makefile.ag common/stream/stream.c common/utils/mcrypt.c common/utils/mcrypt.h configure.ag monetdb5/extras/jaql/Makefile.ag monetdb5/extras/mal_optimizer_template/Makefile.ag monetdb5/mal/Makefile.ag monetdb5/mal/mal_authorize.c monetdb5/mal/mal_client.c monetdb5/mal/mal_client.h monetdb5/mal/mal_debugger.c monetdb5/mal/mal_readline.c monetdb5/mal/mal_scenario.c monetdb5/mal/mal_session.c monetdb5/mal/mal_session.h monetdb5/modules/atoms/Makefile.ag monetdb5/modules/kernel/Makefile.ag monetdb5/modules/mal/Makefile.ag monetdb5/modules/mal/mal_mapi.c monetdb5/modules/mal/mal_mapi.h monetdb5/optimizer/Makefile.ag monetdb5/scheduler/Makefile.ag sql/backends/monet5/LSST/Makefile.ag sql/backends/monet5/Makefile.ag sql/backends/monet5/UDF/Makefile.ag sql/backends/monet5/sql_result.c sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_scenario.h sql/backends/monet5/vaults/Makefile.ag sql/storage/bat/Makefile.ag tools/merovingian/daemon/client.c tools/mserver/Makefile.ag tools/mserver/mserver5.c Branch: Protobuf Log Message:

Protocol buffer prototype for the record

diffs (truncated from 2665 to 300 lines):

diff --git a/clients/mapiclient/Makefile.ag b/clients/mapiclient/Makefile.ag --- a/clients/mapiclient/Makefile.ag +++ b/clients/mapiclient/Makefile.ag @@ -16,7 +16,7 @@ # All Rights Reserved.

MTSAFE -INCLUDES = ../mapilib ../../common/options ../../common/stream $(READLINE_INCS) +INCLUDES = ../mapilib ../../common/options ../../common/stream $(READLINE_INCS) $(PROTOBUF_C_CFLAGS)

lib_mcutil = { NOINST @@ -35,8 +35,8 @@ bin_msqldump = { SOURCES = msqldump.c LIBS = libmcutil ../mapilib/libmapi \ ../../common/stream/libstream \

• $(curl_LIBS)
  

-}

• $(curl_LIBS)
  

• }

bin_stethoscope = { SOURCES = stethoscope.c diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -347,13 +347,9 @@ fetch_line(MapiHdl hdl) static int fetch_row(MapiHdl hdl) {

• char *reply;

• do {

• if ((reply = fetch_line(hdl)) == NULL)
  

• 	return 0;
  

• } while (*reply != '[' && *reply != '=');

• return mapi_split_line(hdl);

• // TODO: this is almost a NOP, because we already have everything from the

PB. fetch_line() just has to pay attention to the export size in the future.

• return mapi_fetch_row(hdl);

}

static void @@ -668,11 +664,14 @@ CSVrenderer(MapiHdl hdl) s = mapi_get_name(hdl, i); if (s == NULL) s = "";

• 	mnstr_printf(toConsole, "%s%s",
  

• 	printf( "%s%s",
  	     i == 0 ? "" : sep, s ? s : "");
  

  }

• mnstr_printf(toConsole, "\n");
  

• printf( "\n");
  

  }
• // TODO: switch on type!
• while (!mnstr_errnr(toConsole) && (fields = fetch_row(hdl)) != 0) { for (i = 0; i < fields; i++) { s = mapi_fetch_field(hdl, i);

@@ -681,37 +680,38 @@ CSVrenderer(MapiHdl hdl) if (strchr(s, *sep) != NULL || strchr(s, '\n') != NULL || strchr(s, '"') != NULL) {

• 		mnstr_printf(toConsole, "%s\"",
  

• 		printf( "%s\"",
  		      i == 0 ? "" : sep);
  

• 		printf("%s",s);
  	while (*s) {
  		switch (*s) {
  		case '\n':
  

• 				mnstr_write(toConsole, "\\n", 1, 2);
  

• 				printf( "\\n");
  			break;
  		case '\t':
  

• 				mnstr_write(toConsole, "\\t", 1, 2);
  

• 				printf( "\\t");
  			break;
  		case '\r':
  

• 				mnstr_write(toConsole, "\\r", 1, 2);
  

• 				printf( "\\r");
  			break;
  		case '\\':
  

• 				mnstr_write(toConsole, "\\\\", 1, 2);
  

• 				printf( "\\\\");
  			break;
  		case '"':
  

• 				mnstr_write(toConsole, "\"\"", 1, 2);
  

• 				printf( "\"\"");
  			break;
  		default:
  

• 				mnstr_write(toConsole, s, 1, 1);
  

• 			break;
  		}
  		s++;
  	}
  

• 		mnstr_write(toConsole, "\"", 1, 1);
  

• 		printf("\"");
  } else
  

• 		mnstr_printf(toConsole, "%s%s",
  

• 		printf( "%s%s",
  		      i == 0 ? "" : sep, s ? s : "");
  

  }

• mnstr_printf(toConsole, "\n");
  

• printf( "\n");
  

  }

}

diff --git a/clients/mapilib/Makefile.ag b/clients/mapilib/Makefile.ag --- a/clients/mapilib/Makefile.ag +++ b/clients/mapilib/Makefile.ag @@ -18,11 +18,11 @@ MTSAFE

INCLUDES = ../../common/options ../../common/stream ../../common/utils \

•    $(MSGCONTROL_FLAGS)
  

•  $(MSGCONTROL_FLAGS) $(PROTOBUF_C_CFLAGS)
  
  

lib_mapi = { VERSION = $(MAPI_VERSION)

• SOURCES = mapi.c mapi.rc

• SOURCES = mapi.c mapi.rc LIBS = $(SOCKET_LIBS) ../../common/stream/libstream \ ../../common/options/libmoptions \ ../../common/utils/libmcrypt $(openssl_LIBS)

diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c --- a/clients/mapilib/mapi.c +++ b/clients/mapilib/mapi.c @@ -772,10 +772,13 @@

#include "monetdb_config.h" #include <stream.h> /* include before mapi.h */

#include <stream_socket.h> #include "mapi.h" #include "mcrypt.h"

+#include "pbstream.h"

#ifdef HAVE_UNISTD_H # include <unistd.h> #endif @@ -946,6 +949,7 @@ struct MapiResultSet { struct MapiColumn *fields; struct MapiRowBuf cache; int commentonly; /* only comments seen so far */

• MapiReply__QueryResult *pb;

};

struct MapiStatement { @@ -1429,7 +1433,6 @@ close_result(MapiHdl hdl) { struct MapiResultSet *result; Mapi mid;

• int i;

  result = hdl->result; if (result == NULL)

@@ -1438,125 +1441,13 @@ close_result(MapiHdl hdl) assert(mid != NULL); if (mid->trace == MAPI_TRACE) printf("closing result set\n");

• if (result->tableid >= 0 && result->querytype != Q_PREPARE) {

• if (mid->active &&
  

•     result->next == NULL &&
  

•     !mid->active->needmore &&
  

•     read_into_cache(mid->active, -1) != MOK)
  

• 	return MERROR;
  

• assert(hdl->npending_close == 0 ||
  

•        (hdl->npending_close > 0 && hdl->pending_close != NULL));
  

• if (mid->active &&
  

•     (mid->active->active != result ||
  

•      result->cache.tuplecount < result->row_count))
  

• {
  

• 	/* results for which we got all tuples at the initial
  

• 	 * response, need not to be closed as the server already
  

• 	 * did that immediately */
  

• 	if (result->row_count > result->tuple_count) {
  

• 		/* can't write "X" commands now, so save for later */
  

• 		REALLOC(hdl->pending_close, hdl->npending_close + 1);
  

• 		hdl->pending_close[hdl->npending_close] = result->tableid;
  

• 		hdl->npending_close++;
  

• 	}
  

• } else if (mid->to != NULL) {
  

• 	/* first close saved up to-be-closed tables */
  

• 	for (i = 0; i < hdl->npending_close; i++) {
  

• 		char msg[256];
  

• 		snprintf(msg, sizeof(msg), "Xclose %d\n", hdl->pending_close[i]);
  

• 		mapi_log_record(mid, msg);
  

• 		mid->active = hdl;
  

• 		if (mnstr_printf(mid->to, "%s", msg) < 0 ||
  

• 		    mnstr_flush(mid->to)) {
  

• 			close_connection(mid);
  

• 			mapi_setError(mid, mnstr_error(mid->to), "mapi_close_handle",
  

MTIMEOUT);

• 			break;
  

• 		}
  

• 		read_into_cache(hdl, 0);
  

• 	}
  

• 	hdl->npending_close = 0;
  

• 	if (hdl->pending_close)
  

• 		free(hdl->pending_close);
  

• 	hdl->pending_close = NULL;
  

• 	if (mid->to != NULL && result->tuple_count < result->row_count) {
  

• 		char msg[256];
  

• 		snprintf(msg, sizeof(msg), "Xclose %d\n", result->tableid);
  

• 		mapi_log_record(mid, msg);
  

• 		mid->active = hdl;
  

• 		if (mnstr_printf(mid->to, "%s", msg) < 0 ||
  

• 		    mnstr_flush(mid->to)) {
  

• 			close_connection(mid);
  

• 			mapi_setError(mid, mnstr_error(mid->to), "mapi_close_handle",
  

MTIMEOUT);

• 		} else
  

• 			read_into_cache(hdl, 0);
  

• 	}
  

• }
  

• result->tableid = -1;
  

• if (hdl->active != NULL) {

• // TODO: cleanup
  

  }

• if (mid->active == hdl &&

•    hdl->active == result &&
  

•    read_into_cache(hdl, -1) != MOK)
  

• return MERROR;
  

• assert(hdl->active != result);
• if (result->fields) {

• for (i = 0; i < result->maxfields; i++) {
  

• 	if (result->fields[i].tablename)
  

• 		free(result->fields[i].tablename);
  

• 	if (result->fields[i].columnname)
  

• 		free(result->fields[i].columnname);
  

• 	if (result->fields[i].columntype)
  

• 		free(result->fields[i].columntype);
  

• }
  

• free(result->fields);
  

• }
• result->fields = NULL;
• result->maxfields = result->fieldcnt = 0;
• if (result->cache.line) {

• for (i = 0; i < result->cache.writer; i++) {
  

• 	if (result->cache.line[i].rows)
  

• 		free(result->cache.line[i].rows);
  

• 	if (result->cache.line[i].anchors) {
  

• 		int j;
  

• 		for (j = 0; j < result->cache.line[i].fldcnt; j++)
  

• 			if (result->cache.line[i].anchors[j]) {
  

• 				free(result->cache.line[i].anchors[j]);
  

• 				result->cache.line[i].anchors[j] = NULL;
  

• 			}
  

• 		free(result->cache.line[i].anchors);
  

• 	}
  

• 	if (result->cache.line[i].lens)
  

• 		free(result->cache.line[i].lens);
  

• }
  

• free(result->cache.line);
  

• result->cache.line = NULL;
  

• result->cache.tuplecount = 0;
  

• }
• if (result->errorstr)

• free(result->errorstr);
  

• result->errorstr = NULL;
• result->hdl = NULL;
• hdl->result = result->next;
• if (hdl->result == NULL)

• hdl->lastresult = NULL;
  

• result->next = NULL;
• free(result);

• return MOK;

}

-static void -add_error(struct MapiResultSet *result, char *error) -{

• /* concatenate the error messages */
• size_t size = result->errorstr ? strlen(result->errorstr) : 0;
• REALLOC(result->errorstr, size + strlen(error) + 2);
• strcpy(result->errorstr + size, error);
• strcat(result->errorstr + size, "\n");

-}

char * mapi_result_error(MapiHdl hdl) { @@ -1571,17 +1462,11 @@ MapiMsg mapi_next_result(MapiHdl hdl) { mapi_hdl_check(hdl, "mapi_next_result");

• while (hdl->result != NULL) {

+// TODO: we only like to export more

• while (hdl->active != NULL) { if (close_result(hdl) != MOK) return MERROR;

• if (hdl->result &&
  

•     (hdl->result->querytype == -1 ||
  

• 	 /* basically exclude Q_PARSE and Q_BLOCK */
  

---

checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list