developers-list April 2015

developers-list@monetdb.org

7 participants
5 discussions

by Bastian Köcher

Hi, I want to write an optimizer for MonetDB in C++. So I included the necessary headers into my project, but I ran into some compile errors and warnings. I created a patch which solves some of these problems. Regards, Bastian

7 years, 3 months

Re: MonetDB: default - remove jsonstore code we do not need for the ...

by Stefan Manegold

Arjen, are these left-overs occurrences of "jsonstore" in the default branch still used / required, or can they also be removed? ======== .../default/source/MonetDB $ hg locate | grep --color -i jsonstore sql/scripts/41_jsonstore.sql .../default/source/MonetDB $ hg locate -X \**/Tests/*.stable.* | xargs grep -l -i jsonstore MonetDB.spec NT/installer32/MonetDB5-SQL-Installer.vdproj NT/installer64/MonetDB5-SQL-Installer.vdproj NT/monetdb_config.h.in NT/rules.msc clients/Tests/All monetdb5/mal/mal.c sql/backends/monet5/sql_upgrades.c sql/scripts/41_jsonstore.sql sql/scripts/Makefile.ag testing/Mtest.py.in ======== Thanks! Stefan ----- On Apr 21, 2015, at 12:37 PM, Sjoerd Mullender commits(a)monetdb.org wrote: > Changeset: e6f13746b75b for MonetDB > URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e6f13746b75b > Removed Files: > sql/backends/monet5/rest/73_jsonstore.mal > sql/backends/monet5/rest/Makefile.ag > sql/backends/monet5/rest/Tests/All > sql/backends/monet5/rest/Tests/jsonstore00.sql > sql/backends/monet5/rest/Tests/jsonstore00.stable.err > sql/backends/monet5/rest/Tests/jsonstore00.stable.out > sql/backends/monet5/rest/Tests/jsonstore01.sql > sql/backends/monet5/rest/Tests/jsonstore01.stable.err > sql/backends/monet5/rest/Tests/jsonstore01.stable.out > sql/backends/monet5/rest/jsonstore.mal > sql/backends/monet5/rest/rest_jsonstore.c > sql/backends/monet5/rest/rest_jsonstore.h > sql/backends/monet5/rest/rest_jsonstore_handle_get.c > sql/backends/monet5/rest/rest_jsonstore_handle_get.h > Modified Files: > configure.ag > monetdb5/mal/Makefile.ag > monetdb5/mal/mal_http_daemon.c > sql/backends/monet5/Makefile.ag > Branch: default > Log Message: > > remove jsonstore code we do not need for the next feature release > > > diffs (truncated from 2238 to 300 lines): > > diff --git a/configure.ag b/configure.ag > --- a/configure.ag > +++ b/configure.ag > @@ -188,30 +188,13 @@ no-auto) > ;; > esac > > -dft_jsonstore=no > -AC_ARG_ENABLE(jsonstore, > - AS_HELP_STRING([--enable-jsonstore], > - [enable support for MonetDB/jsonstore (default=no)]), > - enable_jsonstore=$enableval, > - enable_jsonstore=$dft_jsonstore) > - > -dft_microhttpd=$enable_jsonstore > +dft_microhttpd=no > AC_ARG_ENABLE(microhttpd, > AS_HELP_STRING([--enable-microhttpd], > - [enable support for the builtin MircoHTTPD daemon (default: implied by > --enable-jsonstore)]), > + [enable support for the builtin MircoHTTPD daemon]), > enable_microhttpd=$enableval, > enable_microhttpd=$dft_microhttpd) > > -case "$enable_jsonstore-$enable_microhttpd" in > -yes-no) > - AC_MSG_ERROR([jsonstore require microhttpd]) > - ;; > -auto-no) > - enable_jsonstore=no > - disable_jsonstore="(jsonstore requires microhttpd)" > - ;; > -esac > - > dft_gsl=auto > AC_ARG_ENABLE(gsl, > AS_HELP_STRING([--enable-gsl], > @@ -2508,7 +2491,7 @@ if test x"$have_atomic_ops" = x"yes" ; t > fi > > dnl MicroHTTPD library > -dnl required for MonetDB5 SQL jsonstore, optional otherwise > +dnl optional > org_have_libmicrohttpd=auto > have_libmicrohttpd=$org_have_libmicrohttpd > if test "x$enable_microhttpd" != xno; then > @@ -2526,24 +2509,6 @@ if test x$enable_microhttpd != xno; then > AC_DEFINE(HAVE_MICROHTTPD, 1, [Define if the microhttpd server is to be > enabled]) > fi > > -org_have_liburiparser=auto > -have_liburiparser=$org_have_liburiparser > -if test "x$enable_jsonstore" != xno; then > - dnl Uriparser library > - dnl required for MonetDB5 SQL jsonstore, optional otherwise > - PKG_CHECK_MODULES([liburiparser], [liburiparser], > - [have_liburiparser=yes], > - [have_liburiparser=no; disable_jsonstore="(liburiparser not found)"; if test > "x$enable_jsonstore" = xyes; then AC_MSG_ERROR([LibUriParser library not found > but required for jsonstore]); fi]) > - > - if test "$have_libmicrohttpd-$have_liburiparser" = yes-yes; then > - AC_DEFINE(HAVE_JSONSTORE, 1, [Define that you want to build the jsonstore > library]) > - elif test "x$enable_jsonstore" = xyes; then > - AC_MSG_ERROR([cannot enable jsonstore > $why_have_libmicrohttpd$why_have_liburiparser]) > - else > - enable_jsonstore=no > - fi > -fi > - > dnl Check for gsl library for statistical functions > org_have_gsl=no > have_gsl=$org_have_gsl > @@ -2920,7 +2885,6 @@ AM_CONDITIONAL(HAVE_GDK, test "x$enable_ > AM_CONDITIONAL(HAVE_MONETDB5, test "x$enable_monetdb5" = xyes -o > "x$enable_monetdb5" = xauto) > AM_CONDITIONAL(HAVE_SQL, test "x$enable_sql" = xyes -o "x$enable_sql" = xauto) > AM_CONDITIONAL(HAVE_GEOM, test "x$enable_geom" = xyes -o "x$enable_geom" = > xauto) > -AM_CONDITIONAL(HAVE_JSONSTORE, test "x$enable_jsonstore" = xyes -o > "x$enable_jsonstore" = xauto) > AM_CONDITIONAL(HAVE_TESTING, test "x$enable_testing" = xyes -o > "x$enable_testing" = xauto) > AM_CONDITIONAL(HAVE_DEVELOPER, test "x$enable_developer" = xyes) > > @@ -3277,7 +3241,6 @@ for comp in \ > 'console ' \ > 'gsl ' \ > 'fits ' \ > - 'jsonstore ' \ > 'microhttpd ' \ > 'rintegration' \ > 'odbc ' \ > diff --git a/monetdb5/mal/Makefile.ag b/monetdb5/mal/Makefile.ag > --- a/monetdb5/mal/Makefile.ag > +++ b/monetdb5/mal/Makefile.ag > @@ -9,7 +9,6 @@ INCLUDES = ../../common/options \ > ../../common/utils \ > ../../clients/mapilib \ > ../../gdk \ > - #../../sql/backends/monet5/rest \ > $(READLINE_INCS) \ > $(libmicrohttpd_CFLAGS) > MTSAFE > diff --git a/monetdb5/mal/mal_http_daemon.c b/monetdb5/mal/mal_http_daemon.c > --- a/monetdb5/mal/mal_http_daemon.c > +++ b/monetdb5/mal/mal_http_daemon.c > @@ -12,262 +12,6 @@ > > #include "monetdb_config.h" > > -#ifdef HAVE_MICROHTTPD > -#include <sys/types.h> > -#include <sys/select.h> > -#include <sys/socket.h> > -#include <stdint.h> > -#include <stdarg.h> > -#include <stdio.h> > - > -#include "mal_client.h" > -#include "mal_http_daemon.h" > -#include "mal_private.h" > -#include <microhttpd.h> > - > -static MT_Id hdthread; > -static int volatile hdrunning; > -struct MHD_Daemon *http_daemon; > -#define PORT 8998 > -#define POSTBUFFERSIZE 512 > -#define MAXNAMESIZE 512 > -#define MAXANSWERSIZE 512 > - > -#define GET 0 > -#define POST 1 > - > -static > -http_request_handler http_handler; > - > -void register_http_handler(http_request_handler handler) { > - http_handler = handler; > -} > - > -struct connection_info_struct > -{ > - int connectiontype; > - char *answerstring; > - struct MHD_PostProcessor *postprocessor; > -}; > - > -static int > -send_page (struct MHD_Connection *connection, const char * url, > - const char * method, char *page, char * postdata) > -{ > - int ret; > - int rest; > - struct MHD_Response *response; > - > - rest = (*http_handler)(url, method, &page, postdata); > - (void)rest; > - response = > - MHD_create_response_from_buffer (strlen (page), > - (void *) page, > - MHD_RESPMEM_MUST_COPY); > - if (!response) > - return MHD_NO; > - > - ret = MHD_queue_response (connection, MHD_HTTP_OK, response); > - MHD_destroy_response (response); > - > - return ret; > -} > - > -static int > -iterate_post (void *coninfo_cls, enum MHD_ValueKind kind, const char *key, > - const char *filename, const char *content_type, > - const char *transfer_encoding, const char *data, uint64_t off, > - size_t size) > -{ > - struct connection_info_struct *con_info = coninfo_cls; > - char *answerstring; > - > - (void)key; > - (void)kind; > - (void)filename; > - (void)content_type; > - (void)transfer_encoding; > - (void)off; > - > - if (strcmp (key, "json") == 0) > - { > - if ((size > 0) && (size <= MAXNAMESIZE)) > - { > - answerstring = malloc (MAXANSWERSIZE); > - if (!answerstring) > - return MHD_NO; > - > - snprintf (answerstring, MAXANSWERSIZE, "%s", data); > - con_info->answerstring = answerstring; > - } else > - con_info->answerstring = NULL; > - > - return MHD_NO; > - } > - > - if (strcmp (key, "file") == 0) > - { > - if ((size > 0) && (size <= MAXNAMESIZE)) > - { > - answerstring = malloc (MAXANSWERSIZE); > - if (!answerstring) > - return MHD_NO; > - > - snprintf (answerstring, MAXANSWERSIZE, "%s", data); > - con_info->answerstring = answerstring; > - } else > - con_info->answerstring = NULL; > - > - return MHD_NO; > - } > - > - return MHD_YES; > -} > - > -static void > -request_completed (void *cls, struct MHD_Connection *connection, > - void **con_cls, enum MHD_RequestTerminationCode toe) > -{ > - struct connection_info_struct *con_info = *con_cls; > - > - (void)cls; > - (void)connection; > - (void)toe; > - > - if (con_info == NULL) > - return; > - > - if (con_info->connectiontype == POST) > - { > - MHD_destroy_post_processor (con_info->postprocessor); > - if (con_info->answerstring) > - free (con_info->answerstring); > - } > - > - free (con_info); > - *con_cls = NULL; > -} > - > -static int > -answer_to_connection (void *cls, struct MHD_Connection *connection, > - const char *url, const char *method, > - const char *version, const char *upload_data, > - size_t *upload_data_size, void **con_cls) > -{ > - char * page = NULL; > - struct connection_info_struct *con_info = NULL; > - int *done = cls; > - char *answerstring = NULL; > - char *errorpage = > - "<html><body>" > - "Failed to handle error in http request." > - "</body></html>"; > - > - (void)version; > - > - if (*con_cls == NULL) { > - con_info = malloc (sizeof (struct connection_info_struct)); > - if (con_info == NULL) > - return MHD_NO; > - con_info->answerstring = NULL; > - > - if (strcmp (method, "POST") == 0) { > - con_info->postprocessor = > - MHD_create_post_processor (connection, > - POSTBUFFERSIZE, > - iterate_post, > - (void *) con_info); > - > - if (con_info->postprocessor == NULL) { > - free (con_info); > - return MHD_NO; > - } > - con_info->connectiontype = POST; > - } else { > - con_info->connectiontype = GET; > - } > - *con_cls = (void *) con_info; > - return MHD_YES; > - } > - > - if (strcmp (method, "POST") == 0) { > - con_info = *con_cls; > - if (*upload_data_size != 0) { > - MHD_post_process (con_info->postprocessor, upload_data, > - *upload_data_size); > - *upload_data_size = 0; > - return MHD_YES; > - } else { > - if (con_info->answerstring != NULL) { > - return send_page(connection, url, method, page, > - con_info->answerstring); > - } > - } > - } > - > - if (strcmp (method, "PUT") == 0) { > - if (*upload_data_size != 0) { > - // TODO: check free answerstring > _______________________________________________ > checkin-list mailing list > checkin-list(a)monetdb.org > https://www.monetdb.org/mailman/listinfo/checkin-list -- | Stefan.Manegold(a)CWI.nl | DB Architectures (DA) | | www.CWI.nl/~manegold/ | Science Park 123 (L321) | | +31 (0)20 592-4212 | 1098 XG Amsterdam (NL) |

7 years, 3 months

Re: developers-list Digest, Raúl

by José Raúl Pérez Rodríguez

Thank you very much Hannes, Then I can assume that queries to a distributed table can' t be sorted. It is a serious limitation in MonetDb scaling, I would like to work to help in adding this capability. Raúl > Today's Topics: > > 1. Re: Querying a monetdb cluster (Hannes M?hleisen) > > > Hello Jos? Ra?l, > >> On 09.04.2015, at 12:45, Jos? Ra?l P?rez Rodr?guez >> <joseraul.perezrodriguez(a)gmail.com> wrote: >> >> I maked a test inserting data into each particular node of a two node >> cluster, but I don't see the way of making queries to the cluster, I >> known that multiplex funnel can return data from several nodes, >> joining data obtained from each node, but with multiplex funnel, I >> can't sort the data. > This is not supported yet. > > Best, > > Hannes

7 years, 3 months

Querying a monetdb cluster

by José Raúl Pérez Rodríguez

Hi, I am new in monetdb. How do I make a query to a cluster? I maked a test inserting data into each particular node of a two node cluster, but I don't see the way of making queries to the cluster, I known that multiplex funnel can return data from several nodes, joining data obtained from each node, but with multiplex funnel, I can't sort the data. There is not doc that explains this problem?

7 years, 3 months

MonetDB, data loss in restart server

by Luciano Sasso

Hello, I have used the MonetDB in production for some time, and I have nothing to complain about, are tables with more than 30 columns and 30 million records, do analysis through cubes in Pentaho / Saiku. Have restarted the server several times, and never had problems with loss of data.But today, after performing a query that was consuming a lot of machine, stopped the database, and when I started, data from some columns were blank. How can it, will the data were still uncommitted? when performing a query the MonetDB changes the data or reorganized? which may have occurred, I am concerned. -- Luciano Sasso Vieira Gestor - Inteligência de Negócios luciano(a)gsgroup.com.br <http://www.gsgroup.com.br> | tel: 17 3353-0833 | cel: 17 9706-9335 www.gsgroup.com.br <http://www.gsgroup.com.br> --- Este email foi escaneado pelo Avast antivírus. http://www.avast.com

7 years, 3 months

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

2004

developers-list April 2015