MonetDB: default - add support for unix sockets

Stefan Manegold Stefan.Manegold at cwi.nl
Fri Sep 6 08:18:14 CEST 2013


Thanks!

For details see
http://monetdb.cwi.nl/testweb/web/testgrid.php?serial=48740:6d8d3779113d&order=platform,arch,compiler&targets=GNU-Darwin-i386-propcheck,GNU-Darwin-powerpc-propcheck,GNU-Darwin-x86_64-oid32,GNU-Darwin-x86_64,GNU-Fedora-x86_64-assert-propcheck,GNU-Fedora-x86_64-dbfarm,GNU-Fedora-x86_64-oid32-assert-propcheck,GNU-Fedora-x86_64-oid32-dbfarm,GNU-Fedora-x86_64-thrs=1-assert,GNU-Fedora-x86_64-thrs=1-propcheck-dist,GNU-Fedora-x86_64-thrs=32-propcheck-rpm,GNU-Fedora-x86_64-thrs=53-assert,GNU-FreeBSD-x86_64,GNU-Gentoo-powerpc-assert-dbfarm,GNU-Solaris-sparc-dbfarm,GNU-Solaris-sparcv9-dbfarm,GNU-Solaris-sparcv9-oid32-dbfarm,GNU-Ubuntu-i386-assert-propcheck-dbfarm,Int-Fedora-x86_64-assert,Int-Fedora-x86_64-oid32-assert,Int-Fedora-x86_64-oid32-propcheck,Int-Fedora-x86_64-propcheck,Int-Windows7-x86_64-assert,Int-Windows7-x86_64-oid32-assert,Int-WindowsXP-i386-assert,Mic-Windows7-i386-installer&module=sql

in particular
http://monetdb.cwi.nl/testweb/web/showtestoutput.php?serial=48740:6d8d3779113d&target=GNU-Darwin-i386-propcheck&module=sql&test=sql%2Ftest%2Fmapi%2Fpython2_dbapi&which=err
http://monetdb.cwi.nl/testweb/web/showtestoutput.php?serial=48740:6d8d3779113d&target=Int-WindowsXP-i386-assert&module=sql&test=sql%2Ftest%2Fmapi%2Fpython2_dbapi&which=err
http://monetdb.cwi.nl/testweb/web/showtestoutput.php?serial=48740:6d8d3779113d&target=GNU-Darwin-i386-propcheck&module=sql&test=sql%2Ftest%2Fmapi%2Fpython3_dbapi&which=err
http://monetdb.cwi.nl/testweb/web/showtestoutput.php?serial=48740:6d8d3779113d&target=Int-WindowsXP-i386-assert&module=sql&test=sql%2Ftest%2Fmapi%2Fpython3_dbapi&which=err

Stefan

----- Original Message -----
> 
> ok, i'll fix it tomorrow.
> 
> 
> On 09/05/2013 06:56 PM, Stefan Manegold wrote:
> > Hi Gijs,
> >
> > in case you did not notice this, yet:
> >
> > This checkin breaks several tests that do test or use the Python interface,
> > resulting in errors like
> >
> > "
> > Traceback (most recent call last):
> >    File "delete_all.SQL.py", line 9, in <module>
> >      dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True)
> >    File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py",
> >    line 49, in __init__
> >      unix_socket=unix_socket)
> >    File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in
> >    connect
> >      self.socket.connect(unix_socket)
> >    File "/usr/lib64/python2.7/socket.py", line 224, in meth
> >      return getattr(self._sock,name)(*args)
> > socket.error: [Errno 2] No such file or directory
> > "
> >
> > or
> >
> > "
> > Traceback (most recent call last):
> >    File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module>
> >      dbh =
> >      monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True)
> >    File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py",
> >    line 49, in __init__
> >      unix_socket=unix_socket)
> >    File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in
> >    connect
> >      self.socket.connect(unix_socket)
> > FileNotFoundError: [Errno 2] No such file or directory
> > "
> >
> >
> > See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent
> > sql/test/mapi`
> >
> > or our nightly TestWeb @
> > http://monetdb.cwi.nl/testweb/web/status.php?branch=default
> > as of tomorrow morning.
> >
> >
> > Best,
> > Stefan
> >
> >
> > On Thu, Sep 05, 2013 at 02:19:01PM +0200, Gijs Molenaar wrote:
> >> Changeset: 0eaa07b061be for MonetDB
> >> URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0eaa07b061be
> >> Modified Files:
> >> 	clients/python2/monetdb/control.py
> >> 	clients/python2/monetdb/mapi.py
> >> 	clients/python2/monetdb/sql/connections.py
> >> 	clients/python2/test/runtests.py
> >> 	clients/python2/test/test_control.py
> >> 	clients/python3/monetdb/control.py
> >> 	clients/python3/monetdb/mapi.py
> >> 	clients/python3/monetdb/sql/connections.py
> >> 	clients/python3/test/runtests.py
> >> 	clients/python3/test/test_control.py
> >> Branch: default
> >> Log Message:
> >>
> >> add support for unix sockets
> >>
> >>
> >> diffs (truncated from 649 to 300 lines):
> >>
> >> diff --git a/clients/python2/monetdb/control.py
> >> b/clients/python2/monetdb/control.py
> >> --- a/clients/python2/monetdb/control.py
> >> +++ b/clients/python2/monetdb/control.py
> >> @@ -8,10 +8,12 @@ def parse_statusline(line):
> >>       parses a sabdb format status line. Support v1 and v2.
> >>   
> >>       """
> >> -    if not line.startswith('=sabdb:'):
> >> +    if line.startswith("="):
> >> +        line = line[1:]
> >> +    if not line.startswith('sabdb:'):
> >>           raise OperationalError('wrong result recieved')
> >>   
> >> -    prot_version, rest = line.split(":", 2)[1:]
> >> +    code, prot_version, rest = line.split(":", 2)
> >>   
> >>       if prot_version not in ["1", "2"]:
> >>           raise InterfaceError("unsupported sabdb protocol")
> >> @@ -60,20 +62,26 @@ class Control:
> >>       Use this module to manage your MonetDB databases. You can create,
> >>       start,
> >>       stop, lock, unlock, destroy your databases and request status
> >>       information.
> >>       """
> >> -    def __init__(self, hostname, port, passphrase):
> >> +    def __init__(self, hostname=None, port=None, passphrase=None,
> >> +                 unix_socket="/tmp/.s.merovingian.50000"):
> >>           self.server = mapi.Connection()
> >>           self.hostname = hostname
> >>           self.port = port
> >>           self.passphrase = passphrase
> >> +        self.unix_socket= unix_socket
> >>   
> >>           # check connection
> >> -        self.server.connect(hostname, port, 'monetdb', passphrase,
> >> -                            'merovingian', 'control')
> >> +        self.server.connect(hostname=hostname, port=port,
> >> username='monetdb',
> >> +                            password=passphrase,
> >> +                            database='merovingian', language='control',
> >> +                            unix_socket=unix_socket)
> >>           self.server.disconnect()
> >>   
> >>       def _send_command(self, database_name, command):
> >> -        self.server.connect(self.hostname, self.port, 'monetdb',
> >> -                            self.passphrase, 'merovingian', 'control')
> >> +        self.server.connect(hostname=self.hostname, port=self.port,
> >> +                            username='monetdb', password=self.passphrase,
> >> +                            database='merovingian', language='control',
> >> +                            unix_socket=self.unix_socket)
> >>           try:
> >>               return self.server.cmd("%s %s\n" % (database_name, command))
> >>           finally:
> >> @@ -165,8 +173,9 @@ class Control:
> >>           """
> >>           properties = self._send_command(database_name, "get")
> >>           values = {}
> >> -        for dirty_line in properties.split("\n"):
> >> -            line = dirty_line[1:]
> >> +        for line in properties.split("\n"):
> >> +            if line.startswith("="):
> >> +                line = line[1:]
> >>               if not line.startswith("#"):
> >>                   if "=" in line:
> >>                       split = line.split("=")
> >> diff --git a/clients/python2/monetdb/mapi.py
> >> b/clients/python2/monetdb/mapi.py
> >> --- a/clients/python2/monetdb/mapi.py
> >> +++ b/clients/python2/monetdb/mapi.py
> >> @@ -70,8 +70,12 @@ class Connection(object):
> >>           self.database = ""
> >>           self.language = ""
> >>   
> >> -    def connect(self, hostname, port, username, password, database,
> >> language):
> >> -        """ setup connection to MAPI server"""
> >> +    def connect(self, database, username, password, language,
> >> hostname=None,
> >> +                port=None, unix_socket=None):
> >> +        """ setup connection to MAPI server
> >> +
> >> +        unix_socket is used if hostname is not defined.
> >> +        """
> >>   
> >>           self.hostname = hostname
> >>           self.port = port
> >> @@ -79,24 +83,34 @@ class Connection(object):
> >>           self.password = password
> >>           self.database = database
> >>           self.language = language
> >> +        self.unix_socket = unix_socket
> >>   
> >> -        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
> >> +        if hostname:
> >> +            self.socket = socket.socket(socket.AF_INET,
> >> socket.SOCK_STREAM)
> >> +            # For performance, mirror MonetDB/src/common/stream.c socket
> >> settings.
> >> +            self.socket.setsockopt(socket.SOL_SOCKET,
> >> socket.SO_KEEPALIVE, 0)
> >> +            self.socket.setsockopt(socket.IPPROTO_TCP,
> >> socket.TCP_NODELAY, 1)
> >> +            self.socket.connect((hostname, port))
> >> +        else:
> >> +            self.socket = socket.socket(socket.AF_UNIX)
> >> +            self.socket.connect(unix_socket)
> >> +            if self.language != 'control':
> >> +                self.socket.send('0')  # don't know why, but we need to
> >> do this
> >>   
> >> -        # For performance, mirror MonetDB/src/common/stream.c socket
> >> settings.
> >> -        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)
> >> -        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
> >> +        if not (self.language == 'control' and not self.hostname):
> >> +            # control doesn't require authentication over socket
> >> +            self._login()
> >>   
> >> -        self.socket.connect((hostname, port))
> >> -        self.__login()
> >> +        self.state = STATE_READY
> >>   
> >> -    def __login(self, iteration=0):
> >> +    def _login(self, iteration=0):
> >>           """ Reads challenge from line, generate response and check if
> >>           everything is okay """
> >>   
> >> -        challenge = self.__getblock()
> >> -        response = self.__challenge_response(challenge)
> >> -        self.__putblock(response)
> >> -        prompt = self.__getblock().strip()
> >> +        challenge = self._getblock()
> >> +        response = self._challenge_response(challenge)
> >> +        self._putblock(response)
> >> +        prompt = self._getblock().strip()
> >>   
> >>           if len(prompt) == 0:
> >>               # Empty response, server is happy
> >> @@ -117,7 +131,7 @@ class Connection(object):
> >>               if redirect[1] == "merovingian":
> >>                   logger.debug("restarting authentication")
> >>                   if iteration <= 10:
> >> -                    self.__login(iteration=iteration + 1)
> >> +                    self._login(iteration=iteration + 1)
> >>                   else:
> >>                       raise OperationalError("maximal number of redirects
> >>                       "
> >>                                              "reached (10)")
> >> @@ -138,9 +152,6 @@ class Connection(object):
> >>           else:
> >>               raise ProgrammingError("unknown state: %s" % prompt)
> >>   
> >> -        self.state = STATE_READY
> >> -        return True
> >> -
> >>       def disconnect(self):
> >>           """ disconnect from the monetdb server """
> >>           self.state = STATE_INIT
> >> @@ -153,8 +164,8 @@ class Connection(object):
> >>           if self.state != STATE_READY:
> >>               raise(ProgrammingError, "Not connected")
> >>   
> >> -        self.__putblock(operation)
> >> -        response = self.__getblock()
> >> +        self._putblock(operation)
> >> +        response = self._getblock()
> >>           if not len(response):
> >>               return ""
> >>           elif response.startswith(MSG_OK):
> >> @@ -166,10 +177,16 @@ class Connection(object):
> >>               return response
> >>           elif response[0] == MSG_ERROR:
> >>               raise OperationalError(response[1:])
> >> +        elif (self.language == 'control' and not self.hostname):
> >> +            if response.startswith("OK"):
> >> +                return response[2:].strip() or ""
> >> +            else:
> >> +                return response
> >>           else:
> >>               raise ProgrammingError("unknown state: %s" % response)
> >>   
> >> -    def __challenge_response(self, challenge):
> >> +
> >> +    def _challenge_response(self, challenge):
> >>           """ generate a response to a mapi login challenge """
> >>           challenges = challenge.split(':')
> >>           salt, identity, protocol, hashes, endian = challenges[:5]
> >> @@ -204,19 +221,36 @@ class Connection(object):
> >>           return ":".join(["BIG", self.username, pwhash, self.language,
> >>                            self.database]) + ":"
> >>   
> >> -    def __getblock(self):
> >> +    def _getblock(self):
> >>           """ read one mapi encoded block """
> >> +        if (self.language == 'control' and not self.hostname):
> >> +            return self._getblock_socket()  # control doesn't do block
> >> +                                            # splitting when using a
> >> socket
> >> +        else:
> >> +            return self._getblock_inet()
> >> +
> >> +    def _getblock_inet(self):
> >>           result = StringIO()
> >>           last = 0
> >>           while not last:
> >> -            flag = self.__getbytes(2)
> >> +            flag = self._getbytes(2)
> >>               unpacked = struct.unpack('<H', flag)[0]  # little endian
> >>               short
> >>               length = unpacked >> 1
> >>               last = unpacked & 1
> >> -            result.write(self.__getbytes(length))
> >> +            result.write(self._getbytes(length))
> >>           return result.getvalue()
> >>   
> >> -    def __getbytes(self, bytes_):
> >> +    def _getblock_socket(self):
> >> +        buffer = StringIO()
> >> +        while True:
> >> +            x = self.socket.recv(1)
> >> +            if len(x):
> >> +                buffer.write(x)
> >> +            else:
> >> +                break
> >> +        return buffer.getvalue().strip()
> >> +
> >> +    def _getbytes(self, bytes_):
> >>           """Read an amount of bytes from the socket"""
> >>           result = StringIO()
> >>           count = bytes_
> >> @@ -228,8 +262,15 @@ class Connection(object):
> >>               result.write(recv)
> >>           return result.getvalue()
> >>   
> >> -    def __putblock(self, block):
> >> +    def _putblock(self, block):
> >>           """ wrap the line in mapi format and put it into the socket """
> >> +        if (self.language == 'control' and not self.hostname):
> >> +            return self.socket.send(block)  # control doesn't do block
> >> +                                            # splitting when using a
> >> socket
> >> +        else:
> >> +            self._putblock_inet(block)
> >> +
> >> +    def _putblock_inet(self, block):
> >>           pos = 0
> >>           last = 0
> >>           while not last:
> >> diff --git a/clients/python2/monetdb/sql/connections.py
> >> b/clients/python2/monetdb/sql/connections.py
> >> --- a/clients/python2/monetdb/sql/connections.py
> >> +++ b/clients/python2/monetdb/sql/connections.py
> >> @@ -28,25 +28,25 @@ class Connection(object):
> >>       """A MonetDB SQL database connection"""
> >>       default_cursor = cursors.Cursor
> >>   
> >> -    def __init__(self, username="monetdb", password="monetdb",
> >> -                 hostname="localhost", port=50000, database="demo",
> >> -                 autocommit=False, user=None, host=None):
> >> +    def __init__(self, database, hostname=None, port=50000,
> >> username="monetdb",
> >> +                 password="monetdb", unix_socket="/tmp/.s.monetdb.50000",
> >> +                 autocommit=False):
> >>           """ Set up a connection to a MonetDB SQL database.
> >>   
> >> -        username   -- username for connection (default: monetdb)
> >> -        password   -- password for connection (default: monetdb)
> >> -        hostname   -- hostname to connect to (default: localhost)
> >> -        port       -- port to connect to (default: 50000)
> >> -        database   -- name of the database (default: demo)
> >> -        autocommit -- enable/disable auto commit (default: False)
> >> +        database    -- name of the database
> >> +        hostname    -- Hostname where monetDB is running
> >> +        port        -- port to connect to (default: 50000)
> >> +        username    -- username for connection (default: "monetdb")
> >> +        password    -- password for connection (default: "monetdb")
> >> +        unix_socket -- socket to connect to. used when hostname not set
> >> +                            (default: "/tmp/.s.monetdb.50000")
> >> +        autocommit  -- enable/disable auto commit (default: False)
> >> +
> >>           """
> >> -        if user is not None:
> >> -            username = user
> >> -        if host is not None:
> >> -            hostname = host
> >>           self.mapi = mapi.Connection()
> >>           self.mapi.connect(hostname=hostname, port=int(port),
> >>           username=username,
> >> -                          password=password, database=database,
> >> language="sql")
> >> +                          password=password, database=database,
> >> language="sql",
> >> +                          unix_socket=unix_socket)
> >>           self.set_autocommit(autocommit)
> >>           self.set_sizeheader(True)
> >>           self.set_replysize(100)
> >> diff --git a/clients/python2/test/runtests.py
> >> b/clients/python2/test/runtests.py
> >> --- a/clients/python2/test/runtests.py
> >> +++ b/clients/python2/test/runtests.py
> >> @@ -45,6 +45,8 @@ TSTDB = os.environ.get('TSTDB', 'demo')
> >>   TSTHOSTNAME = os.environ.get('TSTHOSTNAME', 'localhost')
> >>   TSTUSERNAME = os.environ.get('TSTUSERNAME', 'monetdb')
> >>   TSTPASSWORD = os.environ.get('TSTPASSWORD', 'monetdb')
> >> +#TSTHOSTNAME = None  # set to none if test a socket
> >> +
> >>   
> >>   if os.environ.get("TSTDEBUG", "no") == "yes":
> >>       logging.basicConfig(level=logging.DEBUG)
> >> diff --git a/clients/python2/test/test_control.py
> >> b/clients/python2/test/test_control.py
> >> --- a/clients/python2/test/test_control.py
> >> +++ b/clients/python2/test/test_control.py
> >> @@ -15,6 +15,7 @@
> >>   # Copyright August 2008-2013 MonetDB B.V.
> >>   # All Rights Reserved.
> >>   
> >> +import os
> >>   import unittest
> >>   import logging
> >>   
> >> @@ -33,6 +34,10 @@ from monetdb.exceptions import Operation
> >>   
> >> _______________________________________________
> >> checkin-list mailing list
> >> checkin-list at monetdb.org
> >> http://mail.monetdb.org/mailman/listinfo/checkin-list
> 
> 

-- 
| Stefan.Manegold at CWI.nl | DB Architectures   (DA) |
| www.CWI.nl/~manegold/  | Science Park 123 (L321) |
| +31 (0)20 592-4212     | 1098 XG Amsterdam  (NL) |




More information about the developers-list mailing list