MonetDB: default - add support for unix sockets

Gijs Molenaar g.j.molenaar at uva.nl
Thu Sep 5 20:21:03 CEST 2013


ok, i'll fix it tomorrow.


On 09/05/2013 06:56 PM, Stefan Manegold wrote:
> Hi Gijs,
>
> in case you did not notice this, yet:
>
> This checkin breaks several tests that do test or use the Python interface,
> resulting in errors like
>
> "
> Traceback (most recent call last):
>    File "delete_all.SQL.py", line 9, in <module>
>      dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True)
>    File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__
>      unix_socket=unix_socket)
>    File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect
>      self.socket.connect(unix_socket)
>    File "/usr/lib64/python2.7/socket.py", line 224, in meth
>      return getattr(self._sock,name)(*args)
> socket.error: [Errno 2] No such file or directory
> "
>
> or
>
> "
> Traceback (most recent call last):
>    File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module>
>      dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True)
>    File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__
>      unix_socket=unix_socket)
>    File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect
>      self.socket.connect(unix_socket)
> FileNotFoundError: [Errno 2] No such file or directory
> "
>
>
> See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`
>
> or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default
> as of tomorrow morning.
>
>
> Best,
> Stefan
>
>
> On Thu, Sep 05, 2013 at 02:19:01PM +0200, Gijs Molenaar wrote:
>> Changeset: 0eaa07b061be for MonetDB
>> URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0eaa07b061be
>> Modified Files:
>> 	clients/python2/monetdb/control.py
>> 	clients/python2/monetdb/mapi.py
>> 	clients/python2/monetdb/sql/connections.py
>> 	clients/python2/test/runtests.py
>> 	clients/python2/test/test_control.py
>> 	clients/python3/monetdb/control.py
>> 	clients/python3/monetdb/mapi.py
>> 	clients/python3/monetdb/sql/connections.py
>> 	clients/python3/test/runtests.py
>> 	clients/python3/test/test_control.py
>> Branch: default
>> Log Message:
>>
>> add support for unix sockets
>>
>>
>> diffs (truncated from 649 to 300 lines):
>>
>> diff --git a/clients/python2/monetdb/control.py b/clients/python2/monetdb/control.py
>> --- a/clients/python2/monetdb/control.py
>> +++ b/clients/python2/monetdb/control.py
>> @@ -8,10 +8,12 @@ def parse_statusline(line):
>>       parses a sabdb format status line. Support v1 and v2.
>>   
>>       """
>> -    if not line.startswith('=sabdb:'):
>> +    if line.startswith("="):
>> +        line = line[1:]
>> +    if not line.startswith('sabdb:'):
>>           raise OperationalError('wrong result recieved')
>>   
>> -    prot_version, rest = line.split(":", 2)[1:]
>> +    code, prot_version, rest = line.split(":", 2)
>>   
>>       if prot_version not in ["1", "2"]:
>>           raise InterfaceError("unsupported sabdb protocol")
>> @@ -60,20 +62,26 @@ class Control:
>>       Use this module to manage your MonetDB databases. You can create, start,
>>       stop, lock, unlock, destroy your databases and request status information.
>>       """
>> -    def __init__(self, hostname, port, passphrase):
>> +    def __init__(self, hostname=None, port=None, passphrase=None,
>> +                 unix_socket="/tmp/.s.merovingian.50000"):
>>           self.server = mapi.Connection()
>>           self.hostname = hostname
>>           self.port = port
>>           self.passphrase = passphrase
>> +        self.unix_socket= unix_socket
>>   
>>           # check connection
>> -        self.server.connect(hostname, port, 'monetdb', passphrase,
>> -                            'merovingian', 'control')
>> +        self.server.connect(hostname=hostname, port=port, username='monetdb',
>> +                            password=passphrase,
>> +                            database='merovingian', language='control',
>> +                            unix_socket=unix_socket)
>>           self.server.disconnect()
>>   
>>       def _send_command(self, database_name, command):
>> -        self.server.connect(self.hostname, self.port, 'monetdb',
>> -                            self.passphrase, 'merovingian', 'control')
>> +        self.server.connect(hostname=self.hostname, port=self.port,
>> +                            username='monetdb', password=self.passphrase,
>> +                            database='merovingian', language='control',
>> +                            unix_socket=self.unix_socket)
>>           try:
>>               return self.server.cmd("%s %s\n" % (database_name, command))
>>           finally:
>> @@ -165,8 +173,9 @@ class Control:
>>           """
>>           properties = self._send_command(database_name, "get")
>>           values = {}
>> -        for dirty_line in properties.split("\n"):
>> -            line = dirty_line[1:]
>> +        for line in properties.split("\n"):
>> +            if line.startswith("="):
>> +                line = line[1:]
>>               if not line.startswith("#"):
>>                   if "=" in line:
>>                       split = line.split("=")
>> diff --git a/clients/python2/monetdb/mapi.py b/clients/python2/monetdb/mapi.py
>> --- a/clients/python2/monetdb/mapi.py
>> +++ b/clients/python2/monetdb/mapi.py
>> @@ -70,8 +70,12 @@ class Connection(object):
>>           self.database = ""
>>           self.language = ""
>>   
>> -    def connect(self, hostname, port, username, password, database, language):
>> -        """ setup connection to MAPI server"""
>> +    def connect(self, database, username, password, language, hostname=None,
>> +                port=None, unix_socket=None):
>> +        """ setup connection to MAPI server
>> +
>> +        unix_socket is used if hostname is not defined.
>> +        """
>>   
>>           self.hostname = hostname
>>           self.port = port
>> @@ -79,24 +83,34 @@ class Connection(object):
>>           self.password = password
>>           self.database = database
>>           self.language = language
>> +        self.unix_socket = unix_socket
>>   
>> -        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
>> +        if hostname:
>> +            self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
>> +            # For performance, mirror MonetDB/src/common/stream.c socket settings.
>> +            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)
>> +            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
>> +            self.socket.connect((hostname, port))
>> +        else:
>> +            self.socket = socket.socket(socket.AF_UNIX)
>> +            self.socket.connect(unix_socket)
>> +            if self.language != 'control':
>> +                self.socket.send('0')  # don't know why, but we need to do this
>>   
>> -        # For performance, mirror MonetDB/src/common/stream.c socket settings.
>> -        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)
>> -        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
>> +        if not (self.language == 'control' and not self.hostname):
>> +            # control doesn't require authentication over socket
>> +            self._login()
>>   
>> -        self.socket.connect((hostname, port))
>> -        self.__login()
>> +        self.state = STATE_READY
>>   
>> -    def __login(self, iteration=0):
>> +    def _login(self, iteration=0):
>>           """ Reads challenge from line, generate response and check if
>>           everything is okay """
>>   
>> -        challenge = self.__getblock()
>> -        response = self.__challenge_response(challenge)
>> -        self.__putblock(response)
>> -        prompt = self.__getblock().strip()
>> +        challenge = self._getblock()
>> +        response = self._challenge_response(challenge)
>> +        self._putblock(response)
>> +        prompt = self._getblock().strip()
>>   
>>           if len(prompt) == 0:
>>               # Empty response, server is happy
>> @@ -117,7 +131,7 @@ class Connection(object):
>>               if redirect[1] == "merovingian":
>>                   logger.debug("restarting authentication")
>>                   if iteration <= 10:
>> -                    self.__login(iteration=iteration + 1)
>> +                    self._login(iteration=iteration + 1)
>>                   else:
>>                       raise OperationalError("maximal number of redirects "
>>                                              "reached (10)")
>> @@ -138,9 +152,6 @@ class Connection(object):
>>           else:
>>               raise ProgrammingError("unknown state: %s" % prompt)
>>   
>> -        self.state = STATE_READY
>> -        return True
>> -
>>       def disconnect(self):
>>           """ disconnect from the monetdb server """
>>           self.state = STATE_INIT
>> @@ -153,8 +164,8 @@ class Connection(object):
>>           if self.state != STATE_READY:
>>               raise(ProgrammingError, "Not connected")
>>   
>> -        self.__putblock(operation)
>> -        response = self.__getblock()
>> +        self._putblock(operation)
>> +        response = self._getblock()
>>           if not len(response):
>>               return ""
>>           elif response.startswith(MSG_OK):
>> @@ -166,10 +177,16 @@ class Connection(object):
>>               return response
>>           elif response[0] == MSG_ERROR:
>>               raise OperationalError(response[1:])
>> +        elif (self.language == 'control' and not self.hostname):
>> +            if response.startswith("OK"):
>> +                return response[2:].strip() or ""
>> +            else:
>> +                return response
>>           else:
>>               raise ProgrammingError("unknown state: %s" % response)
>>   
>> -    def __challenge_response(self, challenge):
>> +
>> +    def _challenge_response(self, challenge):
>>           """ generate a response to a mapi login challenge """
>>           challenges = challenge.split(':')
>>           salt, identity, protocol, hashes, endian = challenges[:5]
>> @@ -204,19 +221,36 @@ class Connection(object):
>>           return ":".join(["BIG", self.username, pwhash, self.language,
>>                            self.database]) + ":"
>>   
>> -    def __getblock(self):
>> +    def _getblock(self):
>>           """ read one mapi encoded block """
>> +        if (self.language == 'control' and not self.hostname):
>> +            return self._getblock_socket()  # control doesn't do block
>> +                                            # splitting when using a socket
>> +        else:
>> +            return self._getblock_inet()
>> +
>> +    def _getblock_inet(self):
>>           result = StringIO()
>>           last = 0
>>           while not last:
>> -            flag = self.__getbytes(2)
>> +            flag = self._getbytes(2)
>>               unpacked = struct.unpack('<H', flag)[0]  # little endian short
>>               length = unpacked >> 1
>>               last = unpacked & 1
>> -            result.write(self.__getbytes(length))
>> +            result.write(self._getbytes(length))
>>           return result.getvalue()
>>   
>> -    def __getbytes(self, bytes_):
>> +    def _getblock_socket(self):
>> +        buffer = StringIO()
>> +        while True:
>> +            x = self.socket.recv(1)
>> +            if len(x):
>> +                buffer.write(x)
>> +            else:
>> +                break
>> +        return buffer.getvalue().strip()
>> +
>> +    def _getbytes(self, bytes_):
>>           """Read an amount of bytes from the socket"""
>>           result = StringIO()
>>           count = bytes_
>> @@ -228,8 +262,15 @@ class Connection(object):
>>               result.write(recv)
>>           return result.getvalue()
>>   
>> -    def __putblock(self, block):
>> +    def _putblock(self, block):
>>           """ wrap the line in mapi format and put it into the socket """
>> +        if (self.language == 'control' and not self.hostname):
>> +            return self.socket.send(block)  # control doesn't do block
>> +                                            # splitting when using a socket
>> +        else:
>> +            self._putblock_inet(block)
>> +
>> +    def _putblock_inet(self, block):
>>           pos = 0
>>           last = 0
>>           while not last:
>> diff --git a/clients/python2/monetdb/sql/connections.py b/clients/python2/monetdb/sql/connections.py
>> --- a/clients/python2/monetdb/sql/connections.py
>> +++ b/clients/python2/monetdb/sql/connections.py
>> @@ -28,25 +28,25 @@ class Connection(object):
>>       """A MonetDB SQL database connection"""
>>       default_cursor = cursors.Cursor
>>   
>> -    def __init__(self, username="monetdb", password="monetdb",
>> -                 hostname="localhost", port=50000, database="demo",
>> -                 autocommit=False, user=None, host=None):
>> +    def __init__(self, database, hostname=None, port=50000, username="monetdb",
>> +                 password="monetdb", unix_socket="/tmp/.s.monetdb.50000",
>> +                 autocommit=False):
>>           """ Set up a connection to a MonetDB SQL database.
>>   
>> -        username   -- username for connection (default: monetdb)
>> -        password   -- password for connection (default: monetdb)
>> -        hostname   -- hostname to connect to (default: localhost)
>> -        port       -- port to connect to (default: 50000)
>> -        database   -- name of the database (default: demo)
>> -        autocommit -- enable/disable auto commit (default: False)
>> +        database    -- name of the database
>> +        hostname    -- Hostname where monetDB is running
>> +        port        -- port to connect to (default: 50000)
>> +        username    -- username for connection (default: "monetdb")
>> +        password    -- password for connection (default: "monetdb")
>> +        unix_socket -- socket to connect to. used when hostname not set
>> +                            (default: "/tmp/.s.monetdb.50000")
>> +        autocommit  -- enable/disable auto commit (default: False)
>> +
>>           """
>> -        if user is not None:
>> -            username = user
>> -        if host is not None:
>> -            hostname = host
>>           self.mapi = mapi.Connection()
>>           self.mapi.connect(hostname=hostname, port=int(port), username=username,
>> -                          password=password, database=database, language="sql")
>> +                          password=password, database=database, language="sql",
>> +                          unix_socket=unix_socket)
>>           self.set_autocommit(autocommit)
>>           self.set_sizeheader(True)
>>           self.set_replysize(100)
>> diff --git a/clients/python2/test/runtests.py b/clients/python2/test/runtests.py
>> --- a/clients/python2/test/runtests.py
>> +++ b/clients/python2/test/runtests.py
>> @@ -45,6 +45,8 @@ TSTDB = os.environ.get('TSTDB', 'demo')
>>   TSTHOSTNAME = os.environ.get('TSTHOSTNAME', 'localhost')
>>   TSTUSERNAME = os.environ.get('TSTUSERNAME', 'monetdb')
>>   TSTPASSWORD = os.environ.get('TSTPASSWORD', 'monetdb')
>> +#TSTHOSTNAME = None  # set to none if test a socket
>> +
>>   
>>   if os.environ.get("TSTDEBUG", "no") == "yes":
>>       logging.basicConfig(level=logging.DEBUG)
>> diff --git a/clients/python2/test/test_control.py b/clients/python2/test/test_control.py
>> --- a/clients/python2/test/test_control.py
>> +++ b/clients/python2/test/test_control.py
>> @@ -15,6 +15,7 @@
>>   # Copyright August 2008-2013 MonetDB B.V.
>>   # All Rights Reserved.
>>   
>> +import os
>>   import unittest
>>   import logging
>>   
>> @@ -33,6 +34,10 @@ from monetdb.exceptions import Operation
>>   
>> _______________________________________________
>> checkin-list mailing list
>> checkin-list at monetdb.org
>> http://mail.monetdb.org/mailman/listinfo/checkin-list




More information about the developers-list mailing list