MonetDB: default - add support for unix sockets

Stefan Manegold Stefan.Manegold at cwi.nl
Thu Sep 5 18:56:41 CEST 2013


Hi Gijs,

in case you did not notice this, yet:

This checkin breaks several tests that do test or use the Python interface,
resulting in errors like

"
Traceback (most recent call last):
  File "delete_all.SQL.py", line 9, in <module>
    dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True)
  File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__
    unix_socket=unix_socket)
  File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect
    self.socket.connect(unix_socket)
  File "/usr/lib64/python2.7/socket.py", line 224, in meth
    return getattr(self._sock,name)(*args)
socket.error: [Errno 2] No such file or directory
"

or

"
Traceback (most recent call last):
  File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module>
    dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True)
  File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__
    unix_socket=unix_socket)
  File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect
    self.socket.connect(unix_socket)
FileNotFoundError: [Errno 2] No such file or directory
"


See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`

or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default
as of tomorrow morning.


Best,
Stefan


On Thu, Sep 05, 2013 at 02:19:01PM +0200, Gijs Molenaar wrote:
> Changeset: 0eaa07b061be for MonetDB
> URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0eaa07b061be
> Modified Files:
> 	clients/python2/monetdb/control.py
> 	clients/python2/monetdb/mapi.py
> 	clients/python2/monetdb/sql/connections.py
> 	clients/python2/test/runtests.py
> 	clients/python2/test/test_control.py
> 	clients/python3/monetdb/control.py
> 	clients/python3/monetdb/mapi.py
> 	clients/python3/monetdb/sql/connections.py
> 	clients/python3/test/runtests.py
> 	clients/python3/test/test_control.py
> Branch: default
> Log Message:
> 
> add support for unix sockets
> 
> 
> diffs (truncated from 649 to 300 lines):
> 
> diff --git a/clients/python2/monetdb/control.py b/clients/python2/monetdb/control.py
> --- a/clients/python2/monetdb/control.py
> +++ b/clients/python2/monetdb/control.py
> @@ -8,10 +8,12 @@ def parse_statusline(line):
>      parses a sabdb format status line. Support v1 and v2.
>  
>      """
> -    if not line.startswith('=sabdb:'):
> +    if line.startswith("="):
> +        line = line[1:]
> +    if not line.startswith('sabdb:'):
>          raise OperationalError('wrong result recieved')
>  
> -    prot_version, rest = line.split(":", 2)[1:]
> +    code, prot_version, rest = line.split(":", 2)
>  
>      if prot_version not in ["1", "2"]:
>          raise InterfaceError("unsupported sabdb protocol")
> @@ -60,20 +62,26 @@ class Control:
>      Use this module to manage your MonetDB databases. You can create, start,
>      stop, lock, unlock, destroy your databases and request status information.
>      """
> -    def __init__(self, hostname, port, passphrase):
> +    def __init__(self, hostname=None, port=None, passphrase=None,
> +                 unix_socket="/tmp/.s.merovingian.50000"):
>          self.server = mapi.Connection()
>          self.hostname = hostname
>          self.port = port
>          self.passphrase = passphrase
> +        self.unix_socket= unix_socket
>  
>          # check connection
> -        self.server.connect(hostname, port, 'monetdb', passphrase,
> -                            'merovingian', 'control')
> +        self.server.connect(hostname=hostname, port=port, username='monetdb',
> +                            password=passphrase,
> +                            database='merovingian', language='control',
> +                            unix_socket=unix_socket)
>          self.server.disconnect()
>  
>      def _send_command(self, database_name, command):
> -        self.server.connect(self.hostname, self.port, 'monetdb',
> -                            self.passphrase, 'merovingian', 'control')
> +        self.server.connect(hostname=self.hostname, port=self.port,
> +                            username='monetdb', password=self.passphrase,
> +                            database='merovingian', language='control',
> +                            unix_socket=self.unix_socket)
>          try:
>              return self.server.cmd("%s %s\n" % (database_name, command))
>          finally:
> @@ -165,8 +173,9 @@ class Control:
>          """
>          properties = self._send_command(database_name, "get")
>          values = {}
> -        for dirty_line in properties.split("\n"):
> -            line = dirty_line[1:]
> +        for line in properties.split("\n"):
> +            if line.startswith("="):
> +                line = line[1:]
>              if not line.startswith("#"):
>                  if "=" in line:
>                      split = line.split("=")
> diff --git a/clients/python2/monetdb/mapi.py b/clients/python2/monetdb/mapi.py
> --- a/clients/python2/monetdb/mapi.py
> +++ b/clients/python2/monetdb/mapi.py
> @@ -70,8 +70,12 @@ class Connection(object):
>          self.database = ""
>          self.language = ""
>  
> -    def connect(self, hostname, port, username, password, database, language):
> -        """ setup connection to MAPI server"""
> +    def connect(self, database, username, password, language, hostname=None,
> +                port=None, unix_socket=None):
> +        """ setup connection to MAPI server
> +
> +        unix_socket is used if hostname is not defined.
> +        """
>  
>          self.hostname = hostname
>          self.port = port
> @@ -79,24 +83,34 @@ class Connection(object):
>          self.password = password
>          self.database = database
>          self.language = language
> +        self.unix_socket = unix_socket
>  
> -        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
> +        if hostname:
> +            self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
> +            # For performance, mirror MonetDB/src/common/stream.c socket settings.
> +            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)
> +            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
> +            self.socket.connect((hostname, port))
> +        else:
> +            self.socket = socket.socket(socket.AF_UNIX)
> +            self.socket.connect(unix_socket)
> +            if self.language != 'control':
> +                self.socket.send('0')  # don't know why, but we need to do this
>  
> -        # For performance, mirror MonetDB/src/common/stream.c socket settings.
> -        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)
> -        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
> +        if not (self.language == 'control' and not self.hostname):
> +            # control doesn't require authentication over socket
> +            self._login()
>  
> -        self.socket.connect((hostname, port))
> -        self.__login()
> +        self.state = STATE_READY
>  
> -    def __login(self, iteration=0):
> +    def _login(self, iteration=0):
>          """ Reads challenge from line, generate response and check if
>          everything is okay """
>  
> -        challenge = self.__getblock()
> -        response = self.__challenge_response(challenge)
> -        self.__putblock(response)
> -        prompt = self.__getblock().strip()
> +        challenge = self._getblock()
> +        response = self._challenge_response(challenge)
> +        self._putblock(response)
> +        prompt = self._getblock().strip()
>  
>          if len(prompt) == 0:
>              # Empty response, server is happy
> @@ -117,7 +131,7 @@ class Connection(object):
>              if redirect[1] == "merovingian":
>                  logger.debug("restarting authentication")
>                  if iteration <= 10:
> -                    self.__login(iteration=iteration + 1)
> +                    self._login(iteration=iteration + 1)
>                  else:
>                      raise OperationalError("maximal number of redirects "
>                                             "reached (10)")
> @@ -138,9 +152,6 @@ class Connection(object):
>          else:
>              raise ProgrammingError("unknown state: %s" % prompt)
>  
> -        self.state = STATE_READY
> -        return True
> -
>      def disconnect(self):
>          """ disconnect from the monetdb server """
>          self.state = STATE_INIT
> @@ -153,8 +164,8 @@ class Connection(object):
>          if self.state != STATE_READY:
>              raise(ProgrammingError, "Not connected")
>  
> -        self.__putblock(operation)
> -        response = self.__getblock()
> +        self._putblock(operation)
> +        response = self._getblock()
>          if not len(response):
>              return ""
>          elif response.startswith(MSG_OK):
> @@ -166,10 +177,16 @@ class Connection(object):
>              return response
>          elif response[0] == MSG_ERROR:
>              raise OperationalError(response[1:])
> +        elif (self.language == 'control' and not self.hostname):
> +            if response.startswith("OK"):
> +                return response[2:].strip() or ""
> +            else:
> +                return response
>          else:
>              raise ProgrammingError("unknown state: %s" % response)
>  
> -    def __challenge_response(self, challenge):
> +
> +    def _challenge_response(self, challenge):
>          """ generate a response to a mapi login challenge """
>          challenges = challenge.split(':')
>          salt, identity, protocol, hashes, endian = challenges[:5]
> @@ -204,19 +221,36 @@ class Connection(object):
>          return ":".join(["BIG", self.username, pwhash, self.language,
>                           self.database]) + ":"
>  
> -    def __getblock(self):
> +    def _getblock(self):
>          """ read one mapi encoded block """
> +        if (self.language == 'control' and not self.hostname):
> +            return self._getblock_socket()  # control doesn't do block
> +                                            # splitting when using a socket
> +        else:
> +            return self._getblock_inet()
> +
> +    def _getblock_inet(self):
>          result = StringIO()
>          last = 0
>          while not last:
> -            flag = self.__getbytes(2)
> +            flag = self._getbytes(2)
>              unpacked = struct.unpack('<H', flag)[0]  # little endian short
>              length = unpacked >> 1
>              last = unpacked & 1
> -            result.write(self.__getbytes(length))
> +            result.write(self._getbytes(length))
>          return result.getvalue()
>  
> -    def __getbytes(self, bytes_):
> +    def _getblock_socket(self):
> +        buffer = StringIO()
> +        while True:
> +            x = self.socket.recv(1)
> +            if len(x):
> +                buffer.write(x)
> +            else:
> +                break
> +        return buffer.getvalue().strip()
> +
> +    def _getbytes(self, bytes_):
>          """Read an amount of bytes from the socket"""
>          result = StringIO()
>          count = bytes_
> @@ -228,8 +262,15 @@ class Connection(object):
>              result.write(recv)
>          return result.getvalue()
>  
> -    def __putblock(self, block):
> +    def _putblock(self, block):
>          """ wrap the line in mapi format and put it into the socket """
> +        if (self.language == 'control' and not self.hostname):
> +            return self.socket.send(block)  # control doesn't do block
> +                                            # splitting when using a socket
> +        else:
> +            self._putblock_inet(block)
> +
> +    def _putblock_inet(self, block):
>          pos = 0
>          last = 0
>          while not last:
> diff --git a/clients/python2/monetdb/sql/connections.py b/clients/python2/monetdb/sql/connections.py
> --- a/clients/python2/monetdb/sql/connections.py
> +++ b/clients/python2/monetdb/sql/connections.py
> @@ -28,25 +28,25 @@ class Connection(object):
>      """A MonetDB SQL database connection"""
>      default_cursor = cursors.Cursor
>  
> -    def __init__(self, username="monetdb", password="monetdb",
> -                 hostname="localhost", port=50000, database="demo",
> -                 autocommit=False, user=None, host=None):
> +    def __init__(self, database, hostname=None, port=50000, username="monetdb",
> +                 password="monetdb", unix_socket="/tmp/.s.monetdb.50000",
> +                 autocommit=False):
>          """ Set up a connection to a MonetDB SQL database.
>  
> -        username   -- username for connection (default: monetdb)
> -        password   -- password for connection (default: monetdb)
> -        hostname   -- hostname to connect to (default: localhost)
> -        port       -- port to connect to (default: 50000)
> -        database   -- name of the database (default: demo)
> -        autocommit -- enable/disable auto commit (default: False)
> +        database    -- name of the database
> +        hostname    -- Hostname where monetDB is running
> +        port        -- port to connect to (default: 50000)
> +        username    -- username for connection (default: "monetdb")
> +        password    -- password for connection (default: "monetdb")
> +        unix_socket -- socket to connect to. used when hostname not set
> +                            (default: "/tmp/.s.monetdb.50000")
> +        autocommit  -- enable/disable auto commit (default: False)
> +
>          """
> -        if user is not None:
> -            username = user
> -        if host is not None:
> -            hostname = host
>          self.mapi = mapi.Connection()
>          self.mapi.connect(hostname=hostname, port=int(port), username=username,
> -                          password=password, database=database, language="sql")
> +                          password=password, database=database, language="sql",
> +                          unix_socket=unix_socket)
>          self.set_autocommit(autocommit)
>          self.set_sizeheader(True)
>          self.set_replysize(100)
> diff --git a/clients/python2/test/runtests.py b/clients/python2/test/runtests.py
> --- a/clients/python2/test/runtests.py
> +++ b/clients/python2/test/runtests.py
> @@ -45,6 +45,8 @@ TSTDB = os.environ.get('TSTDB', 'demo')
>  TSTHOSTNAME = os.environ.get('TSTHOSTNAME', 'localhost')
>  TSTUSERNAME = os.environ.get('TSTUSERNAME', 'monetdb')
>  TSTPASSWORD = os.environ.get('TSTPASSWORD', 'monetdb')
> +#TSTHOSTNAME = None  # set to none if test a socket
> +
>  
>  if os.environ.get("TSTDEBUG", "no") == "yes":
>      logging.basicConfig(level=logging.DEBUG)
> diff --git a/clients/python2/test/test_control.py b/clients/python2/test/test_control.py
> --- a/clients/python2/test/test_control.py
> +++ b/clients/python2/test/test_control.py
> @@ -15,6 +15,7 @@
>  # Copyright August 2008-2013 MonetDB B.V.
>  # All Rights Reserved.
>  
> +import os
>  import unittest
>  import logging
>  
> @@ -33,6 +34,10 @@ from monetdb.exceptions import Operation
>  
> _______________________________________________
> checkin-list mailing list
> checkin-list at monetdb.org
> http://mail.monetdb.org/mailman/listinfo/checkin-list

-- 
| Stefan.Manegold at CWI.nl | DB Architectures   (DA) |
|  www.CWI.nl/~manegold  | Science Park 123 (L321) |
|   +31 (0)20 592-4212   | 1098 XG Amsterdam  (NL) |



More information about the developers-list mailing list