Hi Gijs,
in case you did not notice this, yet:
This checkin breaks several tests that do test or use the Python interface, resulting in errors like
" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True) File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "
or
" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module> dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect self.socket.connect(unix_socket) FileNotFoundError: [Errno 2] No such file or directory "
See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`
or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default as of tomorrow morning.
Best, Stefan
On Thu, Sep 05, 2013 at 02:19:01PM +0200, Gijs Molenaar wrote:
Changeset: 0eaa07b061be for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0eaa07b061be Modified Files: clients/python2/monetdb/control.py clients/python2/monetdb/mapi.py clients/python2/monetdb/sql/connections.py clients/python2/test/runtests.py clients/python2/test/test_control.py clients/python3/monetdb/control.py clients/python3/monetdb/mapi.py clients/python3/monetdb/sql/connections.py clients/python3/test/runtests.py clients/python3/test/test_control.py Branch: default Log Message:
add support for unix sockets
diffs (truncated from 649 to 300 lines):
diff --git a/clients/python2/monetdb/control.py b/clients/python2/monetdb/control.py --- a/clients/python2/monetdb/control.py +++ b/clients/python2/monetdb/control.py @@ -8,10 +8,12 @@ def parse_statusline(line): parses a sabdb format status line. Support v1 and v2.
"""
- if not line.startswith('=sabdb:'):
- if line.startswith("="):
line = line[1:]
- if not line.startswith('sabdb:'): raise OperationalError('wrong result recieved')
- prot_version, rest = line.split(":", 2)[1:]
code, prot_version, rest = line.split(":", 2)
if prot_version not in ["1", "2"]: raise InterfaceError("unsupported sabdb protocol")
@@ -60,20 +62,26 @@ class Control: Use this module to manage your MonetDB databases. You can create, start, stop, lock, unlock, destroy your databases and request status information. """
- def __init__(self, hostname, port, passphrase):
def __init__(self, hostname=None, port=None, passphrase=None,
unix_socket="/tmp/.s.merovingian.50000"): self.server = mapi.Connection() self.hostname = hostname self.port = port self.passphrase = passphrase
self.unix_socket= unix_socket # check connection
self.server.connect(hostname, port, 'monetdb', passphrase,
'merovingian', 'control')
self.server.connect(hostname=hostname, port=port, username='monetdb',
password=passphrase,
database='merovingian', language='control',
unix_socket=unix_socket) self.server.disconnect()
def _send_command(self, database_name, command):
self.server.connect(self.hostname, self.port, 'monetdb',
self.passphrase, 'merovingian', 'control')
self.server.connect(hostname=self.hostname, port=self.port,
username='monetdb', password=self.passphrase,
database='merovingian', language='control',
unix_socket=self.unix_socket) try: return self.server.cmd("%s %s\n" % (database_name, command)) finally:
@@ -165,8 +173,9 @@ class Control: """ properties = self._send_command(database_name, "get") values = {}
for dirty_line in properties.split("\n"):
line = dirty_line[1:]
for line in properties.split("\n"):
if line.startswith("="):
line = line[1:] if not line.startswith("#"): if "=" in line: split = line.split("=")
diff --git a/clients/python2/monetdb/mapi.py b/clients/python2/monetdb/mapi.py --- a/clients/python2/monetdb/mapi.py +++ b/clients/python2/monetdb/mapi.py @@ -70,8 +70,12 @@ class Connection(object): self.database = "" self.language = ""
- def connect(self, hostname, port, username, password, database, language):
""" setup connection to MAPI server"""
def connect(self, database, username, password, language, hostname=None,
port=None, unix_socket=None):
""" setup connection to MAPI server
unix_socket is used if hostname is not defined.
""" self.hostname = hostname self.port = port
@@ -79,24 +83,34 @@ class Connection(object): self.password = password self.database = database self.language = language
self.unix_socket = unix_socket
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if hostname:
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# For performance, mirror MonetDB/src/common/stream.c socket settings.
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.socket.connect((hostname, port))
else:
self.socket = socket.socket(socket.AF_UNIX)
self.socket.connect(unix_socket)
if self.language != 'control':
self.socket.send('0') # don't know why, but we need to do this
# For performance, mirror MonetDB/src/common/stream.c socket settings.
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
if not (self.language == 'control' and not self.hostname):
# control doesn't require authentication over socket
self._login()
self.socket.connect((hostname, port))
self.__login()
self.state = STATE_READY
- def __login(self, iteration=0):
- def _login(self, iteration=0): """ Reads challenge from line, generate response and check if everything is okay """
challenge = self.__getblock()
response = self.__challenge_response(challenge)
self.__putblock(response)
prompt = self.__getblock().strip()
challenge = self._getblock()
response = self._challenge_response(challenge)
self._putblock(response)
prompt = self._getblock().strip() if len(prompt) == 0: # Empty response, server is happy
@@ -117,7 +131,7 @@ class Connection(object): if redirect[1] == "merovingian": logger.debug("restarting authentication") if iteration <= 10:
self.__login(iteration=iteration + 1)
self._login(iteration=iteration + 1) else: raise OperationalError("maximal number of redirects " "reached (10)")
@@ -138,9 +152,6 @@ class Connection(object): else: raise ProgrammingError("unknown state: %s" % prompt)
self.state = STATE_READY
return True
- def disconnect(self): """ disconnect from the monetdb server """ self.state = STATE_INIT
@@ -153,8 +164,8 @@ class Connection(object): if self.state != STATE_READY: raise(ProgrammingError, "Not connected")
self.__putblock(operation)
response = self.__getblock()
self._putblock(operation)
response = self._getblock() if not len(response): return "" elif response.startswith(MSG_OK):
@@ -166,10 +177,16 @@ class Connection(object): return response elif response[0] == MSG_ERROR: raise OperationalError(response[1:])
elif (self.language == 'control' and not self.hostname):
if response.startswith("OK"):
return response[2:].strip() or ""
else:
return response else: raise ProgrammingError("unknown state: %s" % response)
- def __challenge_response(self, challenge):
- def _challenge_response(self, challenge): """ generate a response to a mapi login challenge """ challenges = challenge.split(':') salt, identity, protocol, hashes, endian = challenges[:5]
@@ -204,19 +221,36 @@ class Connection(object): return ":".join(["BIG", self.username, pwhash, self.language, self.database]) + ":"
- def __getblock(self):
- def _getblock(self): """ read one mapi encoded block """
if (self.language == 'control' and not self.hostname):
return self._getblock_socket() # control doesn't do block
# splitting when using a socket
else:
return self._getblock_inet()
- def _getblock_inet(self): result = StringIO() last = 0 while not last:
flag = self.__getbytes(2)
flag = self._getbytes(2) unpacked = struct.unpack('<H', flag)[0] # little endian short length = unpacked >> 1 last = unpacked & 1
result.write(self.__getbytes(length))
result.write(self._getbytes(length)) return result.getvalue()
- def __getbytes(self, bytes_):
- def _getblock_socket(self):
buffer = StringIO()
while True:
x = self.socket.recv(1)
if len(x):
buffer.write(x)
else:
break
return buffer.getvalue().strip()
- def _getbytes(self, bytes_): """Read an amount of bytes from the socket""" result = StringIO() count = bytes_
@@ -228,8 +262,15 @@ class Connection(object): result.write(recv) return result.getvalue()
- def __putblock(self, block):
- def _putblock(self, block): """ wrap the line in mapi format and put it into the socket """
if (self.language == 'control' and not self.hostname):
return self.socket.send(block) # control doesn't do block
# splitting when using a socket
else:
self._putblock_inet(block)
- def _putblock_inet(self, block): pos = 0 last = 0 while not last:
diff --git a/clients/python2/monetdb/sql/connections.py b/clients/python2/monetdb/sql/connections.py --- a/clients/python2/monetdb/sql/connections.py +++ b/clients/python2/monetdb/sql/connections.py @@ -28,25 +28,25 @@ class Connection(object): """A MonetDB SQL database connection""" default_cursor = cursors.Cursor
- def __init__(self, username="monetdb", password="monetdb",
hostname="localhost", port=50000, database="demo",
autocommit=False, user=None, host=None):
- def __init__(self, database, hostname=None, port=50000, username="monetdb",
password="monetdb", unix_socket="/tmp/.s.monetdb.50000",
autocommit=False): """ Set up a connection to a MonetDB SQL database.
username -- username for connection (default: monetdb)
password -- password for connection (default: monetdb)
hostname -- hostname to connect to (default: localhost)
port -- port to connect to (default: 50000)
database -- name of the database (default: demo)
autocommit -- enable/disable auto commit (default: False)
database -- name of the database
hostname -- Hostname where monetDB is running
port -- port to connect to (default: 50000)
username -- username for connection (default: "monetdb")
password -- password for connection (default: "monetdb")
unix_socket -- socket to connect to. used when hostname not set
(default: "/tmp/.s.monetdb.50000")
autocommit -- enable/disable auto commit (default: False)
"""
if user is not None:
username = user
if host is not None:
hostname = host self.mapi = mapi.Connection() self.mapi.connect(hostname=hostname, port=int(port), username=username,
password=password, database=database, language="sql")
password=password, database=database, language="sql",
unix_socket=unix_socket) self.set_autocommit(autocommit) self.set_sizeheader(True) self.set_replysize(100)
diff --git a/clients/python2/test/runtests.py b/clients/python2/test/runtests.py --- a/clients/python2/test/runtests.py +++ b/clients/python2/test/runtests.py @@ -45,6 +45,8 @@ TSTDB = os.environ.get('TSTDB', 'demo') TSTHOSTNAME = os.environ.get('TSTHOSTNAME', 'localhost') TSTUSERNAME = os.environ.get('TSTUSERNAME', 'monetdb') TSTPASSWORD = os.environ.get('TSTPASSWORD', 'monetdb') +#TSTHOSTNAME = None # set to none if test a socket
if os.environ.get("TSTDEBUG", "no") == "yes": logging.basicConfig(level=logging.DEBUG) diff --git a/clients/python2/test/test_control.py b/clients/python2/test/test_control.py --- a/clients/python2/test/test_control.py +++ b/clients/python2/test/test_control.py @@ -15,6 +15,7 @@ # Copyright August 2008-2013 MonetDB B.V. # All Rights Reserved.
+import os import unittest import logging
@@ -33,6 +34,10 @@ from monetdb.exceptions import Operation
checkin-list mailing list checkin-list@monetdb.org http://mail.monetdb.org/mailman/listinfo/checkin-list