Re: MonetDB: default - add support for unix sockets
Hi Gijs,
in case you did not notice this, yet:
This checkin breaks several tests that do test or use the Python interface, resulting in errors like
" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True) File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "
or
" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module> dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect self.socket.connect(unix_socket) FileNotFoundError: [Errno 2] No such file or directory "
See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`
or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default as of tomorrow morning.
Best, Stefan
On Thu, Sep 05, 2013 at 02:19:01PM +0200, Gijs Molenaar wrote:
Changeset: 0eaa07b061be for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0eaa07b061be Modified Files: clients/python2/monetdb/control.py clients/python2/monetdb/mapi.py clients/python2/monetdb/sql/connections.py clients/python2/test/runtests.py clients/python2/test/test_control.py clients/python3/monetdb/control.py clients/python3/monetdb/mapi.py clients/python3/monetdb/sql/connections.py clients/python3/test/runtests.py clients/python3/test/test_control.py Branch: default Log Message:
add support for unix sockets
diffs (truncated from 649 to 300 lines):
diff --git a/clients/python2/monetdb/control.py b/clients/python2/monetdb/control.py --- a/clients/python2/monetdb/control.py +++ b/clients/python2/monetdb/control.py @@ -8,10 +8,12 @@ def parse_statusline(line): parses a sabdb format status line. Support v1 and v2.
"""
- if not line.startswith('=sabdb:'):
- if line.startswith("="):
line = line[1:]- if not line.startswith('sabdb:'): raise OperationalError('wrong result recieved')
- prot_version, rest = line.split(":", 2)[1:]
code, prot_version, rest = line.split(":", 2)
if prot_version not in ["1", "2"]: raise InterfaceError("unsupported sabdb protocol")
@@ -60,20 +62,26 @@ class Control: Use this module to manage your MonetDB databases. You can create, start, stop, lock, unlock, destroy your databases and request status information. """
- def __init__(self, hostname, port, passphrase):
def __init__(self, hostname=None, port=None, passphrase=None,
unix_socket="/tmp/.s.merovingian.50000"): self.server = mapi.Connection() self.hostname = hostname self.port = port self.passphrase = passphraseself.unix_socket= unix_socket # check connection
self.server.connect(hostname, port, 'monetdb', passphrase,'merovingian', 'control')
self.server.connect(hostname=hostname, port=port, username='monetdb',password=passphrase,database='merovingian', language='control',unix_socket=unix_socket) self.server.disconnect()def _send_command(self, database_name, command):
self.server.connect(self.hostname, self.port, 'monetdb',self.passphrase, 'merovingian', 'control')
self.server.connect(hostname=self.hostname, port=self.port,username='monetdb', password=self.passphrase,database='merovingian', language='control',unix_socket=self.unix_socket) try: return self.server.cmd("%s %s\n" % (database_name, command)) finally:@@ -165,8 +173,9 @@ class Control: """ properties = self._send_command(database_name, "get") values = {}
for dirty_line in properties.split("\n"):line = dirty_line[1:]
for line in properties.split("\n"):if line.startswith("="):line = line[1:] if not line.startswith("#"): if "=" in line: split = line.split("=")diff --git a/clients/python2/monetdb/mapi.py b/clients/python2/monetdb/mapi.py --- a/clients/python2/monetdb/mapi.py +++ b/clients/python2/monetdb/mapi.py @@ -70,8 +70,12 @@ class Connection(object): self.database = "" self.language = ""
- def connect(self, hostname, port, username, password, database, language):
""" setup connection to MAPI server"""
def connect(self, database, username, password, language, hostname=None,
port=None, unix_socket=None):""" setup connection to MAPI serverunix_socket is used if hostname is not defined.""" self.hostname = hostname self.port = port@@ -79,24 +83,34 @@ class Connection(object): self.password = password self.database = database self.language = language
self.unix_socket = unix_socket
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if hostname:self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)# For performance, mirror MonetDB/src/common/stream.c socket settings.self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)self.socket.connect((hostname, port))else:self.socket = socket.socket(socket.AF_UNIX)self.socket.connect(unix_socket)if self.language != 'control':self.socket.send('0') # don't know why, but we need to do this
# For performance, mirror MonetDB/src/common/stream.c socket settings.self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
if not (self.language == 'control' and not self.hostname):# control doesn't require authentication over socketself._login()
self.socket.connect((hostname, port))self.__login()
self.state = STATE_READY
- def __login(self, iteration=0):
- def _login(self, iteration=0): """ Reads challenge from line, generate response and check if everything is okay """
challenge = self.__getblock()response = self.__challenge_response(challenge)self.__putblock(response)prompt = self.__getblock().strip()
challenge = self._getblock()response = self._challenge_response(challenge)self._putblock(response)prompt = self._getblock().strip() if len(prompt) == 0: # Empty response, server is happy@@ -117,7 +131,7 @@ class Connection(object): if redirect[1] == "merovingian": logger.debug("restarting authentication") if iteration <= 10:
self.__login(iteration=iteration + 1)
self._login(iteration=iteration + 1) else: raise OperationalError("maximal number of redirects " "reached (10)")@@ -138,9 +152,6 @@ class Connection(object): else: raise ProgrammingError("unknown state: %s" % prompt)
self.state = STATE_READYreturn True- def disconnect(self): """ disconnect from the monetdb server """ self.state = STATE_INIT
@@ -153,8 +164,8 @@ class Connection(object): if self.state != STATE_READY: raise(ProgrammingError, "Not connected")
self.__putblock(operation)response = self.__getblock()
self._putblock(operation)response = self._getblock() if not len(response): return "" elif response.startswith(MSG_OK):@@ -166,10 +177,16 @@ class Connection(object): return response elif response[0] == MSG_ERROR: raise OperationalError(response[1:])
elif (self.language == 'control' and not self.hostname):if response.startswith("OK"):return response[2:].strip() or ""else:return response else: raise ProgrammingError("unknown state: %s" % response)
- def __challenge_response(self, challenge):
- def _challenge_response(self, challenge): """ generate a response to a mapi login challenge """ challenges = challenge.split(':') salt, identity, protocol, hashes, endian = challenges[:5]
@@ -204,19 +221,36 @@ class Connection(object): return ":".join(["BIG", self.username, pwhash, self.language, self.database]) + ":"
- def __getblock(self):
- def _getblock(self): """ read one mapi encoded block """
if (self.language == 'control' and not self.hostname):return self._getblock_socket() # control doesn't do block# splitting when using a socketelse:return self._getblock_inet()- def _getblock_inet(self): result = StringIO() last = 0 while not last:
flag = self.__getbytes(2)
flag = self._getbytes(2) unpacked = struct.unpack('<H', flag)[0] # little endian short length = unpacked >> 1 last = unpacked & 1
result.write(self.__getbytes(length))
result.write(self._getbytes(length)) return result.getvalue()
- def __getbytes(self, bytes_):
- def _getblock_socket(self):
buffer = StringIO()while True:x = self.socket.recv(1)if len(x):buffer.write(x)else:breakreturn buffer.getvalue().strip()- def _getbytes(self, bytes_): """Read an amount of bytes from the socket""" result = StringIO() count = bytes_
@@ -228,8 +262,15 @@ class Connection(object): result.write(recv) return result.getvalue()
- def __putblock(self, block):
- def _putblock(self, block): """ wrap the line in mapi format and put it into the socket """
if (self.language == 'control' and not self.hostname):return self.socket.send(block) # control doesn't do block# splitting when using a socketelse:self._putblock_inet(block)- def _putblock_inet(self, block): pos = 0 last = 0 while not last:
diff --git a/clients/python2/monetdb/sql/connections.py b/clients/python2/monetdb/sql/connections.py --- a/clients/python2/monetdb/sql/connections.py +++ b/clients/python2/monetdb/sql/connections.py @@ -28,25 +28,25 @@ class Connection(object): """A MonetDB SQL database connection""" default_cursor = cursors.Cursor
- def __init__(self, username="monetdb", password="monetdb",
hostname="localhost", port=50000, database="demo",autocommit=False, user=None, host=None):
- def __init__(self, database, hostname=None, port=50000, username="monetdb",
password="monetdb", unix_socket="/tmp/.s.monetdb.50000",autocommit=False): """ Set up a connection to a MonetDB SQL database.
username -- username for connection (default: monetdb)password -- password for connection (default: monetdb)hostname -- hostname to connect to (default: localhost)port -- port to connect to (default: 50000)database -- name of the database (default: demo)autocommit -- enable/disable auto commit (default: False)
database -- name of the databasehostname -- Hostname where monetDB is runningport -- port to connect to (default: 50000)username -- username for connection (default: "monetdb")password -- password for connection (default: "monetdb")unix_socket -- socket to connect to. used when hostname not set(default: "/tmp/.s.monetdb.50000")autocommit -- enable/disable auto commit (default: False)"""
if user is not None:username = userif host is not None:hostname = host self.mapi = mapi.Connection() self.mapi.connect(hostname=hostname, port=int(port), username=username,password=password, database=database, language="sql")
password=password, database=database, language="sql",unix_socket=unix_socket) self.set_autocommit(autocommit) self.set_sizeheader(True) self.set_replysize(100)diff --git a/clients/python2/test/runtests.py b/clients/python2/test/runtests.py --- a/clients/python2/test/runtests.py +++ b/clients/python2/test/runtests.py @@ -45,6 +45,8 @@ TSTDB = os.environ.get('TSTDB', 'demo') TSTHOSTNAME = os.environ.get('TSTHOSTNAME', 'localhost') TSTUSERNAME = os.environ.get('TSTUSERNAME', 'monetdb') TSTPASSWORD = os.environ.get('TSTPASSWORD', 'monetdb') +#TSTHOSTNAME = None # set to none if test a socket
if os.environ.get("TSTDEBUG", "no") == "yes": logging.basicConfig(level=logging.DEBUG) diff --git a/clients/python2/test/test_control.py b/clients/python2/test/test_control.py --- a/clients/python2/test/test_control.py +++ b/clients/python2/test/test_control.py @@ -15,6 +15,7 @@ # Copyright August 2008-2013 MonetDB B.V. # All Rights Reserved.
+import os import unittest import logging
@@ -33,6 +34,10 @@ from monetdb.exceptions import Operation
checkin-list mailing list checkin-list@monetdb.org http://mail.monetdb.org/mailman/listinfo/checkin-list
ok, i'll fix it tomorrow.
On 09/05/2013 06:56 PM, Stefan Manegold wrote:
Hi Gijs,
in case you did not notice this, yet:
This checkin breaks several tests that do test or use the Python interface, resulting in errors like
" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True) File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "
or
" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module> dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect self.socket.connect(unix_socket) FileNotFoundError: [Errno 2] No such file or directory "
See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`
or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default as of tomorrow morning.
Best, Stefan
On Thu, Sep 05, 2013 at 02:19:01PM +0200, Gijs Molenaar wrote:
Changeset: 0eaa07b061be for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0eaa07b061be Modified Files: clients/python2/monetdb/control.py clients/python2/monetdb/mapi.py clients/python2/monetdb/sql/connections.py clients/python2/test/runtests.py clients/python2/test/test_control.py clients/python3/monetdb/control.py clients/python3/monetdb/mapi.py clients/python3/monetdb/sql/connections.py clients/python3/test/runtests.py clients/python3/test/test_control.py Branch: default Log Message:
add support for unix sockets
diffs (truncated from 649 to 300 lines):
diff --git a/clients/python2/monetdb/control.py b/clients/python2/monetdb/control.py --- a/clients/python2/monetdb/control.py +++ b/clients/python2/monetdb/control.py @@ -8,10 +8,12 @@ def parse_statusline(line): parses a sabdb format status line. Support v1 and v2.
"""
- if not line.startswith('=sabdb:'):
- if line.startswith("="):
line = line[1:]- if not line.startswith('sabdb:'): raise OperationalError('wrong result recieved')
- prot_version, rest = line.split(":", 2)[1:]
code, prot_version, rest = line.split(":", 2)
if prot_version not in ["1", "2"]: raise InterfaceError("unsupported sabdb protocol")
@@ -60,20 +62,26 @@ class Control: Use this module to manage your MonetDB databases. You can create, start, stop, lock, unlock, destroy your databases and request status information. """
- def __init__(self, hostname, port, passphrase):
def __init__(self, hostname=None, port=None, passphrase=None,
unix_socket="/tmp/.s.merovingian.50000"): self.server = mapi.Connection() self.hostname = hostname self.port = port self.passphrase = passphraseself.unix_socket= unix_socket # check connection
self.server.connect(hostname, port, 'monetdb', passphrase,'merovingian', 'control')
self.server.connect(hostname=hostname, port=port, username='monetdb',password=passphrase,database='merovingian', language='control',unix_socket=unix_socket) self.server.disconnect() def _send_command(self, database_name, command):
self.server.connect(self.hostname, self.port, 'monetdb',self.passphrase, 'merovingian', 'control')
self.server.connect(hostname=self.hostname, port=self.port,username='monetdb', password=self.passphrase,database='merovingian', language='control',unix_socket=self.unix_socket) try: return self.server.cmd("%s %s\n" % (database_name, command)) finally:@@ -165,8 +173,9 @@ class Control: """ properties = self._send_command(database_name, "get") values = {}
for dirty_line in properties.split("\n"):line = dirty_line[1:]
for line in properties.split("\n"):if line.startswith("="):line = line[1:] if not line.startswith("#"): if "=" in line: split = line.split("=")diff --git a/clients/python2/monetdb/mapi.py b/clients/python2/monetdb/mapi.py --- a/clients/python2/monetdb/mapi.py +++ b/clients/python2/monetdb/mapi.py @@ -70,8 +70,12 @@ class Connection(object): self.database = "" self.language = ""
- def connect(self, hostname, port, username, password, database, language):
""" setup connection to MAPI server"""
def connect(self, database, username, password, language, hostname=None,
port=None, unix_socket=None):""" setup connection to MAPI serverunix_socket is used if hostname is not defined.""" self.hostname = hostname self.port = port@@ -79,24 +83,34 @@ class Connection(object): self.password = password self.database = database self.language = language
self.unix_socket = unix_socket
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if hostname:self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)# For performance, mirror MonetDB/src/common/stream.c socket settings.self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)self.socket.connect((hostname, port))else:self.socket = socket.socket(socket.AF_UNIX)self.socket.connect(unix_socket)if self.language != 'control':self.socket.send('0') # don't know why, but we need to do this
# For performance, mirror MonetDB/src/common/stream.c socket settings.self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
if not (self.language == 'control' and not self.hostname):# control doesn't require authentication over socketself._login()
self.socket.connect((hostname, port))self.__login()
self.state = STATE_READY
- def __login(self, iteration=0):
- def _login(self, iteration=0): """ Reads challenge from line, generate response and check if everything is okay """
challenge = self.__getblock()response = self.__challenge_response(challenge)self.__putblock(response)prompt = self.__getblock().strip()
challenge = self._getblock()response = self._challenge_response(challenge)self._putblock(response)prompt = self._getblock().strip() if len(prompt) == 0: # Empty response, server is happy@@ -117,7 +131,7 @@ class Connection(object): if redirect[1] == "merovingian": logger.debug("restarting authentication") if iteration <= 10:
self.__login(iteration=iteration + 1)
self._login(iteration=iteration + 1) else: raise OperationalError("maximal number of redirects " "reached (10)")@@ -138,9 +152,6 @@ class Connection(object): else: raise ProgrammingError("unknown state: %s" % prompt)
self.state = STATE_READYreturn Truedef disconnect(self): """ disconnect from the monetdb server """ self.state = STATE_INIT@@ -153,8 +164,8 @@ class Connection(object): if self.state != STATE_READY: raise(ProgrammingError, "Not connected")
self.__putblock(operation)response = self.__getblock()
self._putblock(operation)response = self._getblock() if not len(response): return "" elif response.startswith(MSG_OK):@@ -166,10 +177,16 @@ class Connection(object): return response elif response[0] == MSG_ERROR: raise OperationalError(response[1:])
elif (self.language == 'control' and not self.hostname):if response.startswith("OK"):return response[2:].strip() or ""else:return response else: raise ProgrammingError("unknown state: %s" % response)
- def __challenge_response(self, challenge):
- def _challenge_response(self, challenge): """ generate a response to a mapi login challenge """ challenges = challenge.split(':') salt, identity, protocol, hashes, endian = challenges[:5]
@@ -204,19 +221,36 @@ class Connection(object): return ":".join(["BIG", self.username, pwhash, self.language, self.database]) + ":"
- def __getblock(self):
- def _getblock(self): """ read one mapi encoded block """
if (self.language == 'control' and not self.hostname):return self._getblock_socket() # control doesn't do block# splitting when using a socketelse:return self._getblock_inet()- def _getblock_inet(self): result = StringIO() last = 0 while not last:
flag = self.__getbytes(2)
flag = self._getbytes(2) unpacked = struct.unpack('<H', flag)[0] # little endian short length = unpacked >> 1 last = unpacked & 1
result.write(self.__getbytes(length))
result.write(self._getbytes(length)) return result.getvalue()
- def __getbytes(self, bytes_):
- def _getblock_socket(self):
buffer = StringIO()while True:x = self.socket.recv(1)if len(x):buffer.write(x)else:breakreturn buffer.getvalue().strip()- def _getbytes(self, bytes_): """Read an amount of bytes from the socket""" result = StringIO() count = bytes_
@@ -228,8 +262,15 @@ class Connection(object): result.write(recv) return result.getvalue()
- def __putblock(self, block):
- def _putblock(self, block): """ wrap the line in mapi format and put it into the socket """
if (self.language == 'control' and not self.hostname):return self.socket.send(block) # control doesn't do block# splitting when using a socketelse:self._putblock_inet(block)- def _putblock_inet(self, block): pos = 0 last = 0 while not last:
diff --git a/clients/python2/monetdb/sql/connections.py b/clients/python2/monetdb/sql/connections.py --- a/clients/python2/monetdb/sql/connections.py +++ b/clients/python2/monetdb/sql/connections.py @@ -28,25 +28,25 @@ class Connection(object): """A MonetDB SQL database connection""" default_cursor = cursors.Cursor
- def __init__(self, username="monetdb", password="monetdb",
hostname="localhost", port=50000, database="demo",autocommit=False, user=None, host=None):
- def __init__(self, database, hostname=None, port=50000, username="monetdb",
password="monetdb", unix_socket="/tmp/.s.monetdb.50000",autocommit=False): """ Set up a connection to a MonetDB SQL database.
username -- username for connection (default: monetdb)password -- password for connection (default: monetdb)hostname -- hostname to connect to (default: localhost)port -- port to connect to (default: 50000)database -- name of the database (default: demo)autocommit -- enable/disable auto commit (default: False)
database -- name of the databasehostname -- Hostname where monetDB is runningport -- port to connect to (default: 50000)username -- username for connection (default: "monetdb")password -- password for connection (default: "monetdb")unix_socket -- socket to connect to. used when hostname not set(default: "/tmp/.s.monetdb.50000")autocommit -- enable/disable auto commit (default: False)"""
if user is not None:username = userif host is not None:hostname = host self.mapi = mapi.Connection() self.mapi.connect(hostname=hostname, port=int(port), username=username,password=password, database=database, language="sql")
password=password, database=database, language="sql",unix_socket=unix_socket) self.set_autocommit(autocommit) self.set_sizeheader(True) self.set_replysize(100)diff --git a/clients/python2/test/runtests.py b/clients/python2/test/runtests.py --- a/clients/python2/test/runtests.py +++ b/clients/python2/test/runtests.py @@ -45,6 +45,8 @@ TSTDB = os.environ.get('TSTDB', 'demo') TSTHOSTNAME = os.environ.get('TSTHOSTNAME', 'localhost') TSTUSERNAME = os.environ.get('TSTUSERNAME', 'monetdb') TSTPASSWORD = os.environ.get('TSTPASSWORD', 'monetdb') +#TSTHOSTNAME = None # set to none if test a socket
if os.environ.get("TSTDEBUG", "no") == "yes": logging.basicConfig(level=logging.DEBUG)
diff --git a/clients/python2/test/test_control.py b/clients/python2/test/test_control.py --- a/clients/python2/test/test_control.py +++ b/clients/python2/test/test_control.py @@ -15,6 +15,7 @@ # Copyright August 2008-2013 MonetDB B.V. # All Rights Reserved.
+import os import unittest import logging
@@ -33,6 +34,10 @@ from monetdb.exceptions import Operation
checkin-list mailing list checkin-list@monetdb.org http://mail.monetdb.org/mailman/listinfo/checkin-list
Thanks!
For details see http://monetdb.cwi.nl/testweb/web/testgrid.php?serial=48740:6d8d3779113d&...
in particular http://monetdb.cwi.nl/testweb/web/showtestoutput.php?serial=48740:6d8d377911... http://monetdb.cwi.nl/testweb/web/showtestoutput.php?serial=48740:6d8d377911... http://monetdb.cwi.nl/testweb/web/showtestoutput.php?serial=48740:6d8d377911... http://monetdb.cwi.nl/testweb/web/showtestoutput.php?serial=48740:6d8d377911...
Stefan
----- Original Message -----
ok, i'll fix it tomorrow.
On 09/05/2013 06:56 PM, Stefan Manegold wrote:
Hi Gijs,
in case you did not notice this, yet:
This checkin breaks several tests that do test or use the Python interface, resulting in errors like
" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True) File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "
or
" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module> dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect self.socket.connect(unix_socket) FileNotFoundError: [Errno 2] No such file or directory "
See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`
or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default as of tomorrow morning.
Best, Stefan
On Thu, Sep 05, 2013 at 02:19:01PM +0200, Gijs Molenaar wrote:
Changeset: 0eaa07b061be for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0eaa07b061be Modified Files: clients/python2/monetdb/control.py clients/python2/monetdb/mapi.py clients/python2/monetdb/sql/connections.py clients/python2/test/runtests.py clients/python2/test/test_control.py clients/python3/monetdb/control.py clients/python3/monetdb/mapi.py clients/python3/monetdb/sql/connections.py clients/python3/test/runtests.py clients/python3/test/test_control.py Branch: default Log Message:
add support for unix sockets
diffs (truncated from 649 to 300 lines):
diff --git a/clients/python2/monetdb/control.py b/clients/python2/monetdb/control.py --- a/clients/python2/monetdb/control.py +++ b/clients/python2/monetdb/control.py @@ -8,10 +8,12 @@ def parse_statusline(line): parses a sabdb format status line. Support v1 and v2.
"""
- if not line.startswith('=sabdb:'):
- if line.startswith("="):
line = line[1:]- if not line.startswith('sabdb:'): raise OperationalError('wrong result recieved')
- prot_version, rest = line.split(":", 2)[1:]
code, prot_version, rest = line.split(":", 2)
if prot_version not in ["1", "2"]: raise InterfaceError("unsupported sabdb protocol")
@@ -60,20 +62,26 @@ class Control: Use this module to manage your MonetDB databases. You can create, start, stop, lock, unlock, destroy your databases and request status information. """
- def __init__(self, hostname, port, passphrase):
def __init__(self, hostname=None, port=None, passphrase=None,
unix_socket="/tmp/.s.merovingian.50000"): self.server = mapi.Connection() self.hostname = hostname self.port = port self.passphrase = passphraseself.unix_socket= unix_socket # check connection
self.server.connect(hostname, port, 'monetdb', passphrase,'merovingian', 'control')
self.server.connect(hostname=hostname, port=port,username='monetdb',
password=passphrase,database='merovingian', language='control',unix_socket=unix_socket) self.server.disconnect() def _send_command(self, database_name, command):
self.server.connect(self.hostname, self.port, 'monetdb',self.passphrase, 'merovingian', 'control')
self.server.connect(hostname=self.hostname, port=self.port,username='monetdb', password=self.passphrase,database='merovingian', language='control',unix_socket=self.unix_socket) try: return self.server.cmd("%s %s\n" % (database_name, command)) finally:@@ -165,8 +173,9 @@ class Control: """ properties = self._send_command(database_name, "get") values = {}
for dirty_line in properties.split("\n"):line = dirty_line[1:]
for line in properties.split("\n"):if line.startswith("="):line = line[1:] if not line.startswith("#"): if "=" in line: split = line.split("=")diff --git a/clients/python2/monetdb/mapi.py b/clients/python2/monetdb/mapi.py --- a/clients/python2/monetdb/mapi.py +++ b/clients/python2/monetdb/mapi.py @@ -70,8 +70,12 @@ class Connection(object): self.database = "" self.language = ""
- def connect(self, hostname, port, username, password, database,
language):
""" setup connection to MAPI server"""
- def connect(self, database, username, password, language,
hostname=None,
port=None, unix_socket=None):""" setup connection to MAPI serverunix_socket is used if hostname is not defined.""" self.hostname = hostname self.port = port@@ -79,24 +83,34 @@ class Connection(object): self.password = password self.database = database self.language = language
self.unix_socket = unix_socket
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if hostname:self.socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
# For performance, mirror MonetDB/src/common/stream.c socketsettings.
self.socket.setsockopt(socket.SOL_SOCKET,socket.SO_KEEPALIVE, 0)
self.socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY, 1)
self.socket.connect((hostname, port))else:self.socket = socket.socket(socket.AF_UNIX)self.socket.connect(unix_socket)if self.language != 'control':self.socket.send('0') # don't know why, but we need todo this
# For performance, mirror MonetDB/src/common/stream.c socketsettings.
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0)self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
if not (self.language == 'control' and not self.hostname):# control doesn't require authentication over socketself._login()
self.socket.connect((hostname, port))self.__login()
self.state = STATE_READY
- def __login(self, iteration=0):
- def _login(self, iteration=0): """ Reads challenge from line, generate response and check if everything is okay """
challenge = self.__getblock()response = self.__challenge_response(challenge)self.__putblock(response)prompt = self.__getblock().strip()
challenge = self._getblock()response = self._challenge_response(challenge)self._putblock(response)prompt = self._getblock().strip() if len(prompt) == 0: # Empty response, server is happy@@ -117,7 +131,7 @@ class Connection(object): if redirect[1] == "merovingian": logger.debug("restarting authentication") if iteration <= 10:
self.__login(iteration=iteration + 1)
self._login(iteration=iteration + 1) else: raise OperationalError("maximal number of redirects " "reached (10)")@@ -138,9 +152,6 @@ class Connection(object): else: raise ProgrammingError("unknown state: %s" % prompt)
self.state = STATE_READYreturn Truedef disconnect(self): """ disconnect from the monetdb server """ self.state = STATE_INIT@@ -153,8 +164,8 @@ class Connection(object): if self.state != STATE_READY: raise(ProgrammingError, "Not connected")
self.__putblock(operation)response = self.__getblock()
self._putblock(operation)response = self._getblock() if not len(response): return "" elif response.startswith(MSG_OK):@@ -166,10 +177,16 @@ class Connection(object): return response elif response[0] == MSG_ERROR: raise OperationalError(response[1:])
elif (self.language == 'control' and not self.hostname):if response.startswith("OK"):return response[2:].strip() or ""else:return response else: raise ProgrammingError("unknown state: %s" % response)
- def __challenge_response(self, challenge):
- def _challenge_response(self, challenge): """ generate a response to a mapi login challenge """ challenges = challenge.split(':') salt, identity, protocol, hashes, endian = challenges[:5]
@@ -204,19 +221,36 @@ class Connection(object): return ":".join(["BIG", self.username, pwhash, self.language, self.database]) + ":"
- def __getblock(self):
- def _getblock(self): """ read one mapi encoded block """
if (self.language == 'control' and not self.hostname):return self._getblock_socket() # control doesn't do block# splitting when using asocket
else:return self._getblock_inet()- def _getblock_inet(self): result = StringIO() last = 0 while not last:
flag = self.__getbytes(2)
flag = self._getbytes(2) unpacked = struct.unpack('<H', flag)[0] # little endian short length = unpacked >> 1 last = unpacked & 1
result.write(self.__getbytes(length))
result.write(self._getbytes(length)) return result.getvalue()
- def __getbytes(self, bytes_):
- def _getblock_socket(self):
buffer = StringIO()while True:x = self.socket.recv(1)if len(x):buffer.write(x)else:breakreturn buffer.getvalue().strip()- def _getbytes(self, bytes_): """Read an amount of bytes from the socket""" result = StringIO() count = bytes_
@@ -228,8 +262,15 @@ class Connection(object): result.write(recv) return result.getvalue()
- def __putblock(self, block):
- def _putblock(self, block): """ wrap the line in mapi format and put it into the socket """
if (self.language == 'control' and not self.hostname):return self.socket.send(block) # control doesn't do block# splitting when using asocket
else:self._putblock_inet(block)- def _putblock_inet(self, block): pos = 0 last = 0 while not last:
diff --git a/clients/python2/monetdb/sql/connections.py b/clients/python2/monetdb/sql/connections.py --- a/clients/python2/monetdb/sql/connections.py +++ b/clients/python2/monetdb/sql/connections.py @@ -28,25 +28,25 @@ class Connection(object): """A MonetDB SQL database connection""" default_cursor = cursors.Cursor
- def __init__(self, username="monetdb", password="monetdb",
hostname="localhost", port=50000, database="demo",autocommit=False, user=None, host=None):
- def __init__(self, database, hostname=None, port=50000,
username="monetdb",
password="monetdb", unix_socket="/tmp/.s.monetdb.50000",autocommit=False): """ Set up a connection to a MonetDB SQL database.
username -- username for connection (default: monetdb)password -- password for connection (default: monetdb)hostname -- hostname to connect to (default: localhost)port -- port to connect to (default: 50000)database -- name of the database (default: demo)autocommit -- enable/disable auto commit (default: False)
database -- name of the databasehostname -- Hostname where monetDB is runningport -- port to connect to (default: 50000)username -- username for connection (default: "monetdb")password -- password for connection (default: "monetdb")unix_socket -- socket to connect to. used when hostname not set(default: "/tmp/.s.monetdb.50000")autocommit -- enable/disable auto commit (default: False)"""
if user is not None:username = userif host is not None:hostname = host self.mapi = mapi.Connection() self.mapi.connect(hostname=hostname, port=int(port), username=username,password=password, database=database,language="sql")
password=password, database=database,language="sql",
unix_socket=unix_socket) self.set_autocommit(autocommit) self.set_sizeheader(True) self.set_replysize(100)diff --git a/clients/python2/test/runtests.py b/clients/python2/test/runtests.py --- a/clients/python2/test/runtests.py +++ b/clients/python2/test/runtests.py @@ -45,6 +45,8 @@ TSTDB = os.environ.get('TSTDB', 'demo') TSTHOSTNAME = os.environ.get('TSTHOSTNAME', 'localhost') TSTUSERNAME = os.environ.get('TSTUSERNAME', 'monetdb') TSTPASSWORD = os.environ.get('TSTPASSWORD', 'monetdb') +#TSTHOSTNAME = None # set to none if test a socket
if os.environ.get("TSTDEBUG", "no") == "yes": logging.basicConfig(level=logging.DEBUG)
diff --git a/clients/python2/test/test_control.py b/clients/python2/test/test_control.py --- a/clients/python2/test/test_control.py +++ b/clients/python2/test/test_control.py @@ -15,6 +15,7 @@ # Copyright August 2008-2013 MonetDB B.V. # All Rights Reserved.
+import os import unittest import logging
@@ -33,6 +34,10 @@ from monetdb.exceptions import Operation
checkin-list mailing list checkin-list@monetdb.org http://mail.monetdb.org/mailman/listinfo/checkin-list
Hi Stefan, others,
Question:
* Is the path to the socket always /tmp.s.merovingian.<port> and /tmp.s.monetdb.<port> ?
* If not, is it okay if I just make these the defaults?
* The SQL MAPI over the monetdb socket seems to work fine, but the control MAPI seems to be quite slow and for some commands (kill) is hanging or maybe not correctly implemented. The control MAPI protocol over a socket is a bit weird, it differs quite a lot from the other implementations. Maybe you guys have an idea what is going on? i've been looking at the C code and found some hints, (like you need to send a 0 at the beginning of communicating over a socket, but not in the case of control), but i'm a bit stuck now.
I've been working on this so we can control MonetDB databases from python without the need to require to set a managed passphrase. Also i'm working on a SQLAlchemy Dialect:
https://github.com/gijzelaerr/sqlalchemy-monetdb
greetings,
- Gijs
On 05/09/13 18:56, Stefan Manegold wrote:
Hi Gijs,
in case you did not notice this, yet:
This checkin breaks several tests that do test or use the Python interface, resulting in errors like
" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True) File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "
or
" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module> dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect self.socket.connect(unix_socket) FileNotFoundError: [Errno 2] No such file or directory "
See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`
or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default as of tomorrow morning.
Hi Stefan, others,
Question:
- Is the path to the socket always /tmp.s.merovingian.<port> and /tmp.s.monetdb.<port> ?
I honestly don't know. I'd have to look into the documentation or documentation. ( NB, `man monetdbd` says " sockdir For faster access, monetdbd uses UNIX domain sockets for its control mechanism and regular database connections. The sockets are placed as files in the filesystem hierarchy. The sockdir property controls in which directory they are placed. In general this setting should not be changed. "
Maybe Fabian and/or Sjoerd know more?
Stefan
If not, is it okay if I just make these the defaults?
The SQL MAPI over the monetdb socket seems to work fine, but the control MAPI seems to be quite slow and for some commands (kill) is hanging or maybe not correctly implemented. The control MAPI protocol over a socket is a bit weird, it differs quite a lot from the other implementations. Maybe you guys have an idea what is going on? i've been looking at the C code and found some hints, (like you need to send a 0 at the beginning of communicating over a socket, but not in the case of control), but i'm a bit stuck now.
I've been working on this so we can control MonetDB databases from python without the need to require to set a managed passphrase. Also i'm working on a SQLAlchemy Dialect:
https://github.com/gijzelaerr/sqlalchemy-monetdb
greetings,
- Gijs
On 05/09/13 18:56, Stefan Manegold wrote:
Hi Gijs,
in case you did not notice this, yet:
This checkin breaks several tests that do test or use the Python interface, resulting in errors like
" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True) File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "
or
" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module> dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect self.socket.connect(unix_socket) FileNotFoundError: [Errno 2] No such file or directory "
See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`
or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default as of tomorrow morning.
-- Gijs Molenaar http://www.astro.uva.nl/people/gijs-molenaar/
On 06-09-2013 09:42:10 +0200, Gijs Molenaar wrote:
Hi Stefan, others,
Question:
- Is the path to the socket always /tmp.s.merovingian.<port> and /tmp.s.monetdb.<port> ?
yes, unless configured differently
- If not, is it okay if I just make these the defaults?
the "/tmp/" part can be made configurable/default
- The SQL MAPI over the monetdb socket seems to work fine, but the control MAPI seems to be quite slow and for some commands (kill) is hanging or maybe not correctly implemented. The control MAPI protocol over a socket is a bit weird, it differs quite a lot from the other implementations. Maybe you guys have an idea what is going on? i've been looking at the C code and found some hints, (like you need to send a 0 at the beginning of communicating over a socket, but not in the case of control), but i'm a bit stuck now.
control and mapi are different things, socket mode doesn't use blocks, and doesn't login iirc
I think the Java code has comments/explanation.
I've been working on this so we can control MonetDB databases from python without the need to require to set a managed passphrase. Also i'm working on a SQLAlchemy Dialect:
https://github.com/gijzelaerr/sqlalchemy-monetdb
greetings,
- Gijs
On 05/09/13 18:56, Stefan Manegold wrote:
Hi Gijs,
in case you did not notice this, yet:
This checkin breaks several tests that do test or use the Python interface, resulting in errors like
" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True) File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "
or
" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module> dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect self.socket.connect(unix_socket) FileNotFoundError: [Errno 2] No such file or directory "
See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`
or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default as of tomorrow morning.
-- Gijs Molenaar http://www.astro.uva.nl/people/gijs-molenaar/
developers-list mailing list developers-list@monetdb.org http://mail.monetdb.org/mailman/listinfo/developers-list
On 06/09/13 11:14, Fabian Groffen wrote:
On 06-09-2013 09:42:10 +0200, Gijs Molenaar wrote:
Hi Stefan, others,
Question:
- Is the path to the socket always /tmp.s.merovingian.<port> and /tmp.s.monetdb.<port> ?
yes, unless configured differently
- If not, is it okay if I just make these the defaults?
the "/tmp/" part can be made configurable/default
ok, I did that.
- The SQL MAPI over the monetdb socket seems to work fine, but the control MAPI seems to be quite slow and for some commands (kill) is hanging or maybe not correctly implemented. The control MAPI protocol over a socket is a bit weird, it differs quite a lot from the other implementations. Maybe you guys have an idea what is going on? i've been looking at the C code and found some hints, (like you need to send a 0 at the beginning of communicating over a socket, but not in the case of control), but i'm a bit stuck now.
control and mapi are different things, socket mode doesn't use blocks, and doesn't login iirc
I think the Java code has comments/explanation.
Yes, i used that as a base. But java doesn't do unix sockets, so it didn't implement that part. And the protocol is bit different for unix sockets.
On 06-09-2013 11:45:47 +0200, Gijs Molenaar wrote:
control and mapi are different things, socket mode doesn't use blocks, and doesn't login iirc
I think the Java code has comments/explanation.
Yes, i used that as a base. But java doesn't do unix sockets, so it didn't implement that part. And the protocol is bit different for unix sockets.
Right you are. Then you're stuck with the monetdb utility's code. I think you can skip implementing the backwards compatabily fallback code (if I didn't rip it out yet).
Fabian
Gijs,
on a side note, please also be aware the there are no UNIX sockets on Windows.
Thanks! Stefan
----- Original Message -----
Hi Stefan, others,
Question:
Is the path to the socket always /tmp.s.merovingian.<port> and /tmp.s.monetdb.<port> ?
If not, is it okay if I just make these the defaults?
The SQL MAPI over the monetdb socket seems to work fine, but the control MAPI seems to be quite slow and for some commands (kill) is hanging or maybe not correctly implemented. The control MAPI protocol over a socket is a bit weird, it differs quite a lot from the other implementations. Maybe you guys have an idea what is going on? i've been looking at the C code and found some hints, (like you need to send a 0 at the beginning of communicating over a socket, but not in the case of control), but i'm a bit stuck now.
I've been working on this so we can control MonetDB databases from python without the need to require to set a managed passphrase. Also i'm working on a SQLAlchemy Dialect:
https://github.com/gijzelaerr/sqlalchemy-monetdb
greetings,
- Gijs
On 05/09/13 18:56, Stefan Manegold wrote:
Hi Gijs,
in case you did not notice this, yet:
This checkin breaks several tests that do test or use the Python interface, resulting in errors like
" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True) File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "
or
" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module> dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect self.socket.connect(unix_socket) FileNotFoundError: [Errno 2] No such file or directory "
See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`
or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default as of tomorrow morning.
-- Gijs Molenaar http://www.astro.uva.nl/people/gijs-molenaar/
I think this makes localhost default for Windows:
http://dev.monetdb.org/hg/MonetDB/rev/70bbc1306bde
but I'm not able to test it.
On 06/09/13 11:31, Stefan Manegold wrote:
Gijs,
on a side note, please also be aware the there are no UNIX sockets on Windows.
Thanks! Stefan
----- Original Message -----
Hi Stefan, others,
Question:
Is the path to the socket always /tmp.s.merovingian.<port> and /tmp.s.monetdb.<port> ?
If not, is it okay if I just make these the defaults?
The SQL MAPI over the monetdb socket seems to work fine, but the control MAPI seems to be quite slow and for some commands (kill) is hanging or maybe not correctly implemented. The control MAPI protocol over a socket is a bit weird, it differs quite a lot from the other implementations. Maybe you guys have an idea what is going on? i've been looking at the C code and found some hints, (like you need to send a 0 at the beginning of communicating over a socket, but not in the case of control), but i'm a bit stuck now.
I've been working on this so we can control MonetDB databases from python without the need to require to set a managed passphrase. Also i'm working on a SQLAlchemy Dialect:
https://github.com/gijzelaerr/sqlalchemy-monetdb
greetings,
- Gijs
On 05/09/13 18:56, Stefan Manegold wrote:
Hi Gijs,
in case you did not notice this, yet:
This checkin breaks several tests that do test or use the Python interface, resulting in errors like
" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True) File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "
or
" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module> dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect self.socket.connect(unix_socket) FileNotFoundError: [Errno 2] No such file or directory "
See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`
or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default as of tomorrow morning.
-- Gijs Molenaar http://www.astro.uva.nl/people/gijs-molenaar/
thanks!
Nightly testing will tell us tomorrow morning ;-)
Stefan
Gijs Molenaar g.j.molenaar@uva.nl wrote:
I think this makes localhost default for Windows:
http://dev.monetdb.org/hg/MonetDB/rev/70bbc1306bde
but I'm not able to test it.
On 06/09/13 11:31, Stefan Manegold wrote:
Gijs,
on a side note, please also be aware the there are no UNIX sockets on
Windows.
Thanks! Stefan
----- Original Message -----
Hi Stefan, others,
Question:
Is the path to the socket always /tmp.s.merovingian.<port> and /tmp.s.monetdb.<port> ?
If not, is it okay if I just make these the defaults?
The SQL MAPI over the monetdb socket seems to work fine, but the control MAPI seems to be quite slow and for some commands (kill) is hanging or maybe not correctly implemented. The control MAPI protocol over a socket is a bit weird, it differs quite a lot from the other implementations. Maybe you guys have an idea what is going on? i've been looking at the C code and found some hints, (like you need to send a 0 at the beginning of communicating over a socket, but not in the case of control), but i'm a bit stuck
now.
I've been working on this so we can control MonetDB databases from python without the need to require to set a managed passphrase. Also i'm working on a SQLAlchemy Dialect:
https://github.com/gijzelaerr/sqlalchemy-monetdb
greetings,
- Gijs
On 05/09/13 18:56, Stefan Manegold wrote:
Hi Gijs,
in case you did not notice this, yet:
This checkin breaks several tests that do test or use the Python
interface,
resulting in errors like
" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh =
monetdb.sql.Connection(port=port,database=db,autocommit=True)
File
"PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py",
line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line
96, in
connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "
or
" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in
<module> >>> dbh = >>> monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) >>> File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", >>> line 49, in __init__ >>> unix_socket=unix_socket) >>> File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in >>> connect >>> self.socket.connect(unix_socket) >>> FileNotFoundError: [Errno 2] No such file or directory >>> " >>> >>> >>> See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent >>> sql/test/mapi` >>> >>> or our nightly TestWeb @ >>> http://monetdb.cwi.nl/testweb/web/status.php?branch=default >>> as of tomorrow morning. >> >> -- >> Gijs Molenaar >> http://www.astro.uva.nl/people/gijs-molenaar/ >> >> >
-- Gijs Molenaar http://www.astro.uva.nl/people/gijs-molenaar/
participants (3)
-
Fabian Groffen -
Gijs Molenaar -
Stefan Manegold