Changeset: 5ab4a71aa65b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5ab4a71aa65b
Modified Files:
gdk/gdk_atoms.h
sql/test/remote/Tests/creds.SQL.py
sql/test/remote/Tests/different_user.SQL.py
sql/test/remote/Tests/invalid_creds.SQL.py
sql/test/remote/Tests/ssbm.SQL.py
Branch: linear-hashing
Log Message:
Merge with Nov2019 branch.
diffs (truncated from 544 to 300 lines):
diff --git a/gdk/gdk_atoms.h b/gdk/gdk_atoms.h
--- a/gdk/gdk_atoms.h
+++ b/gdk/gdk_atoms.h
@@ -165,8 +165,8 @@ gdk_export const ptr ptr_nil;
* In all algorithms across GDK, you will find switches on the types
* (bte, sht, int, flt, dbl, lng, hge, str). They respectively
* represent an octet, a 16-bit int, a 32-bit int, a 32-bit float, a
- * 64-bit double, a 64-bit int, and a pointer-sized location of a
- * char-buffer (ended by a zero char).
+ * 64-bit double, a 64-bit int, a 128-bit int, and a pointer-sized location
+ * of a char-buffer (ended by a zero char).
*
* In contrast, the types (bit, ptr, bat, oid) are derived types. They
* do not occur in the switches. The ATOMstorage macro maps them
diff --git a/sql/test/remote/Tests/creds.SQL.py
b/sql/test/remote/Tests/creds.SQL.py
--- a/sql/test/remote/Tests/creds.SQL.py
+++ b/sql/test/remote/Tests/creds.SQL.py
@@ -89,43 +89,45 @@ def create_workers(fn_template, nworkers
# Start supervisor database
supervisorport = freeport()
os.mkdir(os.path.join(TMPDIR, "supervisor"))
-supervisorproc = process.server(mapiport=supervisorport, dbname="supervisor",
dbfarm=os.path.join(TMPDIR, "supervisor"), stdin=process.PIPE,
stdout=process.PIPE)
-supervisorconn = pymonetdb.connect(database='supervisor', port=supervisorport,
autocommit=True)
-supervisor_uri =
"mapi:monetdb://localhost:{}/supervisor".format(supervisorport)
-c = supervisorconn.cursor()
+try:
+ supervisorproc = process.server(mapiport=supervisorport,
dbname="supervisor", dbfarm=os.path.join(TMPDIR, "supervisor"),
stdin=process.PIPE, stdout=process.PIPE)
+ supervisorconn = pymonetdb.connect(database='supervisor',
port=supervisorport, autocommit=True)
+ supervisor_uri =
"mapi:monetdb://localhost:{}/supervisor".format(supervisorport)
+ c = supervisorconn.cursor()
-# Create the movies table and load the data
-movies_filename=os.getenv("TSTDATAPATH")+"/netflix_data/movies.csv"
-movies_create = "CREATE TABLE movies {}".format(MOVIES_TABLE_DEF)
-c.execute(movies_create)
-load_movies = "COPY INTO movies FROM '{}' USING DELIMITERS
',','\n','\"'".format(movies_filename)
-c.execute(load_movies)
+ # Create the movies table and load the data
+ movies_filename=os.getenv("TSTDATAPATH")+"/netflix_data/movies.csv"
+ movies_create = "CREATE TABLE movies {}".format(MOVIES_TABLE_DEF)
+ c.execute(movies_create)
+ load_movies = "COPY INTO movies FROM '{}' USING DELIMITERS
',','\n','\"'".format(movies_filename)
+ c.execute(load_movies)
-# Declare the ratings merge table on supervisor
-mtable = "CREATE MERGE TABLE ratings {}".format(RATINGS_TABLE_DEF)
-c.execute(mtable)
+ # Declare the ratings merge table on supervisor
+ mtable = "CREATE MERGE TABLE ratings {}".format(RATINGS_TABLE_DEF)
+ c.execute(mtable)
-# Create the workers and load the ratings data
-fn_template=os.getenv("TSTDATAPATH")+"/netflix_data/ratings_sample_{}.csv"
-cmovies = "CREATE REMOTE TABLE movies {} ON '{}' WITH USER 'monetdb' PASSWORD
'monetdb'".format(MOVIES_TABLE_DEF, supervisor_uri)
-workers = create_workers(fn_template, NWORKERS, cmovies, RATINGS_TABLE_DEF_FK)
+ # Create the workers and load the ratings data
+ fn_template=os.getenv("TSTDATAPATH")+"/netflix_data/ratings_sample_{}.csv"
+ cmovies = "CREATE REMOTE TABLE movies {} ON '{}' WITH USER 'monetdb'
PASSWORD 'monetdb'".format(MOVIES_TABLE_DEF, supervisor_uri)
+ try:
+ workers = create_workers(fn_template, NWORKERS, cmovies,
RATINGS_TABLE_DEF_FK)
-# Create the remote tables on supervisor
-for wrec in workers:
- rtable = "CREATE REMOTE TABLE ratings{} {} on '{}' WITH USER 'monetdb'
PASSWORD 'monetdb'".format(wrec['num'], RATINGS_TABLE_DEF, wrec['mapi'])
- c.execute(rtable)
+ # Create the remote tables on supervisor
+ for wrec in workers:
+ rtable = "CREATE REMOTE TABLE ratings{} {} on '{}' WITH USER
'monetdb' PASSWORD 'monetdb'".format(wrec['num'], RATINGS_TABLE_DEF,
wrec['mapi'])
+ c.execute(rtable)
- atable = "ALTER TABLE ratings add table ratings{}".format(wrec['num'])
- c.execute(atable)
+ atable = "ALTER TABLE ratings add table
ratings{}".format(wrec['num'])
+ c.execute(atable)
-# Run the queries
-c.execute("SELECT COUNT(*) FROM ratings0")
-print("{} rows in remote table".format(c.fetchall()[0][0]))
+ # Run the queries
+ c.execute("SELECT COUNT(*) FROM ratings0")
+ print("{} rows in remote table".format(c.fetchall()[0][0]))
-c.execute("SELECT COUNT(*) FROM ratings")
-print("{} rows in merge table".format(c.fetchall()[0][0]))
-
-for wrec in workers:
- wrec['proc'].communicate()
-
-supervisorproc.communicate()
+ c.execute("SELECT COUNT(*) FROM ratings")
+ print("{} rows in merge table".format(c.fetchall()[0][0]))
+ finally:
+ for wrec in workers:
+ wrec['proc'].communicate()
+finally:
+ supervisorproc.communicate()
diff --git a/sql/test/remote/Tests/different_user.SQL.py
b/sql/test/remote/Tests/different_user.SQL.py
--- a/sql/test/remote/Tests/different_user.SQL.py
+++ b/sql/test/remote/Tests/different_user.SQL.py
@@ -90,58 +90,60 @@ def create_workers(fn_template, nworkers
# Start supervisor database
supervisorport = freeport()
os.mkdir(os.path.join(TMPDIR, "supervisor"))
-supervisorproc = process.server(mapiport=supervisorport, dbname="supervisor",
dbfarm=os.path.join(TMPDIR, "supervisor"), stdin=process.PIPE,
stdout=process.PIPE)
-supervisorconn = pymonetdb.connect(database='supervisor', port=supervisorport,
autocommit=True)
-supervisor_uri =
"mapi:monetdb://localhost:{}/supervisor".format(supervisorport)
-c = supervisorconn.cursor()
+try:
+ supervisorproc = process.server(mapiport=supervisorport,
dbname="supervisor", dbfarm=os.path.join(TMPDIR, "supervisor"),
stdin=process.PIPE, stdout=process.PIPE)
+ supervisorconn = pymonetdb.connect(database='supervisor',
port=supervisorport, autocommit=True)
+ supervisor_uri =
"mapi:monetdb://localhost:{}/supervisor".format(supervisorport)
+ c = supervisorconn.cursor()
-# Create user/schema on supervisor
-
-c.execute("CREATE USER \"supervisor_user\" WITH PASSWORD 'supervisor_pass'
NAME 'Supervisor user' schema \"sys\"")
-c.execute("CREATE SCHEMA \"supervisor_schema\" AUTHORIZATION
\"supervisor_user\"")
-c.execute("ALTER USER \"supervisor_user\" SET SCHEMA \"supervisor_schema\"");
-c.execute("SET SCHEMA \"supervisor_schema\"")
+ # Create user/schema on supervisor
-# Create the movies table and load the data
-movies_filename=os.getenv("TSTDATAPATH")+"/netflix_data/movies.csv"
-movies_create = "CREATE TABLE movies {}".format(MOVIES_TABLE_DEF)
-c.execute(movies_create)
-load_movies = "COPY INTO movies FROM '{}' USING DELIMITERS
',','\n','\"'".format(movies_filename)
-c.execute(load_movies)
+ c.execute("CREATE USER \"supervisor_user\" WITH PASSWORD 'supervisor_pass'
NAME 'Supervisor user' schema \"sys\"")
+ c.execute("CREATE SCHEMA \"supervisor_schema\" AUTHORIZATION
\"supervisor_user\"")
+ c.execute("ALTER USER \"supervisor_user\" SET SCHEMA
\"supervisor_schema\"");
+ c.execute("SET SCHEMA \"supervisor_schema\"")
-# Declare the ratings merge table on supervisor
-mtable = "CREATE MERGE TABLE ratings {}".format(RATINGS_TABLE_DEF)
-c.execute(mtable)
+ # Create the movies table and load the data
+ movies_filename=os.getenv("TSTDATAPATH")+"/netflix_data/movies.csv"
+ movies_create = "CREATE TABLE movies {}".format(MOVIES_TABLE_DEF)
+ c.execute(movies_create)
+ load_movies = "COPY INTO movies FROM '{}' USING DELIMITERS
',','\n','\"'".format(movies_filename)
+ c.execute(load_movies)
-# Create the workers and load the ratings data
-fn_template=os.getenv("TSTDATAPATH")+"/netflix_data/ratings_sample_{}.csv"
-cmovies = "CREATE REMOTE TABLE movies {} ON '{}/supervisor_schema/movies' WITH
USER 'supervisor_user' PASSWORD 'supervisor_pass'".format(MOVIES_TABLE_DEF,
supervisor_uri)
-workers = create_workers(fn_template, NWORKERS, cmovies, RATINGS_TABLE_DEF_FK)
+ # Declare the ratings merge table on supervisor
+ mtable = "CREATE MERGE TABLE ratings {}".format(RATINGS_TABLE_DEF)
+ c.execute(mtable)
-# Create the remote tables on supervisor
-for wrec in workers:
- rtable = "CREATE REMOTE TABLE ratings{} {} on '{}'".format(wrec['num'],
RATINGS_TABLE_DEF, wrec['mapi'])
- c.execute(rtable)
+ # Create the workers and load the ratings data
+ fn_template=os.getenv("TSTDATAPATH")+"/netflix_data/ratings_sample_{}.csv"
+ cmovies = "CREATE REMOTE TABLE movies {} ON '{}/supervisor_schema/movies'
WITH USER 'supervisor_user' PASSWORD
'supervisor_pass'".format(MOVIES_TABLE_DEF, supervisor_uri)
+ try:
+ workers = create_workers(fn_template, NWORKERS, cmovies,
RATINGS_TABLE_DEF_FK)
- atable = "ALTER TABLE ratings add table ratings{}".format(wrec['num'])
- c.execute(atable)
+ # Create the remote tables on supervisor
+ for wrec in workers:
+ rtable = "CREATE REMOTE TABLE ratings{} {} on
'{}'".format(wrec['num'], RATINGS_TABLE_DEF, wrec['mapi'])
+ c.execute(rtable)
+
+ atable = "ALTER TABLE ratings add table
ratings{}".format(wrec['num'])
+ c.execute(atable)
-# Run the queries
-try:
- c.execute("SELECT COUNT(*) FROM ratings0")
- print("{} rows in remote table".format(c.fetchall()[0][0]))
-except pymonetdb.OperationalError as e1:
- print("OperationalError:", file=sys.stderr)
- print("# " + e1.message, file=sys.stderr)
+ # Run the queries
+ try:
+ c.execute("SELECT COUNT(*) FROM ratings0")
+ print("{} rows in remote table".format(c.fetchall()[0][0]))
+ except pymonetdb.OperationalError as e1:
+ print("OperationalError:", file=sys.stderr)
+ print("# " + e1.message, file=sys.stderr)
-try:
- c.execute("SELECT COUNT(*) FROM ratings")
- print("{} rows in merge table".format(c.fetchall()[0][0]))
-except pymonetdb.OperationalError as e2:
- print("OperationalError:", file=sys.stderr)
- print("# " + e2.message, file=sys.stderr)
-
-for wrec in workers:
- wrec['proc'].communicate()
-
-supervisorproc.communicate()
+ try:
+ c.execute("SELECT COUNT(*) FROM ratings")
+ print("{} rows in merge table".format(c.fetchall()[0][0]))
+ except pymonetdb.OperationalError as e2:
+ print("OperationalError:", file=sys.stderr)
+ print("# " + e2.message, file=sys.stderr)
+ finally:
+ for wrec in workers:
+ wrec['proc'].communicate()
+finally:
+ supervisorproc.communicate()
diff --git a/sql/test/remote/Tests/invalid_creds.SQL.py
b/sql/test/remote/Tests/invalid_creds.SQL.py
--- a/sql/test/remote/Tests/invalid_creds.SQL.py
+++ b/sql/test/remote/Tests/invalid_creds.SQL.py
@@ -90,51 +90,53 @@ def create_workers(fn_template, nworkers
# Start supervisor database
supervisorport = freeport()
os.mkdir(os.path.join(TMPDIR, "supervisor"))
-supervisorproc = process.server(mapiport=supervisorport, dbname="supervisor",
dbfarm=os.path.join(TMPDIR, "supervisor"), stdin=process.PIPE,
stdout=process.PIPE)
-supervisorconn = pymonetdb.connect(database='supervisor', port=supervisorport,
autocommit=True)
-supervisor_uri =
"mapi:monetdb://localhost:{}/supervisor".format(supervisorport)
-c = supervisorconn.cursor()
+try:
+ supervisorproc = process.server(mapiport=supervisorport,
dbname="supervisor", dbfarm=os.path.join(TMPDIR, "supervisor"),
stdin=process.PIPE, stdout=process.PIPE)
+ supervisorconn = pymonetdb.connect(database='supervisor',
port=supervisorport, autocommit=True)
+ supervisor_uri =
"mapi:monetdb://localhost:{}/supervisor".format(supervisorport)
+ c = supervisorconn.cursor()
-# Create the movies table and load the data
-movies_filename=os.getenv("TSTDATAPATH")+"/netflix_data/movies.csv"
-movies_create = "CREATE TABLE movies {}".format(MOVIES_TABLE_DEF)
-c.execute(movies_create)
-load_movies = "COPY INTO movies FROM '{}' USING DELIMITERS
',','\n','\"'".format(movies_filename)
-c.execute(load_movies)
+ # Create the movies table and load the data
+ movies_filename=os.getenv("TSTDATAPATH")+"/netflix_data/movies.csv"
+ movies_create = "CREATE TABLE movies {}".format(MOVIES_TABLE_DEF)
+ c.execute(movies_create)
+ load_movies = "COPY INTO movies FROM '{}' USING DELIMITERS
',','\n','\"'".format(movies_filename)
+ c.execute(load_movies)
-# Declare the ratings merge table on supervisor
-mtable = "CREATE MERGE TABLE ratings {}".format(RATINGS_TABLE_DEF)
-c.execute(mtable)
+ # Declare the ratings merge table on supervisor
+ mtable = "CREATE MERGE TABLE ratings {}".format(RATINGS_TABLE_DEF)
+ c.execute(mtable)
-# Create the workers and load the ratings data
-fn_template=os.getenv("TSTDATAPATH")+"/netflix_data/ratings_sample_{}.csv"
-cmovies = "CREATE REMOTE TABLE movies {} ON '{}' WITH USER 'nonexistent'
PASSWORD 'badpass'".format(MOVIES_TABLE_DEF, supervisor_uri)
-workers = create_workers(fn_template, NWORKERS, cmovies, RATINGS_TABLE_DEF_FK)
+ # Create the workers and load the ratings data
+ fn_template=os.getenv("TSTDATAPATH")+"/netflix_data/ratings_sample_{}.csv"
+ cmovies = "CREATE REMOTE TABLE movies {} ON '{}' WITH USER 'nonexistent'
PASSWORD 'badpass'".format(MOVIES_TABLE_DEF, supervisor_uri)
+ try:
+ workers = create_workers(fn_template, NWORKERS, cmovies,
RATINGS_TABLE_DEF_FK)
-# Create the remote tables on supervisor
-for wrec in workers:
- rtable = "CREATE REMOTE TABLE ratings{} {} on '{}' WITH USER 'invaliduser'
PASSWORD 'invalidpass'".format(wrec['num'], RATINGS_TABLE_DEF, wrec['mapi'])
- c.execute(rtable)
+ # Create the remote tables on supervisor
+ for wrec in workers:
+ rtable = "CREATE REMOTE TABLE ratings{} {} on '{}' WITH USER
'invaliduser' PASSWORD 'invalidpass'".format(wrec['num'], RATINGS_TABLE_DEF,
wrec['mapi'])
+ c.execute(rtable)
- atable = "ALTER TABLE ratings add table ratings{}".format(wrec['num'])
- c.execute(atable)
+ atable = "ALTER TABLE ratings add table
ratings{}".format(wrec['num'])
+ c.execute(atable)
-# Run the queries
-try:
- c.execute("SELECT COUNT(*) FROM ratings0")
- print("{} rows in remote table".format(c.fetchall()[0][0]))
-except pymonetdb.OperationalError as e1:
- print("OperationalError:", file=sys.stderr)
- print("# " + e1.args[0], file=sys.stderr)
+ # Run the queries
+ try:
+ c.execute("SELECT COUNT(*) FROM ratings0")
+ print("{} rows in remote table".format(c.fetchall()[0][0]))
+ except pymonetdb.OperationalError as e1:
+ print("OperationalError:", file=sys.stderr)
+ print("# " + e1.args[0], file=sys.stderr)
-try:
- c.execute("SELECT COUNT(*) FROM ratings")
- print("{} rows in merge table".format(c.fetchall()[0][0]))
-except pymonetdb.OperationalError as e2:
- print("OperationalError:", file=sys.stderr)
- print("# " + e2.args[0], file=sys.stderr)
-
-for wrec in workers:
- wrec['proc'].communicate()
-
-supervisorproc.communicate()
+ try:
+ c.execute("SELECT COUNT(*) FROM ratings")
+ print("{} rows in merge table".format(c.fetchall()[0][0]))
+ except pymonetdb.OperationalError as e2:
+ print("OperationalError:", file=sys.stderr)
+ print("# " + e2.args[0], file=sys.stderr)
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
