New submission from Nik Galitsky <nicolay.ye...@exxonmobil.com>:
Python 3.2 on linux (RHEL 5.3) x86_64 build from source code.
Configure options:
./configure --prefix=/scratch/Python-3.2 --enable-big-digits=30
--with-universal-archs=all --with-fpectl --enable-shared
Built with GCC 4.3.3 with major options
-g3 -O3 -m64 -fPIC.
Testcase that shows the issue:
#import numpy
import pickle
print("begin")
#a = numpy.zeros((2.5e9 / 8,), dtype = numpy.float64)
a = ('a' * (2 ** 31))
print("allocated")
#print(a);
pickle.dumps(a, pickle.DEFAULT_PROTOCOL)
print("end")
The problem as I can see it is that in pickle.py most types defined either as 2
bytes, or 4 bytes.
For example it is peppered with lines like:
self.write(SOMETYPE + pack("<i", n) + obj)
while pickling,
when unpickling:
len = mloads('i' + self.read(4))
Which limits the range and the size of the datatype that can be pickled, if I
understand correctly.
replacing in pickle.py above lines with something like
self.write(SOMETYPE + pack("<Q", n) + obj)
and
len = mloads('Q' + self.read(8))
lets above testcase run to completion.
Othervise it crashes (on Python 2.7.1 with SIGSEGV) on Python 3.2 strace shows:
.......
open("/scratch/Python-3.2/lib/python3.2/lib-dynload/_pickle.cpython-32m.so",
O_RDONLY) = 4
fstat(4, {st_mode=S_IFREG|0755, st_size=412939, ...}) = 0
open("/scratch/hpl005/UIT_test/apps_exc/Python-3.2/lib/python3.2/lib-dynload/_pickle.cpython-32m.so",
O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300>\0\0\0\0\0\0"...,
832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=412939, ...}) = 0
mmap(NULL, 2185384, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) =
0x2b05b5f68000
mprotect(0x2b05b5f7b000, 2093056, PROT_NONE) = 0
mmap(0x2b05b617a000, 16384, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x12000) = 0x2b05b617a000
close(5) = 0
close(4) = 0
close(3) = 0
write(1, "begin\n", 6begin
) = 6
mmap(NULL, 4294971392, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2b05b617e000
write(1, "allocated\n", 10allocated
) = 10
mmap(NULL, 8589938688, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2b06b617f000
mremap(0x2b06b617f000, 8589938688, 2147487744, MREMAP_MAYMOVE) = 0x2b06b617f000
mmap(NULL, 4294971392, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2b0736180000
munmap(0x2b06b617f000, 2147487744) = 0
munmap(0x2b0736180000, 4294971392) = 0
write(2, "Traceback (most recent call last"..., 35Traceback (most recent call
last):
) = 35
write(2, " File \"pickle_long.py\", line 9,"..., 45 File "pickle_long.py",
line 9, in <module>
) = 45
open("pickle_long.py", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=251, ...}) = 0
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7ffff9f7c9e0) = -1 ENOTTY
(Inappropriate ioctl for device)
fstat(3, {st_mode=S_IFREG|0644, st_size=251, ...}) = 0
lseek(3, 0, SEEK_CUR) = 0
dup(3) = 4
fcntl(4, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0644, st_size=251, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b06b617f000
lseek(4, 0, SEEK_CUR) = 0
read(4, "#import numpy\n\nimport pickle\npri"..., 4096) = 251
close(4) = 0
munmap(0x2b06b617f000, 4096) = 0
lseek(3, 0, SEEK_SET) = 0
lseek(3, 0, SEEK_CUR) = 0
read(3, "#import numpy\n\nimport pickle\npri"..., 4096) = 251
close(3) = 0
write(2, " pickle.dumps(a, pickle.DEFAU"..., 45 pickle.dumps(a,
pickle.DEFAULT_PROTOCOL)
) = 45
write(2, "SystemError: error return withou"..., 48SystemError: error return
without exception set
) = 48
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x2b05b118e4c0},
{0x2b05b0e7a570, [], SA_RESTORER, 0x2b05b118e4c0}, 8) = 0
munmap(0x2b05b617e000, 4294971392) = 0
exit_group(1) = ?
Why is this limitation?
Please advise.
----------
components: Interpreter Core
messages: 131060
nosy: nyevik
priority: normal
severity: normal
status: open
title: pickle limits most datatypes
type: crash
versions: Python 3.2
_______________________________________
Python tracker <rep...@bugs.python.org>
<http://bugs.python.org/issue11564>
_______________________________________
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
http://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
