Steffen Daode Nurpmeso <sdao...@googlemail.com> added the comment:
I was able to spend more time on that this afternoon.
'Got an unkillable diff(1) along the way which required me to
force a cold reboot. Well.
I attach a C version (11277.mmap.c) which i've used for testing.
The file 11277.zsum32.c is a quick-and-dirty C program to
calculate CRC-32 and Adler-32 checksums (i had none for the
latter and maybe you want to test some more, so); it requires zlib.
I also attach 11277.1.diff which updates test/test_zlib.py, though
this is rather useless, because that still results in a bus error.
This is the real interesting thing however, because the C version
actually works quite well for the chosen value, and the resulting
files are identical, as zsum32 shows:
Adler-32 <14b9018b> CRC-32 <c6e340bf> -- test_python_413/@test_413_tmp
Adler-32 <14b9018b> CRC-32 <c6e340bf> -- c-mmap-testfile
I thought
os.fsync(f.fileno())
does the trick because it does it in C (hi, Charles-Francois),
but no.
So what do i come up with?
Nothing. A quick look into 11277.mmap.c will show you this:
/* *Final* sizes (string written after lseek(2): "abcd") */
...
/* Tested good */
//0x100000000 - PAGESIZE - 5,
//0x100000000 - 4,
//0x100000000 - 3,
//0x100000000 - 1,
0x100000000 + PAGESIZE + 4,
//0x100000000 + PAGESIZE + 5,
/* Tested bad */
//0x100000000,
//0x100000000 + PAGESIZE,
//0x100000000 + PAGESIZE + 1,
//0x100000000 + PAGESIZE + 3,
Hm!
Now i have to go but maybe i can do some more testing tomorrow to
answer the question why test_zlib.py fails even though there is
the fsync() and even though the values work in C.
Any comments?
----------
Added file: http://bugs.python.org/file21671/11277.1.diff
Added file: http://bugs.python.org/file21672/11277.mmap.c
Added file: http://bugs.python.org/file21673/11277.zsum32.c
_______________________________________
Python tracker <rep...@bugs.python.org>
<http://bugs.python.org/issue11277>
_______________________________________diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py
--- a/Lib/test/test_zlib.py
+++ b/Lib/test/test_zlib.py
@@ -3,6 +3,7 @@
import binascii
import random
import sys
+import os
from test.support import precisionbigmemtest, _1G, _4G
zlib = support.import_module('zlib')
@@ -68,9 +69,10 @@
def setUp(self):
with open(support.TESTFN, "wb+") as f:
- f.seek(_4G)
- f.write(b"asdf")
- with open(support.TESTFN, "rb") as f:
+ f.seek(_4G + mmap.PAGESIZE)
+ f.write(b"abcd")
+ f.flush()
+ os.fsync(f.fileno())
self.mapping = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
def tearDown(self):
@@ -82,9 +84,8 @@
@unittest.skipUnless(support.is_resource_enabled("largefile"),
"May use lots of disk space.")
def test_big_buffer(self):
- self.assertEqual(zlib.crc32(self.mapping), 3058686908)
- self.assertEqual(zlib.adler32(self.mapping), 82837919)
-
+ self.assertEqual(zlib.crc32(self.mapping), 0xc6e340bf)
+ self.assertEqual(zlib.adler32(self.mapping), 0x14b9018b)
class ExceptionTestCase(unittest.TestCase):
# make sure we generate some expected errors
#include <errno.h>
#include <signal.h>
#include <stdio.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/uio.h>
#define PATH "c-mmap-testfile"
#define PAGESIZE 4096
static void sighdl(int);
static void
sighdl(int signo)
{
const char errmsg[] = "\nSignal occurred, cleaning up\n";
(void)signo;
(void)signal(SIGSEGV, SIG_DFL);
(void)signal(SIGBUS, SIG_DFL);
write(2, errmsg, sizeof(errmsg)-1);
(void)unlink(PATH);
return;
}
int
main(void) {
int fd, estat = 0;
void *addr;
auto struct stat s;
/* *Final* sizes (string written after lseek(2): "abcd") */
const size_t *ct, tests[] = {
/* Tested good */
//0x100000000 - PAGESIZE - 5,
//0x100000000 - 4,
//0x100000000 - 3,
//0x100000000 - 1,
0x100000000 + PAGESIZE + 4,
//0x100000000 + PAGESIZE + 5,
/* Tested bad */
//0x100000000,
//0x100000000 + PAGESIZE,
//0x100000000 + PAGESIZE + 1,
//0x100000000 + PAGESIZE + 3,
0
};
if (signal(SIGSEGV, &sighdl) == SIG_ERR)
goto jerror;
if (signal(SIGBUS, &sighdl) == SIG_ERR)
goto jerror;
for (ct = tests; *ct != 0; ++ct) {
fprintf(stderr, "Size %lu/0x%lX: open", *ct, *ct);
fd = open(PATH, O_RDWR|O_TRUNC|O_CREAT, 0666);
if (fd < 0)
goto jerror;
fprintf(stderr, ". ");
fprintf(stderr, "lseek");
if (lseek(fd, *ct-4, SEEK_END) < 0)
goto jerror;
fprintf(stderr, ". ");
fprintf(stderr, "write");
if (write(fd, "abcd", 4) != 4)
goto jerror;
fprintf(stderr, ". ");
fprintf(stderr, "fsync");
if (fsync(fd) != 0)
goto jerror;
fprintf(stderr, ". ");
fprintf(stderr, "fstat");
if (fstat(fd, &s) != 0)
goto jerror;
fprintf(stderr, ". ");
if (*ct != (size_t)s.st_size) {
fprintf(stderr, "fstat size mismatch: %lu is not %lu\n",
(size_t)s.st_size, *ct);
continue;
}
fprintf(stderr, "mmap");
addr = mmap(NULL, s.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
if (addr == NULL)
goto jerror;
fprintf(stderr, ". ");
(void)close(fd);
fprintf(stderr, "[0]");
if (((char*)addr)[0] != '\0')
goto jerror;
fprintf(stderr, ". ");
fprintf(stderr, "[s.st_size-4]");
if (((char*)addr)[s.st_size-4] != 'a')
goto jerror;
fprintf(stderr, ". ");
fprintf(stderr, "munmap");
if (munmap(addr, s.st_size) != 0)
goto jerror;
fprintf(stderr, ".");
fprintf(stderr, "\n");
}
jleave:
(void)unlink(PATH);
return estat;
jerror:
fprintf(stderr, "\n%s\n", strerror(errno));
estat = 1;
goto jleave;
}
/* POSIX crc32/adler32 checksum program; requires zlib (a.k.a. libz)
* Compile: cc -o zsum32 zsum32.c -lz
*/
#include <zlib.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#define BUFSIZE (8192*4)
int
main(int argc, char **argv) {
int estat = 0, erri = 0;
FILE *f;
uLong adler, crc;
const char *errs = NULL;
auto char buffer[BUFSIZE];
if (argc != 2) {
erri = EINVAL;
errs = "Usage zsum32 FILE";
goto jerror_noc;
}
++argv;
f = fopen(*argv, "rb");
if (f == NULL) {
erri = errno;
errs = "fopen()ing the file failed";
goto jerror_noc;
}
adler = adler32(0L, Z_NULL, 0);
crc = 0;
for (;;) {
size_t rb = fread(buffer, 1, BUFSIZE, f);
if (rb > 0) {
adler = adler32(adler, (const Bytef*)buffer, (uInt)rb);
crc = crc32(crc, (const Bytef*)buffer, (uInt)rb);
continue;
}
if (feof(f))
break;
erri = errno;
errs = "fread()ing the file failed";
goto jerror;
}
printf("Adler-32 <%lx> CRC-32 <%lx> -- %s\n",
(unsigned long)adler, (unsigned long)crc, *argv);
(void)fclose(f);
jleave:
return estat;
jerror:
(void)fclose(f);
jerror_noc:
fprintf(stderr, "%s: %s\n", errs, strerror(erri));
estat = 1;
goto jleave;
}
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
http://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
