Hi,
I tried to run 'make check' with PyPy (PyPy3 and PyPy 7.3), but
it failed by "To many open files", even if "ulimit -n" returns
175833. I don't think test processes used descriptor so much and
I couldn't find what makes the limit so smaller.
However I could reduce max number of descriptors simultaneously used,
by looking over the source of the test suite.
(attached source test-suite-care-resources-patch.txt)
commit message:
[[[
Ensure close file descriptors by using context manager in test suite.
Generally, it is true that as file descriptors are closed when their
underlying objects are deleted, we don't need close them explicitly.
However if a Python language implementation that uses other strategy
than reference count model, there is no warranty it will happen
immediately after when those objects lose the last reference. So we
use context manager to ensure close() is called immediately after
those objects to be unnecessary. This helps us to run 'make check'
with PyPy even smaler limit of number of open files, although we
don't recommend use it because it is much slower than CPython for
this purpose.
* build/run_tests.py
(TestHarness._init_c_tests): Use context manager to write file.
(TestHarness.Job.execute,
TestHarness.CollectingThread._count_c_tests,
Testharness.CollectingThread._count_py_tests,):
Use context manager for Popen object.
* subversion/tests/cmdline/svntest/actions.py
(load_repo, load_dumpfile): Use context manager to read file contents.
* subversion/tests/cmdline/svntest/main.py
(trust_ssl_cert): Use context manager to read file contents.
* subversion/tests/cmdline/svntest/sandbox.py
(Sandbox._ensure_authz get_content): New function.
(Sandbox._ensure_authz): Use it.
* subversion/tests/cmdline/svntest/tree.py
(get_text): Use context manager to read file contents.
* subversion/tests/cmdline/svntest/wc.py
(State.from_wc): Use context manager to read file contents.
]]]
I don't think we support to run tests on PyPy but also I don't think
it is worthless.
Any thoughts?
Cheers,
--
Yasuhito FUTATSUKI <futat...@yf.bsclub.org>
Index: subversion/tests/cmdline/svntest/actions.py
===================================================================
--- subversion/tests/cmdline/svntest/actions.py (revision 1883200)
+++ subversion/tests/cmdline/svntest/actions.py (working copy)
@@ -507,7 +507,8 @@
normalize_props = False):
"Loads the dumpfile into sbox"
if not dump_str:
- dump_str = open(dumpfile_path, "rb").read()
+ with open(dumpfile_path, "rb") as fp:
+ dump_str = fp.read()
# Create a virgin repos and working copy
main.safe_rmtree(sbox.repo_dir, 1)
@@ -2076,7 +2077,9 @@
def load_dumpfile(filename):
"Return the contents of the FILENAME assuming that it is a dump file"
- return open(filename, "rb").readlines()
+ with open(filename, "rb") as fp:
+ dump_str = fp.readlines()
+ return dump_str
def hook_failure_message(hook_name):
"""Return the error message that the client prints for failure of the
Index: subversion/tests/cmdline/svntest/main.py
===================================================================
--- subversion/tests/cmdline/svntest/main.py (revision 1883200)
+++ subversion/tests/cmdline/svntest/main.py (working copy)
@@ -742,9 +742,9 @@
"""
cert_rep = ''
- fp = open(ssl_cert, 'r')
- for line in fp.readlines()[1:-1]:
- cert_rep = cert_rep + line.strip()
+ with open(ssl_cert, 'r') as fp:
+ for line in fp.readlines()[1:-1]:
+ cert_rep = cert_rep + line.strip()
parsed_url = urlparse(ssl_url)
netloc_url = '%s://%s' % (parsed_url.scheme, parsed_url.netloc)
Index: subversion/tests/cmdline/svntest/sandbox.py
===================================================================
--- subversion/tests/cmdline/svntest/sandbox.py (revision 1883200)
+++ subversion/tests/cmdline/svntest/sandbox.py (working copy)
@@ -167,12 +167,17 @@
def _ensure_authz(self):
"make sure the repository is accessible"
+ def get_content(f):
+ with open(f, 'r') as fp:
+ content = fp.read()
+ return content
+
if self.repo_url.startswith("http"):
default_authz = "[/]\n* = rw\n"
if (svntest.main.options.parallel == 0
and (not os.path.isfile(self.authz_file)
- or open(self.authz_file,'r').read() != default_authz)):
+ or get_content(self.authz_file) != default_authz)):
tmp_authz_file = os.path.join(svntest.main.work_dir, "authz-" +
self.name)
with open(tmp_authz_file, 'w') as f:
Index: subversion/tests/cmdline/svntest/tree.py
===================================================================
--- subversion/tests/cmdline/svntest/tree.py (revision 1883200)
+++ subversion/tests/cmdline/svntest/tree.py (working copy)
@@ -553,7 +553,9 @@
if not os.path.isfile(path):
return None
- return open(path, 'r').read()
+ with open(path, 'r') as fp:
+ content = fp.read()
+ return content
def get_child(node, name):
Index: subversion/tests/cmdline/svntest/wc.py
===================================================================
--- subversion/tests/cmdline/svntest/wc.py (revision 1883200)
+++ subversion/tests/cmdline/svntest/wc.py (working copy)
@@ -687,11 +687,11 @@
if os.path.isfile(node):
try:
if keep_eol_style:
-
- contents = io.open(node, 'r', newline='',
- encoding='utf-8').read()
+ with io.open(node, 'r', newline='', encoding='utf-8') as fp:
+ contents = fp.read()
else:
- contents = io.open(node, 'r', encoding='utf-8').read()
+ with io.open(node, 'r', encoding='utf-8') as fp:
+ contents = fp.read()
if not isinstance(contents, str):
# Python 2: contents is read as an unicode object,
# but we expect it is a str.
@@ -699,7 +699,8 @@
except:
# If the file contains non UTF-8 character, we treat its
# content as binary represented as a bytes object.
- contents = open(node, 'rb').read()
+ with open(node, 'rb') as fp:
+ contents = fp.read()
else:
contents = None
desc[repos_join(parent, name)] = StateItem(contents=contents)
Index: build/run_tests.py
===================================================================
--- build/run_tests.py (revision 1883200)
+++ build/run_tests.py (working copy)
@@ -203,8 +203,9 @@
authzparent = os.path.join(self.builddir, subdir)
if not os.path.exists(authzparent):
os.makedirs(authzparent);
- open(os.path.join(authzparent, 'authz'), 'w').write('[/]\n'
- '* = rw\n')
+ with open(os.path.join(authzparent, 'authz'), 'w') as fp:
+ fp.write('[/]\n'
+ '* = rw\n')
# ### Support --repos-template
if self.opts.list_tests is not None:
@@ -351,15 +352,15 @@
def execute(self, harness):
start_time = datetime.now()
- prog = subprocess.Popen(self._command_line(harness),
+ with subprocess.Popen(self._command_line(harness),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
- cwd=self.progdir)
+ cwd=self.progdir) as prog:
- self.stdout_lines = prog.stdout.readlines()
- self.stderr_lines = prog.stderr.readlines()
- prog.wait()
- self.result = prog.returncode
+ self.stdout_lines = prog.stdout.readlines()
+ self.stderr_lines = prog.stderr.readlines()
+ prog.wait()
+ self.result = prog.returncode
self.taken = datetime.now() - start_time
class CollectingThread(threading.Thread):
@@ -377,21 +378,23 @@
def _count_c_tests(self, progabs, progdir, progbase):
'Run a c test, escaping parameters as required.'
cmdline = [ progabs, '--list' ]
- prog = subprocess.Popen(cmdline, stdout=subprocess.PIPE, cwd=progdir)
- lines = prog.stdout.readlines()
- self.result.append(TestHarness.Job(len(lines) - 2, False, progabs,
- progdir, progbase))
- prog.wait()
+ with subprocess.Popen(cmdline, stdout=subprocess.PIPE,
+ cwd=progdir) as prog:
+ lines = prog.stdout.readlines()
+ self.result.append(TestHarness.Job(len(lines) - 2, False, progabs,
+ progdir, progbase))
+ prog.wait()
def _count_py_tests(self, progabs, progdir, progbase):
'Run a c test, escaping parameters as required.'
cmdline = [ sys.executable, progabs, '--list' ]
- prog = subprocess.Popen(cmdline, stdout=subprocess.PIPE, cwd=progdir)
- lines = prog.stdout.readlines()
+ with subprocess.Popen(cmdline, stdout=subprocess.PIPE,
+ cwd=progdir) as prog:
+ lines = prog.stdout.readlines()
- for i in range(0, len(lines) - 2):
- self.result.append(TestHarness.Job(i + 1, True, progabs,
- progdir, progbase))
+ for i in range(0, len(lines) - 2):
+ self.result.append(TestHarness.Job(i + 1, True, progabs,
+ progdir, progbase))
prog.wait()
def run(self):
