laforge has submitted this change. (
https://gerrit.osmocom.org/c/pysim/+/42551?usp=email )
Change subject: docs: auto-generate Card Filesystem Reference
......................................................................
docs: auto-generate Card Filesystem Reference
Add a Sphinx extension (docs/pysim_fs_sphinx.py) that hooks into the
builder-inited event and generates docs/filesystem.rst before Sphinx
reads any source files.
The generated page contains a hierarchical listing of all implemented
EFs and DFs, organised by application/specification (UICC/TS 102 221,
ADF.USIM/TS 31.102, ADF.ISIM/TS 31.103, SIM/TS 51.011). For each file,
the class docstring and any _test_de_encode / _test_decode vectors
are included as an encoding/decoding example table.
docs/filesystem.rst is fully generated at build time and is therefore
added to .gitignore.
Add tests/unittests/test_fs_coverage.py that walks all pySim.* modules
and verifies that every CardProfile, CardApplication, and standalone
CardDF subclass with EF/DF children is either listed in the SECTIONS
(and will appear in the docs) or explicitly EXCLUDED.
Change-Id: I06ddeefc6c11e04d7c24e116f3f39c8a6635856f
Related: OS#6316
---
M .gitignore
M docs/conf.py
M docs/index.rst
A docs/pysim_fs_sphinx.py
A tests/unittests/test_fs_coverage.py
5 files changed, 417 insertions(+), 1 deletion(-)
Approvals:
dexter: Looks good to me, but someone else must approve; Verified
laforge: Looks good to me, approved
Jenkins Builder: Verified
diff --git a/.gitignore b/.gitignore
index 0cd5b14..3f1f645 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
/docs/_*
/docs/generated
+/docs/filesystem.rst
/.cache
/.local
/build
diff --git a/docs/conf.py b/docs/conf.py
index 26e8e2a..2e23fae 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -13,6 +13,7 @@
import os
import sys
sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath('.')) # for local extensions
(pysim_fs_sphinx, ...)
# -- Project information -----------------------------------------------------
@@ -39,7 +40,8 @@
"sphinx.ext.autodoc",
"sphinxarg.ext",
"sphinx.ext.autosectionlabel",
- "sphinx.ext.napoleon"
+ "sphinx.ext.napoleon",
+ "pysim_fs_sphinx",
]
# Add any paths that contain templates here, relative to this directory.
@@ -78,6 +80,7 @@
# of autosectionlabel duplicate-label warnings - suppress them.
autosectionlabel_maxdepth = 3
suppress_warnings = [
+ 'autosectionlabel.filesystem',
'autosectionlabel.saip-tool',
'autosectionlabel.shell',
'autosectionlabel.smpp2sim',
diff --git a/docs/index.rst b/docs/index.rst
index a6ed7b9..8908c4e 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -39,6 +39,7 @@
:caption: Contents:
shell
+ filesystem
trace
legacy
smpp2sim
diff --git a/docs/pysim_fs_sphinx.py b/docs/pysim_fs_sphinx.py
new file mode 100644
index 0000000..829fc31
--- /dev/null
+++ b/docs/pysim_fs_sphinx.py
@@ -0,0 +1,267 @@
+"""
+Sphinx extension: auto-generate docs/filesystem.rst from the pySim EF class
hierarchy.
+
+Hooked into Sphinx's ``builder-inited`` event so the file is always regenerated
+from the live Python classes before Sphinx reads any source files.
+
+The table of root objects to document is in SECTIONS near the top of this file.
+EXCLUDED lists CardProfile/CardApplication subclasses intentionally omitted
from
+SECTIONS, with reasons. Both tables are read by
tests/unittests/test_fs_coverage.py
+to ensure every class with EF/DF content is accounted for.
+"""
+
+import importlib
+import inspect
+import json
+import os
+import sys
+import textwrap
+
+# Ensure pySim is importable when this module is loaded as a Sphinx extension
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__),
'..')))
+
+from pySim.filesystem import (CardApplication, CardDF, CardMF, CardEF, # noqa:
E402
+ TransparentEF, TransRecEF, LinFixedEF, CyclicEF,
BerTlvEF)
+from pySim.profile import CardProfile # noqa: E402
+
+
+# Generic EF base classes whose docstrings describe the *type* of file
+# (Transparent, LinFixed, ...) rather than a specific file's content.
+# Suppress those boilerplate texts in the per-EF entries; they are only
+# useful once, at the top of the document or in a dedicated glossary.
+_EF_BASE_TYPES = frozenset([TransparentEF,
+ TransRecEF,
+ LinFixedEF,
+ CyclicEF,
+ BerTlvEF])
+
+
+# ---------------------------------------------------------------------------
+# Sections: (heading, module, class-name)
+# The class must be either a CardProfile (uses .files_in_mf) or a CardDF
+# subclass (uses .children).
+# ---------------------------------------------------------------------------
+SECTIONS = [
+ ('MF / TS 102 221 (UICC)',
+ 'pySim.ts_102_221', 'CardProfileUICC'),
+ ('ADF.USIM / TS 31.102',
+ 'pySim.ts_31_102', 'ADF_USIM'),
+ ('ADF.ISIM / TS 31.103',
+ 'pySim.ts_31_103', 'ADF_ISIM'),
+ ('ADF.HPSIM / TS 31.104',
+ 'pySim.ts_31_104', 'ADF_HPSIM'),
+ ('DF.GSM + DF.TELECOM / TS 51.011 (SIM)',
+ 'pySim.ts_51_011', 'CardProfileSIM'),
+ ('CDMA / IS-820 (RUIM)',
+ 'pySim.cdma_ruim', 'CardProfileRUIM'),
+ ('DF.EIRENE / GSM-R',
+ 'pySim.gsm_r', 'DF_EIRENE'),
+ ('DF.SYSTEM / sysmocom SJA2+SJA5',
+ 'pySim.sysmocom_sja2', 'DF_SYSTEM'),
+]
+
+# ---------------------------------------------------------------------------
+# Excluded: {(module, class-name)}
+# CardProfile and CardApplication subclasses that have EF/DF children but are
+# intentionally absent from SECTIONS. Keeping this list explicit lets
+# test_fs_coverage.py detect newly added classes that the developer forgot to
+# add to either table.
+# ---------------------------------------------------------------------------
+EXCLUDED = {
+ # eUICC profiles inherit files_in_mf verbatim from CardProfileUICC; the
+ # eUICC-specific content lives in ISD-R / ISD-P applications, not in MF.
+ ('pySim.euicc', 'CardProfileEuiccSGP02'),
+ ('pySim.euicc', 'CardProfileEuiccSGP22'),
+ ('pySim.euicc', 'CardProfileEuiccSGP32'),
+ # CardApplication* classes are thin wrappers that embed an ADF_* instance.
+ # The ADF contents are already documented via the corresponding ADF_* entry
+ # in SECTIONS above.
+ ('pySim.ts_31_102', 'CardApplicationUSIM'),
+ ('pySim.ts_31_102', 'CardApplicationUSIMnonIMSI'),
+ ('pySim.ts_31_103', 'CardApplicationISIM'),
+ ('pySim.ts_31_104', 'CardApplicationHPSIM'),
+}
+
+# RST underline characters ordered by nesting depth
+_HEADING_CHARS = ['=', '=', '-', '~', '^', '"']
+# Level 0 uses '=' with overline (page title).
+# Level 1 uses '=' without overline (major sections).
+# Levels 2+ use the remaining characters for DFs.
+
+
+# ---------------------------------------------------------------------------
+# RST formatting helpers
+# ---------------------------------------------------------------------------
+
+def _heading(title: str, level: int) -> str:
+ """Return an RST heading string. Level 0 gets an overline."""
+ char = _HEADING_CHARS[level]
+ rule = char * len(title)
+ if level == 0:
+ return f'{rule}\n{title}\n{rule}\n\n'
+ return f'{title}\n{rule}\n\n'
+
+
+def _json_default(obj):
+ """Fallback serialiser: bytes -> hex, anything else -> repr."""
+ if isinstance(obj, (bytes, bytearray)):
+ return obj.hex()
+ return repr(obj)
+
+
+def _examples_block(cls) -> str:
+ """Return RST code-block examples (one per vector), or '' if none exist.
+
+ Each example is rendered as a ``json5`` code-block with the hex-encoded
+ binary as a ``// comment`` on the first line, followed by the decoded JSON.
+ ``json5`` is used instead of ``json`` so that Pygments does not flag the
+ ``//`` comment as a syntax error.
+ """
+ vectors = []
+ for attr in ('_test_de_encode', '_test_decode'):
+ v = getattr(cls, attr, None)
+ if v:
+ vectors.extend(v)
+ if not vectors:
+ return ''
+
+ lines = ['**Examples**\n\n']
+
+ for t in vectors:
+ # 2-tuple: (encoded, decoded)
+ # 3-tuple: (encoded, record_nr, decoded) — LinFixedEF / CyclicEF
+ if len(t) >= 3:
+ encoded, record_nr, decoded = t[0], t[1], t[2]
+ comment = f'record {record_nr}: {encoded.lower()}'
+ else:
+ encoded, decoded = t[0], t[1]
+ comment = f'file: {encoded.lower()}'
+
+ json_str = json.dumps(decoded, default=_json_default, indent=2)
+ json_indented = textwrap.indent(json_str, ' ')
+
+ lines.append('.. code-block:: json5\n\n')
+ lines.append(f' // {comment}\n')
+ lines.append(json_indented + '\n')
+ lines.append('\n')
+
+ return ''.join(lines)
+
+
+def _document_ef(ef: CardEF) -> str:
+ """Return RST for a single EF. Uses ``rubric`` to stay out of the TOC."""
+ cls = type(ef)
+
+ parts = [ef.fully_qualified_path_str()]
+ if ef.fid:
+ parts.append(f'({ef.fid.upper()})')
+ if ef.desc:
+ parts.append(f'\u2014 {ef.desc}') # em-dash
+ title = ' '.join(parts)
+
+ lines = [f'.. rubric:: {title}\n\n']
+
+ # Only show a docstring if it is specific to this class. EFs that are
+ # direct instances of a base type (TransparentEF, LinFixedEF, ...) carry
+ # only the generic "what is a TransparentEF" boilerplate; named subclasses
+ # without their own __doc__ have cls.__dict__['__doc__'] == None. Either
+ # way, suppress the text here - it belongs at the document level, not
+ # repeated for every single EF entry.
+ doc = None if cls in _EF_BASE_TYPES else cls.__dict__.get('__doc__')
+ if doc:
+ lines.append(inspect.cleandoc(doc) + '\n\n')
+
+ examples = _examples_block(cls)
+ if examples:
+ lines.append(examples)
+
+ return ''.join(lines)
+
+
+def _document_df(df: CardDF, level: int) -> str:
+ """Return RST for a DF section and all its children, recursively."""
+ parts = [df.fully_qualified_path_str()]
+ if df.fid:
+ parts.append(f'({df.fid.upper()})')
+ if df.desc:
+ parts.append(f'\u2014 {df.desc}') # em-dash
+ title = ' '.join(parts)
+
+ lines = [_heading(title, level)]
+
+ cls = type(df)
+ doc = None if cls in (CardDF, CardMF) else cls.__dict__.get('__doc__')
+ if doc:
+ lines.append(inspect.cleandoc(doc) + '\n\n')
+
+ for child in df.children.values():
+ if isinstance(child, CardDF):
+ lines.append(_document_df(child, level + 1))
+ elif isinstance(child, CardEF):
+ lines.append(_document_ef(child))
+
+ return ''.join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Top-level generator
+# ---------------------------------------------------------------------------
+
+def generate_filesystem_rst() -> str:
+ """Walk all registered sections and return the full RST document as a
string."""
+ out = [
+ '.. This file is auto-generated by docs/pysim_fs_sphinx.py — do not
edit.\n\n',
+ _heading('Card Filesystem Reference', 0),
+ 'This page documents all Elementary Files (EFs) and Dedicated Files
(DFs) '
+ 'implemented in pySim, organised by their location in the card
filesystem.\n\n',
+ ]
+
+ # Track already-documented classes so that DFs/EFs shared between profiles
+ # (e.g. DF.TELECOM / DF.GSM present in both CardProfileSIM and
CardProfileRUIM)
+ # are only emitted once.
+ seen_types: set = set()
+
+ for section_title, module_path, class_name in SECTIONS:
+ module = importlib.import_module(module_path)
+ cls = getattr(module, class_name)
+ obj = cls()
+
+ if isinstance(obj, CardProfile):
+ files = obj.files_in_mf
+ elif isinstance(obj, CardApplication):
+ files = list(obj.adf.children.values())
+ elif isinstance(obj, CardDF):
+ files = list(obj.children.values())
+ else:
+ continue
+
+ # Filter out files whose class was already documented in an earlier
section.
+ files = [f for f in files if type(f) not in seen_types]
+ if not files:
+ continue
+
+ out.append(_heading(section_title, 1))
+
+ for f in files:
+ seen_types.add(type(f))
+ if isinstance(f, CardDF):
+ out.append(_document_df(f, level=2))
+ elif isinstance(f, CardEF):
+ out.append(_document_ef(f))
+
+ return ''.join(out)
+
+
+# ---------------------------------------------------------------------------
+# Sphinx integration
+# ---------------------------------------------------------------------------
+
+def _on_builder_inited(app):
+ output_path = os.path.join(app.srcdir, 'filesystem.rst')
+ with open(output_path, 'w') as fh:
+ fh.write(generate_filesystem_rst())
+
+
+def setup(app):
+ app.connect('builder-inited', _on_builder_inited)
+ return {'version': '0.1', 'parallel_read_safe': True}
diff --git a/tests/unittests/test_fs_coverage.py
b/tests/unittests/test_fs_coverage.py
new file mode 100644
index 0000000..6021665
--- /dev/null
+++ b/tests/unittests/test_fs_coverage.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Verify that every CardProfile / CardApplication subclass with EF/DF content,
+and every standalone CardDF subclass (one not reachable as a child of any
profile
+or application), is either listed in docs/pysim_fs_sphinx.py::SECTIONS or
+explicitly EXCLUDED."""
+
+import unittest
+import importlib
+import inspect
+import pkgutil
+import sys
+import os
+
+# Make docs/pysim_fs_sphinx.py importable without a full Sphinx build.
+_DOCS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..',
'..', 'docs')
+sys.path.insert(0, os.path.abspath(_DOCS_DIR))
+
+import pySim # noqa: E402
+from pySim.filesystem import CardApplication, CardDF, CardMF, CardADF # noqa:
E402
+from pySim.profile import CardProfile # noqa: E402
+from pysim_fs_sphinx import EXCLUDED, SECTIONS # noqa: E402
+
+
+class TestFsCoverage(unittest.TestCase):
+ """Ensure SECTIONS + EXCLUDED together account for all classes with
content."""
+
+ # Base CardDF types that are not concrete filesystem objects on their own.
+ _DF_BASE_TYPES = frozenset([CardDF, CardMF, CardADF])
+
+ @staticmethod
+ def _collect_reachable_df_types(obj) -> set:
+ """Return the set of all CardDF *types* reachable as children of
*obj*."""
+ result = set()
+ if isinstance(obj, CardProfile):
+ children = obj.files_in_mf
+ elif isinstance(obj, CardApplication):
+ result.add(type(obj.adf))
+ children = list(obj.adf.children.values())
+ elif isinstance(obj, CardDF):
+ children = list(obj.children.values())
+ else:
+ return result
+ queue = list(children)
+ while queue:
+ child = queue.pop()
+ if isinstance(child, CardDF):
+ result.add(type(child))
+ queue.extend(child.children.values())
+ return result
+
+ @staticmethod
+ def _has_content(obj) -> bool:
+ """Return True if *obj* owns any EFs/DFs."""
+ if isinstance(obj, CardProfile):
+ return bool(obj.files_in_mf)
+ if isinstance(obj, CardApplication):
+ return bool(obj.adf.children)
+ return False
+
+ def test_all_profiles_and_apps_covered(self):
+ # build a set of (module, class-name) pairs that are already accounted
for
+ covered = {(mod, cls) for (_, mod, cls) in SECTIONS}
+ accounted_for = covered | EXCLUDED
+
+ uncovered = []
+ reachable_df_types = set()
+ loaded_modules = {}
+
+ for modinfo in pkgutil.walk_packages(pySim.__path__, prefix='pySim.'):
+ modname = modinfo.name
+ try:
+ module = importlib.import_module(modname)
+ except Exception: # skip inport errors, if any
+ continue
+ loaded_modules[modname] = module
+
+ for name, cls in inspect.getmembers(module, inspect.isclass):
+ # skip classes that are merely imported by this module
+ if cls.__module__ != modname:
+ continue
+ # examine only subclasses of CardProfile and CardApplication
+ if not issubclass(cls, (CardProfile, CardApplication)):
+ continue
+ # skip the abstract base classes themselves
+ if cls in (CardProfile, CardApplication):
+ continue
+ # classes that require constructor arguments cannot be probed
+ try:
+ obj = cls()
+ except Exception:
+ continue
+
+ # collect all CardDF types reachable from this
profile/application
+ # (used below to identify standalone DFs)
+ reachable_df_types |= self._collect_reachable_df_types(obj)
+
+ if self._has_content(obj) and (modname, name) not in
accounted_for:
+ uncovered.append((modname, name))
+
+ # check standalone CardDFs (such as DF.EIRENE or DF.SYSTEM)
+ for modname, module in loaded_modules.items():
+ for name, cls in inspect.getmembers(module, inspect.isclass):
+ if cls.__module__ != modname:
+ continue
+ if not issubclass(cls, CardDF):
+ continue
+ if cls in self._DF_BASE_TYPES:
+ continue
+ if cls in reachable_df_types:
+ continue
+ try:
+ obj = cls()
+ except Exception:
+ continue
+ if obj.children and (modname, name) not in accounted_for:
+ uncovered.append((modname, name))
+
+ if uncovered:
+ lines = [
+ 'The following classes have EFs/DFs, but not listed in
SECTIONS or EXCLUDED:',
+ *(f' {modname}.{name}' for modname, name in
sorted(uncovered)),
+ 'Please modify docs/pysim_fs_sphinx.py accordingly',
+ ]
+ self.fail('\n'.join(lines))
+
+
+if __name__ == '__main__':
+ unittest.main()
--
To view, visit https://gerrit.osmocom.org/c/pysim/+/42551?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.osmocom.org/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pysim
Gerrit-Branch: master
Gerrit-Change-Id: I06ddeefc6c11e04d7c24e116f3f39c8a6635856f
Gerrit-Change-Number: 42551
Gerrit-PatchSet: 5
Gerrit-Owner: fixeria <[email protected]>
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: dexter <[email protected]>
Gerrit-Reviewer: laforge <[email protected]>
