> Enrico Forestieri wrote:
> > On Mon, Apr 04, 2011 at 05:45:28PM +0200, venom00 wrote:
> >
> > > > do you remember my fear of "only one man understands this
> > > > code" situation in the
> > > > very beginning? :)
> > >
> > > Well, Enrico made useful comments about the code. There
> were a pair of stupid
> > > errors, Julien didn't notice them, so maybe he gave a
> quick read. However any
> > > concrete suggestion or question is welcome. I'll give
> another look at the code
> > > and I'll try to simplify it, if possible, however each
> line is commented.
> >
> > The code should not be a problem if one knows python a bit. However,
> > given that the logic is becoming more involved, one can
> easily miss the
> > big picture and wonder why things are done that way. So, I
> suggest to
> > clearly document at the top of lyxpreview2bitmap.py the
> problems and the
> > strategies adopted for overcoming them.
>
> we are eagerly waiting for the patch :)
Here it is. I've added a general, step-by-step description of what the script
does to the top of the files and some comments here and there. I've moved out of
main functions some code so it's more readable and fixed the FIXME I put which
prevented to use pdflatex as latex executable.
It's tested on the math manual and [examples/instant-preview.lyx], with and
without hyperref.
However looking at the scripts, I think my code is much more understandable than
other code around those files, like the color_pdf without a comment and putting
some mysterious LaTeX code on a single line. But probably my opinion is biased.
venom00
Index: lib/scripts/lyxpreview2bitmap.py
===================================================================
--- lib/scripts/lyxpreview2bitmap.py (revisione 38255)
+++ lib/scripts/lyxpreview2bitmap.py (copia locale)
@@ -19,6 +19,7 @@
# * A latex executable;
# * preview.sty;
# * dvipng;
+# * dv2dt;
# * pngtoppm (if outputing ppm format images).
# preview.sty and dvipng are part of the preview-latex project
@@ -46,6 +47,22 @@
# * a file BASE.metrics, containing info needed by LyX to position
# the images correctly on the screen.
+# What does this script do?
+# 1) Call latex/pdflatex/xelatex/whatever (CONVERTER parameter)
+# 2) If the output is a PDF fallback to legacy
+# 3) Otherwise check each page of the DVI (with dv2dt) looking for
+# PostScript literals, not well supported by dvipng. Pages
+# containing them are passed to the legacy method in a new LaTeX file.
+# 4) Call dvipng on the pages without PS literals
+# 5) Join metrics info coming from both methods (legacy and dvipng)
+# and write them to file
+
+# dvipng is fast but gives problem in several cases, like with
+# PSTricks, TikZ and other packages using PostScript literals
+# for all these cases the legacy route is taken (step 3).
+# Moreover dvipng can't work with PDF files, so, if the CONVERTER
+# paramter is pdflatex we have to fallback to legacy route (step 2).
+
import glob, os, re, string, sys
from legacy_lyxpreview2ppm import legacy_conversion, \
@@ -59,6 +76,8 @@
# Pre-compiled regular expressions.
latex_file_re = re.compile("\.tex$")
+# PATH environment variable
+path = string.split(os.environ["PATH"], os.pathsep)
def usage(prog_name):
return "Usage: %s <format> <latex file> <dpi> <fg color> <bg color>\n"\
@@ -163,65 +182,11 @@
ppm.write(p2p_stdout)
os.remove(png_file)
-
-def main(argv):
- # Parse and manipulate the command line arguments.
- if len(argv) != 6 and len(argv) != 7:
- error(usage(argv[0]))
-
- output_format = string.lower(argv[1])
-
- dir, latex_file = os.path.split(argv[2])
- if len(dir) != 0:
- os.chdir(dir)
-
- dpi = string.atoi(argv[3])
- fg_color = make_texcolor(argv[4], False)
- bg_color = make_texcolor(argv[5], False)
-
- fg_color_gr = make_texcolor(argv[4], True)
- bg_color_gr = make_texcolor(argv[5], True)
-
- # External programs used by the script.
- path = string.split(os.environ["PATH"], os.pathsep)
- if len(argv) == 7:
- latex = argv[6]
- else:
- latex = find_exe_or_terminate(["latex", "pplatex", "platex",
"latex2e"], path)
-
- # This can go once dvipng becomes widespread.
- dvipng = find_exe(["dvipng"], path)
- if dvipng == None:
- # The data is input to legacy_conversion in as similar
- # as possible a manner to that input to the code used in
- # LyX 1.3.x.
- vec = [ argv[0], argv[2], argv[3], argv[1], argv[4], argv[5], latex ]
- return legacy_conversion(vec)
-
- pngtopnm = ""
- if output_format == "ppm":
- pngtopnm = find_exe_or_terminate(["pngtopnm"], path)
-
- # Move color information for PDF into the latex file.
- if not color_pdf(latex_file, bg_color_gr, fg_color_gr):
- error("Unable to move color info into the latex file")
-
- # Compile the latex file.
- latex_call = '%s "%s"' % (latex, latex_file)
-
- latex_status, latex_stdout = run_command(latex_call)
- if latex_status != None:
- warning("%s had problems compiling %s" \
- % (os.path.basename(latex), latex_file))
-
- if latex == "xelatex":
- warning("Using XeTeX")
- # FIXME: skip unnecessary dvips trial in legacy_conversion_step2
- return legacy_conversion_step2(latex_file, dpi, output_format)
-
- # The dvi output file name
- dvi_file = latex_file_re.sub(".dvi", latex_file)
-
+# Returns a tuple of:
+# ps_pages: list of page indexes of pages containing PS literals
+# page_count: total number of pages
+# pages_parameter: parameter for dvipng to exclude pages with PostScript
+def find_ps_pages(dvi_file):
# latex failed
# FIXME: try with pdflatex
if not os.path.isfile(dvi_file):
@@ -257,14 +222,10 @@
if psliteral_re.match(line) != None:
# Literal PostScript special detected!
page_has_ps = True
-
+
+ # Create the -pp parameter for dvipng
pages_parameter = ""
-
- if len(ps_pages) == page_index:
- # All pages need PostScript, so directly use the legacy method.
- vec = [argv[0], argv[2], argv[3], argv[1], argv[4], argv[5], latex]
- return legacy_conversion(vec)
- elif len(ps_pages) > 0:
+ if len(ps_pages) > 0 and len(ps_pages) < page_index:
# Don't process Postscript pages with dvipng by selecting the
# wanted pages through the -pp parameter. E.g., dvipng -pp 4-12,14,64
pages_parameter = " -pp "
@@ -301,20 +262,105 @@
if (not index in ps_pages) and (not skip) and (last != index):
pages_parameter += "-" + str(index)
+ return (ps_pages, page_index, pages_parameter)
+
+def main(argv):
+ # Parse and manipulate the command line arguments.
+ if len(argv) != 6 and len(argv) != 7:
+ error(usage(argv[0]))
+
+ output_format = string.lower(argv[1])
+
+ dir, latex_file = os.path.split(argv[2])
+ if len(dir) != 0:
+ os.chdir(dir)
+
+ dpi = string.atoi(argv[3])
+ fg_color = make_texcolor(argv[4], False)
+ bg_color = make_texcolor(argv[5], False)
+
+ fg_color_gr = make_texcolor(argv[4], True)
+ bg_color_gr = make_texcolor(argv[5], True)
+
+ # External programs used by the script.
+ if len(argv) == 7:
+ latex = argv[6]
+ else:
+ latex = find_exe_or_terminate(["latex", "pplatex", "platex",
"latex2e"], path)
+
+ # This can go once dvipng becomes widespread.
+ dvipng = find_exe(["dvipng"], path)
+ if dvipng == None:
+ # The data is input to legacy_conversion in as similar
+ # as possible a manner to that input to the code used in
+ # LyX 1.3.x.
+ vec = [ argv[0], argv[2], argv[3], argv[1], argv[4], argv[5], latex ]
+ return legacy_conversion(vec)
+
+ pngtopnm = ""
+ if output_format == "ppm":
+ pngtopnm = find_exe_or_terminate(["pngtopnm"], path)
+
+ # Move color information for PDF into the latex file.
+ if not color_pdf(latex_file, bg_color_gr, fg_color_gr):
+ error("Unable to move color info into the latex file")
+
+ # Compile the latex file.
+ latex_call = '%s "%s"' % (latex, latex_file)
+
+ latex_status, latex_stdout = run_command(latex_call)
+ if latex_status != None:
+ warning("%s had problems compiling %s" \
+ % (os.path.basename(latex), latex_file))
+
+ if latex == "xelatex":
+ warning("Using XeTeX")
+ # FIXME: skip unnecessary dvips trial in legacy_conversion_step2
+ return legacy_conversion_step2(latex_file, dpi, output_format)
+
+ # The dvi output file name
+ dvi_file = latex_file_re.sub(".dvi", latex_file)
+
+ # If there's no DVI output, look for PDF and go to legacy or fail
+ if not os.path.isfile(dvi_file):
+ # No DVI, is there a PDF?
+ pdf_file = latex_file_re.sub(".pdf", latex_file)
+ if os.path.isfile(pdf_file):
+ warning("%s produced a PDF output, fallback to legacy." % \
+ (os.path.basename(latex)))
+ return legacy_conversion_step2(latex_file, dpi, output_format)
+ else:
+ error("No DVI or PDF output. %s failed." \
+ % (os.path.basename(latex)))
+
+ # Look for PS literals in DVI pages
+ # ps_pages: list of page indexes of pages containing PS literals
+ # page_count: total number of pages
+ # pages_parameter: parameter for dvipng to exclude pages with PostScript
+ (ps_pages, page_count, pages_parameter) = find_ps_pages(dvi_file)
+
+ # If all pages need PostScript, directly use the legacy method.
+ if len(ps_pages) == page_count:
+ vec = [argv[0], argv[2], argv[3], argv[1], argv[4], argv[5], latex]
+ return legacy_conversion(vec)
+
# Run the dvi file through dvipng.
dvipng_call = '%s -Ttight -depth -height -D %d -fg "%s" -bg "%s" %s "%s"' \
% (dvipng, dpi, fg_color, bg_color, pages_parameter,
dvi_file)
dvipng_status, dvipng_stdout = run_command(dvipng_call)
if dvipng_status != None:
- warning("%s failed to generate images from %s ... looking for PDF" \
+ warning("%s failed to generate images from %s... fallback to legacy
method" \
% (os.path.basename(dvipng), dvi_file))
# FIXME: skip unnecessary dvips trial in legacy_conversion_step2
return legacy_conversion_step2(latex_file, dpi, output_format)
- dvipng_metrics = []
+ # Extract metrics info from dvipng_stdout.
+ metrics_file = latex_file_re.sub(".metrics", latex_file)
+ dvipng_metrics = extract_metrics_info(dvipng_stdout)
+
+ # If some pages require PostScript pass them to legacy method
if len(ps_pages) > 0:
- # Some pages require PostScript.
# Create a new LaTeX file just for the snippets needing
# the legacy method
legacy_latex_file = latex_file_re.sub("_legacy.tex", latex_file)
@@ -326,9 +372,6 @@
legacy_metrics = legacy_conversion(vec, True)[1]
# Now we need to mix metrics data from dvipng and the legacy method
- metrics_file = latex_file_re.sub(".metrics", latex_file)
- dvipng_metrics = extract_metrics_info(dvipng_stdout)
-
original_bitmap = latex_file_re.sub("%d." + output_format,
legacy_latex_file)
destination_bitmap = latex_file_re.sub("%d." + output_format,
latex_file)
@@ -336,13 +379,6 @@
join_metrics_and_rename(dvipng_metrics, legacy_metrics, ps_pages,
original_bitmap, destination_bitmap)
- else:
- # Extract metrics info from dvipng_stdout.
- # In this case we just used dvipng, so no special metrics
- # handling is needed.
- metrics_file = latex_file_re.sub(".metrics", latex_file)
- dvipng_metrics = extract_metrics_info(dvipng_stdout)
-
# Convert images to ppm format if necessary.
if output_format == "ppm":
convert_to_ppm_format(pngtopnm, latex_file_re.sub("", latex_file))
Index: lib/scripts/legacy_lyxpreview2ppm.py
===================================================================
--- lib/scripts/legacy_lyxpreview2ppm.py (revisione 38255)
+++ lib/scripts/legacy_lyxpreview2ppm.py (copia locale)
@@ -15,7 +15,7 @@
# Paul A. Rubin, ru...@msu.edu.
# This script takes a LaTeX file and generates a collection of
-# ppm image files, one per previewed snippet.
+# png or ppm image files, one per previewed snippet.
# Example usage:
# legacy_lyxpreview2bitmap.py 0lyxpreview.tex 128 ppm 000000 faf0e6
@@ -31,7 +31,7 @@
# Decomposing TEXFILE's name as DIR/BASE.tex, this script will,
# if executed successfully, leave in DIR:
# * a (possibly large) number of image files with names
-# like BASE[0-9]+.ppm
+# like BASE[0-9]+.(ppm|png)
# * a file BASE.metrics, containing info needed by LyX to position
# the images correctly on the screen.
@@ -40,6 +40,7 @@
# * preview.sty;
# * dvips;
# * gs;
+# * pdflatex (optional);
# * pnmcrop (optional).
# preview.sty is part of the preview-latex project
@@ -47,12 +48,30 @@
# Alternatively, it can be obtained from
# CTAN/support/preview-latex/
-# The script uses the deprecated dvi->ps->ppm conversion route.
-# If possible, please grab 'dvipng'; it's faster and more robust.
-# If you have it then this script will not be invoked by
-# lyxpreview2bitmap.py.
-# Warning: this legacy support will be removed one day...
+# What does this script do?
+# [legacy_conversion]
+# 1) Call latex to create a DVI file from LaTeX
+# [legacy_conversion_step2]
+# 2) Call dvips to create one PS file for each DVI page
+# 3) If dvips fails look for PDF and call gs to produce bitmaps
+# 4) Otherwise call gs on each PostScript file to produce bitmaps
+# [legacy_conversion_pdflatex]
+# 5) Keep track of pages on which gs failed and pass them to pdflatex
+# 6) Call gs on the PDF output from pdflatex to produce bitmaps
+# 7) Extract and write to file (or return to lyxpreview2bitmap)
+# metrics from both methods (standard and pdflatex)
+# The script uses the old dvi->ps->png conversion route,
+# which is good when using PSTricks, TikZ or other packages involving
+# PostScript literals (steps 1, 2, 4).
+# This script also generates bitmaps from PDF created by a call to
+# lyxpreview2bitmap.py passing "pdflatex" to the CONVERTER parameter
+# (step 3).
+# Finally, there's also has a fallback method based on pdflatex, which
+# is required in certain cases, if hyperref is active for instance,
+# (step 5, 6).
+# If possible, dvipng should be used, as it's much faster.
+
import glob, os, pipes, re, string, sys
from lyxpreview_tools import copyfileobj, error, find_exe, \
@@ -62,6 +81,8 @@
# Pre-compiled regular expression.
latex_file_re = re.compile("\.tex$")
+# PATH environment variable
+path = string.split(os.environ["PATH"], os.pathsep)
def usage(prog_name):
return "Usage: %s <latex file> <dpi> ppm <fg color> <bg color>\n"\
@@ -249,7 +270,6 @@
bg_color_gr = make_texcolor(argv[5], True)
# External programs used by the script.
- path = string.split(os.environ["PATH"], os.pathsep)
latex = find_exe_or_terminate(latex_commands, path)
# Move color information into the latex file.
@@ -266,10 +286,53 @@
return legacy_conversion_step2(latex_file, dpi, output_format, skipMetrics)
+# Creates a new LaTeX file from the original with pages specified in
+# failed_pages, pass it through pdflatex and updates the metrics
+# from the standard legacy route
+def legacy_conversion_pdflatex(latex_file, failed_pages, legacy_metrics, gs,
+ gs_device, gs_ext, alpha, resolution, output_format):
+
+ # Search for pdflatex executable
+ pdflatex = find_exe(["pdflatex"], path)
+ if pdflatex == None:
+ warning("Can't find pdflatex. Some pages failed with all the possible
routes.")
+ else:
+ # Create a new LaTeX file from the original but only with failed pages
+ pdf_latex_file = latex_file_re.sub("_pdflatex.tex", latex_file)
+ filter_pages(latex_file, pdf_latex_file, failed_pages)
+
+ # pdflatex call
+ pdflatex_call = '%s "%s"' % (pdflatex, pdf_latex_file)
+ pdflatex_status, pdflatex_stdout = run_command(pdflatex_call)
+
+ pdf_file = latex_file_re.sub(".pdf", pdf_latex_file)
+
+ # GhostScript call to produce bitmaps
+ gs_call = '%s -dNOPAUSE -dBATCH -dSAFER -sDEVICE=%s ' \
+ '-sOutputFile="%s%%d.%s" ' \
+ '-dGraphicsAlphaBit=%d -dTextAlphaBits=%d ' \
+ '-r%f "%s"' \
+ % (gs, gs_device, latex_file_re.sub("", pdf_latex_file), \
+ gs_ext, alpha, alpha, resolution, pdf_file)
+ gs_status, gs_stdout = run_command(gs_call)
+ if gs_status != None:
+ # Give up!
+ warning("Some pages failed with all the possible routes")
+ else:
+ # We've done it!
+ pdf_log_file = latex_file_re.sub(".log", pdf_latex_file)
+ pdf_metrics = legacy_extract_metrics_info(pdf_log_file)
+
+ original_bitmap = latex_file_re.sub("%d." + output_format,
pdf_latex_file)
+ destination_bitmap = latex_file_re.sub("%d." + output_format,
latex_file)
+
+ # Join the metrics with the those from dvips and rename the bitmap
images
+ join_metrics_and_rename(legacy_metrics, pdf_metrics, failed_pages,
+ original_bitmap, destination_bitmap)
+
def legacy_conversion_step2(latex_file, dpi, output_format, skipMetrics =
False):
# External programs used by the script.
- path = string.split(os.environ["PATH"], os.pathsep)
dvips = find_exe_or_terminate(["dvips"], path)
gs = find_exe_or_terminate(["gswin32c", "gs"], path)
pnmcrop = find_exe(["pnmcrop"], path)
@@ -312,6 +375,7 @@
# Generate the bitmap images
if dvips_failed:
+ # dvips failed, maybe there's a PDF, try to produce bitmaps
gs_call = '%s -dNOPAUSE -dBATCH -dSAFER -sDEVICE=%s ' \
'-sOutputFile="%s%%d.%s" ' \
'-dGraphicsAlphaBit=%d -dTextAlphaBits=%d ' \
@@ -323,62 +387,32 @@
if gs_status != None:
error("Failed: %s %s" % (os.path.basename(gs), ps_file))
else:
+ # Model for calling gs on each file
gs_call = '%s -dNOPAUSE -dBATCH -dSAFER -sDEVICE=%s ' \
'-sOutputFile="%s%%d.%s" ' \
'-dGraphicsAlphaBit=%d -dTextAlphaBits=%d ' \
'-r%f "%%s"' \
% (gs, gs_device, latex_file_re.sub("", latex_file), \
gs_ext, alpha, alpha, resolution)
+
i = 0
+ # Collect all the PostScript files (like *.001, *.002, ...)
ps_files = glob.glob("%s.[0-9][0-9][0-9]" % latex_file_re.sub("",
latex_file))
ps_files.sort()
- # Call GhostScript for each page
+ # Call GhostScript for each file
for file in ps_files:
i = i + 1
gs_status, gs_stdout = run_command(gs_call % (i, file))
if gs_status != None:
# gs failed, keep track of this
failed_pages.append(i)
-
# Pass failed pages to pdflatex
if len(failed_pages) > 0:
- pdflatex = find_exe(["pdflatex"], path)
- if pdflatex != None:
- # Create a new LaTeX file from the original but only with failed
pages
- pdf_latex_file = latex_file_re.sub("_pdflatex.tex", latex_file)
- filter_pages(latex_file, pdf_latex_file, failed_pages)
-
- # pdflatex call
- pdflatex_call = '%s "%s"' % (pdflatex, pdf_latex_file)
- pdflatex_status, pdflatex_stdout = run_command(pdflatex_call)
-
- pdf_file = latex_file_re.sub(".pdf", pdf_latex_file)
-
- # GhostScript call to produce bitmaps
- gs_call = '%s -dNOPAUSE -dBATCH -dSAFER -sDEVICE=%s ' \
- '-sOutputFile="%s%%d.%s" ' \
- '-dGraphicsAlphaBit=%d -dTextAlphaBits=%d ' \
- '-r%f "%s"' \
- % (gs, gs_device, latex_file_re.sub("", pdf_latex_file),
\
- gs_ext, alpha, alpha, resolution, pdf_file)
+ legacy_conversion_pdflatex(latex_file, failed_pages, legacy_metrics,
gs,
+ gs_device, gs_ext, alpha, resolution, output_format)
- gs_status, gs_stdout = run_command(gs_call)
- if gs_status != None:
- # Give up!
- warning("Some pages failed with all the possible routes")
- else:
- # We've done it!
- pdf_log_file = latex_file_re.sub(".log", pdf_latex_file)
- pdf_metrics = legacy_extract_metrics_info(pdf_log_file)
-
- original_bitmap = latex_file_re.sub("%d." + output_format,
pdf_latex_file)
- destination_bitmap = latex_file_re.sub("%d." + output_format,
latex_file)
-
- # Join the metrics with the those from dvips and rename the
bitmap images
- join_metrics_and_rename(legacy_metrics, pdf_metrics,
failed_pages, original_bitmap, destination_bitmap)
-
# Crop the images
if pnmcrop != None:
crop_files(pnmcrop, latex_file_re.sub("", latex_file))
