#! /usr/bin/env python

usage:
    lpy [-options(s)] filename(s)

writes:
    filename.html (beware!)

options:
    -autoformat = for processing python source with no formatting cues
    -autosplit = inserts split directives automatically
    -nosplits = disables split directives
    -noindex = disable automatic index
    -printing = uses smaller fonts

download lpy.zip

for more information, contact: Danbala Software

    __Id__ = "$Id: lpy.py.html,v 1.1 2005/01/26 01:13:52 u37519820 Exp $"

Table of Contents

• Introduction
• Imports
• main
• Token class
  • Token_ClassInit
  • Token.__init__
  • Token.asHTML
• XRef class
• autohtml
• escapes4html
• Kick Off
• Index

lpy.py is a tool for code publishing inspired by Knuth's Literate Programming concepts. It's purpose is to make code presentable for reading.

Practically speaking, it's yet-another-Python-to-HTML too, with additions.

Other than regular Python colorizing, we're looking for some special stuff in comments and docstrings. A comment or string starting with ~ is pulled out and passed through a macro engine. The macro engine looks for the @ sign as a macro marker, using several syntaxes:

    @symbol - can be a variable replacement or function call (no arguments)
    @symbol{string} - call a function (or % a string) with one literal string
    @symbol(args) - call a function with args
    @{any Python code} - execute any Python
    @(any Python expression) - evaluate any Python and display result

The Python functions invoked can return a result string or print (to sys.stdout).
 See pymacro for more information about the macro engine.

The macro set has some simplified html macros, like @i{to italicize} to italicize, for example. HTML is simple widely known, but it is verbose, and put lots of noise in the input stream. The macros are intended to make the source tidier. See htmlmax for information about the macro set.

We're also looking for comments or strings starting with which maps to early instead of late execution. The ~ stuff is processed on the way out (late), and the @ on the way in (early), so the @ directives occuring later in the source can affect the behavior of the ~ directives occurring earlier in the source. The contents macros use this mechanism.

One other little thing - any comment starting with #- is replaced with <hr>.

Traditional LP involves writing a mixture of formatting and code, and then extracting the program and document from the source using separate programs ('tangle' and 'weave').

The term inverted literate programming describes producing the document from the actual program source code instead of from a separate source file. That's what this tool does. The concept of having to preprocess a Python script file in order to execute does seem perverse.

There are other useful approaches to LP - eventually there could be a wysiwyg / hyperlinked / outlining / structured / browser coding-and-documenting environment that really understands the program, in order to help the programmer or reader make sense out of it.

This tool is a lot less ambitious, it's simply to help me write better code. Writing readable programs is more fun (and more difficult and time-consuming).

Imports should come first - using from module import *, our symbols can get stomped on.

Imports

    import sys      # we're using sys.argv
    import string   # and some string manipulations

    from pytokens import *
    #Parse, WHITE, NEWLINE, NL, INDENT, DEDENT, OP, COMMENT, STRING, RESERVED

    import pymacro

    pymacro.load('htmlmax')

    from pymacro import html_prefix, html_suffix, html_wrap, gWraps

Invoked without arguments, print the __doc__ string as the usage message.

    def main(argv):
        "for command line invocation - gimme sys.argv[1:]"
        if not len(argv): 
            print __doc__   # usage info
            return
        
        for arg in argv:
            if arg[0] == '-':   # command line - convert to @
                s = pymacro.pymax_process('@SetFlag{'+arg[1:]+'}')
                if s:           # report any output
                    print s
            else:
                PyToHTML(arg, arg+".html")

    def PyToHTML(inputfilename, outputfilename):
        """Convert a Python file to an HTML file"""

        input  = open(inputfilename, 'r')
        output = open(outputfilename, 'w')

        Token_ClassInit()
        tokens = Parse(input, Token)

        output.write(html_prefix(inputfilename))
        if Token.autosplit:
            output.write(pymacro.pymax_process('@startsplit'))
        for t in tokens:
            output.write(t.asHTML())    
        output.write(html_suffix())
        if Token.autosplit:
            output.write(pymacro.pymax_process('@endsplit'))
        if not pymacro.TestFlag('noindex'):
            output.write(Token.xref.asHTML())

Token class

    def Token_ClassInit():

        Token.previous = 0      # for tracking our linked list
        Token.indentlevel = 0   # for tracking syntax levels
        Token.parenlevel = 0
        Token.bracketlevel = 0    
        Token.bracelevel = 0    
        Token.afternewline = 1  # if we came right after a newline
        Token.firstonline = 1   # need to know if we are the first real item
    
        # options:                            
        Token.autoformat = pymacro.TestFlag('autoformat')
        Token.autosplit = pymacro.TestFlag('autosplit')
        Token.xref = XRef()     # symbol cross reference

• Tracking each code element,
• determining if it is 'special' and needs to be passed to the pymacro engine
• syntax coloring of STRING, COMMENT, and RESERVED words,
• Line numbers, and other HTML formatting

    class Token:
        """ Token items are spewed out by the pytokens module """

        def asHTML(self):
            """ render this Token asHTML """
    
            s = self.isSpecial # well, isn't that special?
            if s:
                if not self.preprocessed:
                    s = autohtml(pymacro.pymax_process(escapes4html(s)))
                if 0:   # doesn't work as well as hoped - dedents come too late
                    t = self.indentlevel + 1
                    if self.indentlevel:
                        s = '<ul>' * t + s + '</ul>' * t
                return html_wrap(self.leading, self.trailing, s)
                
            s = ""
            # put out the line numbers
            if (self.token or self.type==DEDENT) and self.isAfterNewLine:
                s = ('<a name="%d">%4d</a>    ' % (self.line, self.line))
    
            type = self.type
            token = self.token
            if type > RESERVED:
                type = RESERVED
            wrap = gWraps.get(type,None)
    
            # check for strings with embedded newlines
            if type == STRING and string.count(token, '\n') > 1:
                t = escapes4html(string.join(
                                   string.split(token, '\n'), '\n        '))
                if wrap:
                    t = wrap % t
                s = s + t + '\n'
            else:
                t = escapes4html(token)
                if wrap:
                    t = wrap % t
                s = s + t
            return s

    class XRef:
        """ XRef maintains a cross reference of symbols by line number """
        
        def __init__(self):

            self.symbols = {}
                    
        def __call__(self, token, line):

            symbols = self.symbols
            if symbols.has_key(token):
                if not symbols[token].count(line): # already on this line
                    symbols[token].append(line)
            else:
                symbols[token] = [line]
            
        def asHTML(self):

            refs = self.symbols
            # two columns, each containing a table of two columns
            r = '<hr><ul><table><tr><td valign=top><table>'
            nk = refs.keys()
            nk.sort()
            bp = (len(nk) + 2) / 3
            i = 0
            for n in nk:
                if i and i % bp == 0:
                    r = r + '</table></td><td valign=top><table>'
                i = i + 1
                r = '%s<tr><td valign=top class="xref">%s</td><td valign=top class="xref">' % (r, n)
                s = ''
                for l in refs[n]:
                    s = '%s <a href="#%d">%d</a> ' % (s, l, l)
                r = r + s + '</td></tr>\n'
            r = r + '</table></td></tr></table></ul>'
            return r

    def autohtml(s):
        s = string.join(string.split(s, '\n\n'), '\n<p>\n')
        
        while 1:
            i = string.find(s, '\n ')
            if i >= 0:
                # beware of pseudo-blank lines and all space lines
                c = 1
                x = i + c + 1
                l = len(s)
                while x < l and s[x] == ' ':
                    c = c + 1
                    x = x + 1
                if i > 0 and s[i-1] == '>': # watch for <p><br>
                    s = s[0:i+1] + c * "&nbsp;" + s[i+c+1:]
                else:
                    s = s[0:i] + "<br>\n" +  c * "&nbsp;" + s[i+c+1:]
            else:
                break
             
        #s = string.join(string.split(s, '\n '), '\n<br>&nbsp;')
        return s

    def escapes4html(s):
        return string.replace(string.replace(s,'&','&amp;'),'<','&lt;')

    if __name__ == '__main__':  
        main(sys.argv[1:])