mirror of
https://github.com/Relintai/scons_gd.git
synced 2025-02-10 16:40:14 +01:00
644 lines
20 KiB
Python
644 lines
20 KiB
Python
#!/usr/bin/env python
|
|
#
|
|
# Copyright (c) 2010 The SCons Foundation
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining
|
|
# a copy of this software and associated documentation files (the
|
|
# "Software"), to deal in the Software without restriction, including
|
|
# without limitation the rights to use, copy, modify, merge, publish,
|
|
# distribute, sublicense, and/or sell copies of the Software, and to
|
|
# permit persons to whom the Software is furnished to do so, subject to
|
|
# the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included
|
|
# in all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
|
|
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
|
|
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
#
|
|
#
|
|
# Module for handling SCons documentation processing.
|
|
#
|
|
|
|
__doc__ = r"""
|
|
This module parses home-brew XML files that document various things
|
|
in SCons. Right now, it handles Builders, functions, construction
|
|
variables, and Tools, but we expect it to get extended in the future.
|
|
|
|
In general, you can use any DocBook tag in the input, and this module
|
|
just adds processing various home-brew tags to try to make life a
|
|
little easier.
|
|
|
|
Builder example:
|
|
|
|
<builder name="BUILDER">
|
|
<summary>
|
|
<para>This is the summary description of an SCons Builder.
|
|
It will get placed in the man page,
|
|
and in the appropriate User's Guide appendix.
|
|
The name of this builder may be interpolated
|
|
anywhere in the document by specifying the
|
|
&b-BUILDER; element. A link to this definition may be
|
|
interpolated by specifying the &b-link-BUILDER; element.
|
|
</para>
|
|
|
|
Unlike normal XML, blank lines are significant in these
|
|
descriptions and serve to separate paragraphs.
|
|
They'll get replaced in DocBook output with appropriate tags
|
|
to indicate a new paragraph.
|
|
|
|
<example>
|
|
print("this is example code, it will be offset and indented")
|
|
</example>
|
|
</summary>
|
|
</builder>
|
|
|
|
Function example:
|
|
|
|
<scons_function name="FUNCTION">
|
|
<arguments signature="SIGTYPE">
|
|
(arg1, arg2, key=value)
|
|
</arguments>
|
|
<summary>
|
|
<para>This is the summary description of an SCons function.
|
|
It will get placed in the man page,
|
|
and in the appropriate User's Guide appendix.
|
|
If the "signature" attribute is specified, SIGTYPE may be one
|
|
of "global", "env" or "both" (the default if omitted is "both"),
|
|
to indicate the signature applies to the global form or the
|
|
environment form, or to generate both with the same signature
|
|
(excepting the insertion of "env.").
|
|
This allows for the cases of
|
|
describing that only one signature should be generated,
|
|
or both signatures should be generated and they differ,
|
|
or both signatures should be generated and they are the same.
|
|
The name of this function may be interpolated
|
|
anywhere in the document by specifying the
|
|
&f-FUNCTION; element or the &f-env-FUNCTION; element.
|
|
Links to this definition may be interpolated by specifying
|
|
the &f-link-FUNCTION: or &f-link-env-FUNCTION; element.
|
|
</para>
|
|
|
|
<example>
|
|
print("this is example code, it will be offset and indented")
|
|
</example>
|
|
</summary>
|
|
</scons_function>
|
|
|
|
Construction variable example:
|
|
|
|
<cvar name="VARIABLE">
|
|
<summary>
|
|
<para>This is the summary description of a construction variable.
|
|
It will get placed in the man page,
|
|
and in the appropriate User's Guide appendix.
|
|
The name of this construction variable may be interpolated
|
|
anywhere in the document by specifying the
|
|
&cv-VARIABLE; element. A link to this definition may be
|
|
interpolated by specifying the &cv-link-VARIABLE; element.
|
|
</para>
|
|
|
|
<example>
|
|
print("this is example code, it will be offset and indented")
|
|
</example>
|
|
</summary>
|
|
</cvar>
|
|
|
|
Tool example:
|
|
|
|
<tool name="TOOL">
|
|
<summary>
|
|
<para>This is the summary description of an SCons Tool.
|
|
It will get placed in the man page,
|
|
and in the appropriate User's Guide appendix.
|
|
The name of this tool may be interpolated
|
|
anywhere in the document by specifying the
|
|
&t-TOOL; element. A link to this definition may be
|
|
interpolated by specifying the &t-link-TOOL; element.
|
|
</para>
|
|
|
|
<example>
|
|
print("this is example code, it will be offset and indented")
|
|
</example>
|
|
</summary>
|
|
</tool>
|
|
"""
|
|
|
|
import os.path
|
|
import re
|
|
import sys
|
|
import copy
|
|
import importlib
|
|
|
|
try:
|
|
from lxml import etree
|
|
except ImportError:
|
|
try:
|
|
import xml.etree.ElementTree as etree
|
|
except ImportError:
|
|
raise ImportError("Failed to import ElementTree from any known place")
|
|
|
|
# patterns to help trim XML passed in as strings
|
|
re_entity = re.compile(r"&([^;]+);")
|
|
re_entity_header = re.compile(r"<!DOCTYPE\s+sconsdoc\s+[^\]]+\]>")
|
|
|
|
# Namespace for the SCons Docbook XSD
|
|
dbxsd = "http://www.scons.org/dbxsd/v1.0"
|
|
# Namsespace pattern to help identify an scons-xml file read as bytes
|
|
dbxsdpat = b'xmlns="%s"' % dbxsd.encode('utf-8')
|
|
# Namespace map identifier for the SCons Docbook XSD
|
|
dbxid = "dbx"
|
|
# Namespace for schema instances
|
|
xsi = "http://www.w3.org/2001/XMLSchema-instance"
|
|
|
|
# Header comment with copyright
|
|
copyright_comment = """
|
|
__COPYRIGHT__
|
|
|
|
This file is processed by the bin/SConsDoc.py module.
|
|
See its docstring for a discussion of the format.
|
|
"""
|
|
|
|
def isSConsXml(fpath):
|
|
""" Check whether the given file is an SCons XML file.
|
|
|
|
It is SCons XML if it contains the default target namespace definition
|
|
described by dbxsdpat
|
|
|
|
"""
|
|
try:
|
|
with open(fpath, 'rb') as f:
|
|
content = f.read()
|
|
if content.find(dbxsdpat) >= 0:
|
|
return True
|
|
except Exception:
|
|
pass
|
|
|
|
return False
|
|
|
|
def remove_entities(content):
|
|
# Cut out entity inclusions
|
|
content = re_entity_header.sub("", content, re.M)
|
|
# Cut out entities themselves
|
|
content = re_entity.sub(lambda match: match.group(1), content)
|
|
|
|
return content
|
|
|
|
default_xsd = os.path.join('doc', 'xsd', 'scons.xsd')
|
|
|
|
ARG = "dbscons"
|
|
|
|
|
|
class Libxml2ValidityHandler:
|
|
|
|
def __init__(self):
|
|
self.errors = []
|
|
self.warnings = []
|
|
|
|
def error(self, msg, data):
|
|
if data != ARG:
|
|
raise Exception("Error handler did not receive correct argument")
|
|
self.errors.append(msg)
|
|
|
|
def warning(self, msg, data):
|
|
if data != ARG:
|
|
raise Exception("Warning handler did not receive correct argument")
|
|
self.warnings.append(msg)
|
|
|
|
|
|
class DoctypeEntity:
|
|
def __init__(self, name_, uri_):
|
|
self.name = name_
|
|
self.uri = uri_
|
|
|
|
def getEntityString(self):
|
|
txt = """ <!ENTITY %(perc)s %(name)s SYSTEM "%(uri)s">
|
|
%(perc)s%(name)s;
|
|
""" % {'perc': perc, 'name': self.name, 'uri': self.uri}
|
|
|
|
return txt
|
|
|
|
|
|
class DoctypeDeclaration:
|
|
def __init__(self, name_=None):
|
|
self.name = name_
|
|
self.entries = []
|
|
if self.name is None:
|
|
# Add default entries
|
|
self.name = "sconsdoc"
|
|
self.addEntity("scons", "../scons.mod")
|
|
self.addEntity("builders-mod", "builders.mod")
|
|
self.addEntity("functions-mod", "functions.mod")
|
|
self.addEntity("tools-mod", "tools.mod")
|
|
self.addEntity("variables-mod", "variables.mod")
|
|
|
|
def addEntity(self, name, uri):
|
|
self.entries.append(DoctypeEntity(name, uri))
|
|
|
|
def createDoctype(self):
|
|
content = '<!DOCTYPE %s [\n' % self.name
|
|
for e in self.entries:
|
|
content += e.getEntityString()
|
|
content += ']>\n'
|
|
|
|
return content
|
|
|
|
class TreeFactory:
|
|
def __init__(self):
|
|
pass
|
|
|
|
@staticmethod
|
|
def newNode(tag, **kwargs):
|
|
return etree.Element(tag, **kwargs)
|
|
|
|
@staticmethod
|
|
def newSubNode(parent, tag, **kwargs):
|
|
return etree.SubElement(parent, tag, **kwargs)
|
|
|
|
@staticmethod
|
|
def newEtreeNode(tag, init_ns=False, **kwargs):
|
|
if init_ns:
|
|
NSMAP = {None: dbxsd,
|
|
'xsi' : xsi}
|
|
return etree.Element(tag, nsmap=NSMAP, **kwargs)
|
|
|
|
return etree.Element(tag, **kwargs)
|
|
|
|
@staticmethod
|
|
def copyNode(node):
|
|
return copy.deepcopy(node)
|
|
|
|
@staticmethod
|
|
def appendNode(parent, child):
|
|
parent.append(child)
|
|
|
|
@staticmethod
|
|
def hasAttribute(node, att):
|
|
return att in node.attrib
|
|
|
|
@staticmethod
|
|
def getAttribute(node, att):
|
|
return node.attrib[att]
|
|
|
|
@staticmethod
|
|
def setAttribute(node, att, value):
|
|
node.attrib[att] = value
|
|
|
|
@staticmethod
|
|
def getText(root):
|
|
return root.text
|
|
|
|
@staticmethod
|
|
def appendCvLink(root, key, lntail):
|
|
linknode = etree.Entity('cv-link-' + key)
|
|
linknode.tail = lntail
|
|
root.append(linknode)
|
|
|
|
@staticmethod
|
|
def setText(root, txt):
|
|
root.text = txt
|
|
|
|
@staticmethod
|
|
def getTail(root):
|
|
return root.tail
|
|
|
|
@staticmethod
|
|
def setTail(root, txt):
|
|
root.tail = txt
|
|
|
|
@staticmethod
|
|
def writeGenTree(root, fp):
|
|
dt = DoctypeDeclaration()
|
|
fp.write(etree.tostring(root, encoding="utf-8",
|
|
pretty_print=True,
|
|
doctype=dt.createDoctype()).decode('utf-8'))
|
|
|
|
@staticmethod
|
|
def writeTree(root, fpath):
|
|
with open(fpath, 'wb') as fp:
|
|
fp.write(etree.tostring(root, encoding="utf-8",
|
|
pretty_print=True))
|
|
|
|
@staticmethod
|
|
def prettyPrintFile(fpath):
|
|
with open(fpath,'rb') as fin:
|
|
tree = etree.parse(fin)
|
|
pretty_content = etree.tostring(tree, encoding="utf-8",
|
|
pretty_print=True)
|
|
|
|
with open(fpath,'wb') as fout:
|
|
fout.write(pretty_content)
|
|
|
|
@staticmethod
|
|
def decorateWithHeader(root):
|
|
root.attrib["{"+xsi+"}schemaLocation"] = "%s %s/scons.xsd" % (dbxsd, dbxsd)
|
|
return root
|
|
|
|
def newXmlTree(self, root):
|
|
""" Return a XML file tree with the correct namespaces set,
|
|
the element root as top entry and the given header comment.
|
|
"""
|
|
NSMAP = {None: dbxsd, 'xsi' : xsi}
|
|
t = etree.Element(root, nsmap=NSMAP)
|
|
return self.decorateWithHeader(t)
|
|
|
|
# singleton to cache parsed xmlschema..
|
|
xmlschema = None
|
|
|
|
@staticmethod
|
|
def validateXml(fpath, xmlschema_context):
|
|
|
|
if TreeFactory.xmlschema is None:
|
|
TreeFactory.xmlschema = etree.XMLSchema(xmlschema_context)
|
|
try:
|
|
doc = etree.parse(fpath)
|
|
except Exception as e:
|
|
print("ERROR: %s fails to parse:"%fpath)
|
|
print(e)
|
|
return False
|
|
doc.xinclude()
|
|
try:
|
|
TreeFactory.xmlschema.assertValid(doc)
|
|
except etree.XMLSchemaValidateError as e:
|
|
print("ERROR: %s fails to validate:" % fpath)
|
|
print(e)
|
|
print(e.error_log.last_error.message)
|
|
print("In file: [%s]" % e.error_log.last_error.filename)
|
|
print("Line : %d" % e.error_log.last_error.line)
|
|
return False
|
|
|
|
except Exception as e:
|
|
print("ERROR: %s fails to validate:" % fpath)
|
|
print(e)
|
|
|
|
return False
|
|
return True
|
|
|
|
@staticmethod
|
|
def findAll(root, tag, ns=None, xp_ctxt=None, nsmap=None):
|
|
expression = ".//{%s}%s" % (nsmap[ns], tag)
|
|
if not ns or not nsmap:
|
|
expression = ".//%s" % tag
|
|
return root.findall(expression)
|
|
|
|
@staticmethod
|
|
def findAllChildrenOf(root, tag, ns=None, xp_ctxt=None, nsmap=None):
|
|
expression = "./{%s}%s/*" % (nsmap[ns], tag)
|
|
if not ns or not nsmap:
|
|
expression = "./%s/*" % tag
|
|
return root.findall(expression)
|
|
|
|
@staticmethod
|
|
def convertElementTree(root):
|
|
""" Convert the given tree of etree.Element
|
|
entries to a list of tree nodes for the
|
|
current XML toolkit.
|
|
"""
|
|
return [root]
|
|
|
|
tf = TreeFactory()
|
|
|
|
|
|
class SConsDocTree:
|
|
def __init__(self):
|
|
self.nsmap = {'dbx': dbxsd}
|
|
self.doc = None
|
|
self.root = None
|
|
self.xpath_context = None
|
|
|
|
def parseContent(self, content, include_entities=True):
|
|
""" Parses the given text content as XML
|
|
|
|
This is the setup portion, called from parseContent in
|
|
an SConsDocHandler instance - see the notes there.
|
|
"""
|
|
if not include_entities:
|
|
content = remove_entities(content)
|
|
# Create domtree from given content string
|
|
self.root = etree.fromstring(content)
|
|
|
|
def parseXmlFile(self, fpath):
|
|
# Create domtree from file
|
|
parser = etree.XMLParser(load_dtd=True, resolve_entities=False)
|
|
domtree = etree.parse(fpath, parser)
|
|
self.root = domtree.getroot()
|
|
|
|
def __del__(self):
|
|
if self.doc is not None:
|
|
self.doc.freeDoc()
|
|
if self.xpath_context is not None:
|
|
self.xpath_context.xpathFreeContext()
|
|
|
|
perc = "%"
|
|
|
|
def validate_all_xml(dpaths, xsdfile=default_xsd):
|
|
xmlschema_context = etree.parse(xsdfile)
|
|
|
|
fpaths = []
|
|
for dp in dpaths:
|
|
if dp.endswith('.xml') and isSConsXml(dp):
|
|
path = '.'
|
|
fpaths.append(dp)
|
|
else:
|
|
for path, dirs, files in os.walk(dp):
|
|
for f in files:
|
|
if f.endswith('.xml'):
|
|
fp = os.path.join(path, f)
|
|
if isSConsXml(fp):
|
|
fpaths.append(fp)
|
|
|
|
fails = []
|
|
fpaths = sorted(fpaths)
|
|
for idx, fp in enumerate(fpaths):
|
|
fpath = os.path.join(path, fp)
|
|
print("%.2f%s (%d/%d) %s" % (float(idx + 1) * 100.0 /float(len(fpaths)),
|
|
perc, idx + 1, len(fpaths), fp))
|
|
|
|
if not tf.validateXml(fp, xmlschema_context):
|
|
fails.append(fp)
|
|
continue
|
|
|
|
if fails:
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
class Item:
|
|
def __init__(self, name):
|
|
self.name = name
|
|
self.sort_name = name.lower()
|
|
if self.sort_name[0] == '_':
|
|
self.sort_name = self.sort_name[1:]
|
|
self.sets = []
|
|
self.uses = []
|
|
self.summary = None
|
|
self.arguments = None
|
|
def cmp_name(self, name):
|
|
if name[0] == '_':
|
|
name = name[1:]
|
|
return name.lower()
|
|
def __eq__(self, other):
|
|
return self.sort_name == other.sort_name
|
|
def __lt__(self, other):
|
|
return self.sort_name < other.sort_name
|
|
|
|
|
|
class Builder(Item):
|
|
pass
|
|
|
|
|
|
class Function(Item):
|
|
pass
|
|
|
|
|
|
class Tool(Item):
|
|
def __init__(self, name):
|
|
super().__init__(name)
|
|
self.entity = self.name.replace('+', 'X')
|
|
|
|
|
|
class ConstructionVariable(Item):
|
|
pass
|
|
|
|
|
|
class Arguments:
|
|
def __init__(self, signature, body=None):
|
|
if not body:
|
|
body = []
|
|
self.body = body
|
|
self.signature = signature
|
|
def __str__(self):
|
|
s = ''.join(self.body).strip()
|
|
result = []
|
|
for m in re.findall(r'([a-zA-Z/_]+|[^a-zA-Z/_]+)', s):
|
|
if ' ' in m:
|
|
m = '"%s"' % m
|
|
result.append(m)
|
|
return ' '.join(result)
|
|
def append(self, data):
|
|
self.body.append(data)
|
|
|
|
|
|
class SConsDocHandler:
|
|
def __init__(self):
|
|
self.builders = {}
|
|
self.functions = {}
|
|
self.tools = {}
|
|
self.cvars = {}
|
|
|
|
def parseItems(self, domelem, xpath_context, nsmap):
|
|
items = []
|
|
|
|
for i in tf.findAll(domelem, "item", dbxid, xpath_context, nsmap):
|
|
txt = tf.getText(i)
|
|
if txt is not None:
|
|
txt = txt.strip()
|
|
if len(txt):
|
|
items.append(txt.strip())
|
|
|
|
return items
|
|
|
|
def parseUsesSets(self, domelem, xpath_context, nsmap):
|
|
uses = []
|
|
sets = []
|
|
|
|
for u in tf.findAll(domelem, "uses", dbxid, xpath_context, nsmap):
|
|
uses.extend(self.parseItems(u, xpath_context, nsmap))
|
|
for s in tf.findAll(domelem, "sets", dbxid, xpath_context, nsmap):
|
|
sets.extend(self.parseItems(s, xpath_context, nsmap))
|
|
|
|
return sorted(uses), sorted(sets)
|
|
|
|
def parseInstance(self, domelem, map, Class,
|
|
xpath_context, nsmap, include_entities=True):
|
|
name = 'unknown'
|
|
if tf.hasAttribute(domelem, 'name'):
|
|
name = tf.getAttribute(domelem, 'name')
|
|
try:
|
|
instance = map[name]
|
|
except KeyError:
|
|
instance = Class(name)
|
|
map[name] = instance
|
|
uses, sets = self.parseUsesSets(domelem, xpath_context, nsmap)
|
|
instance.uses.extend(uses)
|
|
instance.sets.extend(sets)
|
|
if include_entities:
|
|
# Parse summary and function arguments
|
|
for s in tf.findAllChildrenOf(domelem, "summary", dbxid, xpath_context, nsmap):
|
|
if instance.summary is None:
|
|
instance.summary = []
|
|
instance.summary.append(tf.copyNode(s))
|
|
for a in tf.findAll(domelem, "arguments", dbxid, xpath_context, nsmap):
|
|
if instance.arguments is None:
|
|
instance.arguments = []
|
|
instance.arguments.append(tf.copyNode(a))
|
|
|
|
def parseDomtree(self, root, xpath_context=None, nsmap=None, include_entities=True):
|
|
# Process Builders
|
|
for b in tf.findAll(root, "builder", dbxid, xpath_context, nsmap):
|
|
self.parseInstance(b, self.builders, Builder,
|
|
xpath_context, nsmap, include_entities)
|
|
# Process Functions
|
|
for f in tf.findAll(root, "scons_function", dbxid, xpath_context, nsmap):
|
|
self.parseInstance(f, self.functions, Function,
|
|
xpath_context, nsmap, include_entities)
|
|
# Process Tools
|
|
for t in tf.findAll(root, "tool", dbxid, xpath_context, nsmap):
|
|
self.parseInstance(t, self.tools, Tool,
|
|
xpath_context, nsmap, include_entities)
|
|
# Process CVars
|
|
for c in tf.findAll(root, "cvar", dbxid, xpath_context, nsmap):
|
|
self.parseInstance(c, self.cvars, ConstructionVariable,
|
|
xpath_context, nsmap, include_entities)
|
|
|
|
def parseContent(self, content, include_entities=True):
|
|
"""Parse the given content as XML.
|
|
|
|
This method is used when we generate the basic lists of entities
|
|
for the builders, tools and functions. So we usually don't
|
|
bother about namespaces and resolving entities here...
|
|
this is handled in parseXmlFile below (step 2 of the overall process).
|
|
"""
|
|
# Create doctree
|
|
t = SConsDocTree()
|
|
t.parseContent(content, include_entities)
|
|
# Parse it
|
|
self.parseDomtree(t.root, t.xpath_context, t.nsmap, include_entities)
|
|
|
|
def parseXmlFile(self, fpath):
|
|
# Create doctree
|
|
t = SConsDocTree()
|
|
t.parseXmlFile(fpath)
|
|
# Parse it
|
|
self.parseDomtree(t.root, t.xpath_context, t.nsmap)
|
|
|
|
def importfile(path):
|
|
"""Import a Python source file or compiled file given its path."""
|
|
from importlib.util import MAGIC_NUMBER
|
|
with open(path, 'rb') as ifp:
|
|
is_bytecode = MAGIC_NUMBER == ifp.read(len(MAGIC_NUMBER))
|
|
filename = os.path.basename(path)
|
|
name, ext = os.path.splitext(filename)
|
|
if is_bytecode:
|
|
loader = importlib._bootstrap_external.SourcelessFileLoader(name, path)
|
|
else:
|
|
loader = importlib._bootstrap_external.SourceFileLoader(name, path)
|
|
# XXX We probably don't need to pass in the loader here.
|
|
spec = importlib.util.spec_from_file_location(name, path, loader=loader)
|
|
try:
|
|
return importlib._bootstrap._load(spec)
|
|
except ImportError:
|
|
raise Exception(path, sys.exc_info())
|
|
|
|
# Local Variables:
|
|
# tab-width:4
|
|
# indent-tabs-mode:nil
|
|
# End:
|
|
# vim: set expandtab tabstop=4 shiftwidth=4:
|