Commit a76fe5ca authored by Daniel Veillard's avatar Daniel Veillard

integrated the Out Of Memory test from Havoc Pennington #109368 a lot of

* Makefile.am testOOM.c testOOMlib.[ch] : integrated the Out Of
  Memory test from Havoc Pennington #109368
* SAX.c parser.c parserInternals.c tree.c uri.c valid.c
  xmlmemory.c xmlreader.c xmlregexp.c include/libxml/tree.h
  include/libxml/parser.h: a lot of memory allocation cleanups
  based on the results of the OOM testing
* check-relaxng-test-suite2.py: seems I forgot to commit the
  script.
Daniel
parent 18f113da
Thu Apr 24 18:01:46 CEST 2003 Daniel Veillard <daniel@veillard.com>
* Makefile.am testOOM.c testOOMlib.[ch] : integrated the Out Of
Memory test from Havoc Pennington #109368
* SAX.c parser.c parserInternals.c tree.c uri.c valid.c
xmlmemory.c xmlreader.c xmlregexp.c include/libxml/tree.h
include/libxml/parser.h: a lot of memory allocation cleanups
based on the results of the OOM testing
* check-relaxng-test-suite2.py: seems I forgot to commit the
script.
Wed Apr 23 17:16:41 CEST 2003 Daniel Veillard <daniel@veillard.com>
* xmlschemastypes.c: trivial fix for 109774 removing a warning
......
......@@ -116,6 +116,11 @@ testReader_LDFLAGS =
testReader_DEPENDENCIES = $(DEPS)
testReader_LDADD= $(LDADDS)
testOOM_SOURCES=testOOM.c testOOMlib.h testOOMlib.c
testOOM_LDFLAGS =
testOOM_DEPENDENCIES = $(DEPS)
testOOM_LDADD= $(LDADDS)
check-local: tests
testall : tests SVGtests SAXtests
......
......@@ -232,6 +232,8 @@ externalSubset(void *ctx, const xmlChar *name,
ctxt->sax->error(ctxt->userData,
"externalSubset: out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
ctxt->input = oldinput;
ctxt->inputNr = oldinputNr;
ctxt->inputMax = oldinputMax;
......@@ -745,12 +747,25 @@ startDocument(void *ctx)
"SAX.startDocument()\n");
#endif
if (ctxt->html) {
if (ctxt->myDoc == NULL)
#ifdef LIBXML_HTML_ENABLED
if (ctxt->myDoc == NULL)
ctxt->myDoc = htmlNewDocNoDtD(NULL, NULL);
if (ctxt->myDoc == NULL) {
if ((ctxt->sax != NULL) && (ctxt->sax->error != NULL))
ctxt->sax->error(ctxt->userData,
"SAX.startDocument(): out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
return;
}
#else
xmlGenericError(xmlGenericErrorContext,
"libxml2 built without HTML support\n");
ctxt->errNo = XML_ERR_INTERNAL_ERROR;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
return;
#endif
} else {
doc = ctxt->myDoc = xmlNewDoc(ctxt->version);
......@@ -760,6 +775,14 @@ startDocument(void *ctx)
else
doc->encoding = NULL;
doc->standalone = ctxt->standalone;
} else {
if ((ctxt->sax != NULL) && (ctxt->sax->error != NULL))
ctxt->sax->error(ctxt->userData,
"SAX.startDocument(): out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
return;
}
}
if ((ctxt->myDoc != NULL) && (ctxt->myDoc->URL == NULL) &&
......@@ -839,6 +862,17 @@ my_attribute(void *ctx, const xmlChar *fullname, const xmlChar *value,
* Split the full name into a namespace prefix and the tag name
*/
name = xmlSplitQName(ctxt, fullname, &ns);
if (name == NULL) {
if ((ctxt->sax != NULL) && (ctxt->sax->error != NULL))
ctxt->sax->error(ctxt->userData,
"SAX.startElement(): out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
if (ns != NULL)
xmlFree(ns);
return;
}
/*
* Do the last stage of the attribute normalization
......@@ -921,6 +955,18 @@ my_attribute(void *ctx, const xmlChar *fullname, const xmlChar *value,
val = xmlStringDecodeEntities(ctxt, value, XML_SUBSTITUTE_REF,
0,0,0);
ctxt->depth--;
if (val == NULL) {
if ((ctxt->sax != NULL) && (ctxt->sax->error != NULL))
ctxt->sax->error(ctxt->userData,
"SAX.startElement(): out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
xmlFree(ns);
if (name != NULL)
xmlFree(name);
return;
}
} else {
val = (xmlChar *) value;
}
......@@ -1200,14 +1246,19 @@ process_external_subset:
attr->elem, attr->name,
attr->prefix);
if ((tst == attr) || (tst == NULL)) {
xmlChar fn[50];
xmlChar *fulln;
if (attr->prefix != NULL) {
fulln = xmlStrdup(attr->prefix);
fulln = xmlStrcat(fulln, BAD_CAST ":");
fulln = xmlStrcat(fulln, attr->name);
} else {
fulln = xmlStrdup(attr->name);
fulln = xmlBuildQName(attr->name, attr->prefix, fn, 50);
if (fulln == NULL) {
if ((ctxt->sax != NULL) &&
(ctxt->sax->error != NULL))
ctxt->sax->error(ctxt->userData,
"SAX.startElement(): out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
return;
}
/*
......@@ -1229,7 +1280,8 @@ process_external_subset:
my_attribute(ctxt, fulln, attr->defaultValue,
prefix);
}
xmlFree(fulln);
if ((fulln != fn) && (fulln != attr->name))
xmlFree(fulln);
}
}
}
......@@ -1301,7 +1353,14 @@ startElement(void *ctx, const xmlChar *fullname, const xmlChar **atts)
* an attribute at this level.
*/
ret = xmlNewDocNodeEatName(ctxt->myDoc, NULL, name, NULL);
if (ret == NULL) return;
if (ret == NULL) {
if (prefix != NULL)
xmlFree(prefix);
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
return;
}
if (ctxt->myDoc->children == NULL) {
#ifdef DEBUG_SAX_TREE
xmlGenericError(xmlGenericErrorContext, "Setting %s as root\n", name);
......@@ -1587,6 +1646,9 @@ characters(void *ctx, const xmlChar *ch, int len)
if ((ctxt->sax != NULL) && (ctxt->sax->error != NULL))
ctxt->sax->error(ctxt->userData,
"SAX.characters(): out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
return;
}
ctxt->nodemem = size;
......@@ -1596,7 +1658,14 @@ characters(void *ctx, const xmlChar *ch, int len)
ctxt->nodelen += len;
lastChild->content[ctxt->nodelen] = 0;
} else if (coalesceText) {
xmlTextConcat(lastChild, ch, len);
if (xmlTextConcat(lastChild, ch, len)) {
if ((ctxt->sax != NULL) && (ctxt->sax->error != NULL))
ctxt->sax->error(ctxt->userData,
"SAX.characters(): out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
}
if (ctxt->node->children != NULL) {
ctxt->nodelen = xmlStrlen(lastChild->content);
ctxt->nodemem = ctxt->nodelen + 1;
......@@ -1604,10 +1673,19 @@ characters(void *ctx, const xmlChar *ch, int len)
} else {
/* Mixed content, first time */
lastChild = xmlNewTextLen(ch, len);
xmlAddChild(ctxt->node, lastChild);
if (ctxt->node->children != NULL) {
ctxt->nodelen = len;
ctxt->nodemem = len + 1;
if (lastChild == NULL) {
if ((ctxt->sax != NULL) && (ctxt->sax->error != NULL))
ctxt->sax->error(ctxt->userData,
"SAX.characters(): out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
} else {
xmlAddChild(ctxt->node, lastChild);
if (ctxt->node->children != NULL) {
ctxt->nodelen = len;
ctxt->nodemem = len + 1;
}
}
}
}
......
#!/usr/bin/python
import sys
import time
import os
import string
import StringIO
sys.path.append("python")
import libxml2
# Memory debug specific
libxml2.debugMemory(1)
debug = 0
#
# the testsuite description
#
CONF="test/relaxng/testsuite.xml"
LOG="check-relaxng-test-suite2.log"
log = open(LOG, "w")
nb_schemas_tests = 0
nb_schemas_success = 0
nb_schemas_failed = 0
nb_instances_tests = 0
nb_instances_success = 0
nb_instances_failed = 0
libxml2.lineNumbersDefault(1)
#
# Resolver callback
#
resources = {}
def resolver(URL, ID, ctxt):
global resources
if resources.has_key(URL):
return(StringIO.StringIO(resources[URL]))
log.write("Resolver failure: asked %s\n" % (URL))
log.write("resources: %s\n" % (resources))
return None
#
# Load the previous results
#
#results = {}
#previous = {}
#
#try:
# res = libxml2.parseFile(RES)
#except:
# log.write("Could not parse %s" % (RES))
#
# handle a valid instance
#
def handle_valid(node, schema):
global log
global nb_instances_success
global nb_instances_failed
instance = node.prop("dtd")
if instance == None:
instance = ""
child = node.children
while child != None:
if child.type != 'text':
instance = instance + child.serialize()
child = child.next
mem = libxml2.debugMemory(1);
try:
doc = libxml2.parseDoc(instance)
except:
doc = None
if doc == None:
log.write("\nFailed to parse correct instance:\n-----\n")
log.write(instance)
log.write("\n-----\n")
nb_instances_failed = nb_instances_failed + 1
return
if debug:
print "instance line %d" % (node.lineNo())
try:
ctxt = schema.relaxNGNewValidCtxt()
ret = doc.relaxNGValidateDoc(ctxt)
del ctxt
except:
ret = -1
doc.freeDoc()
if mem != libxml2.debugMemory(1):
print "validating instance %d line %d leaks" % (
nb_instances_tests, node.lineNo())
if ret != 0:
log.write("\nFailed to validate correct instance:\n-----\n")
log.write(instance)
log.write("\n-----\n")
nb_instances_failed = nb_instances_failed + 1
else:
nb_instances_success = nb_instances_success + 1
#
# handle an invalid instance
#
def handle_invalid(node, schema):
global log
global nb_instances_success
global nb_instances_failed
instance = node.prop("dtd")
if instance == None:
instance = ""
child = node.children
while child != None:
if child.type != 'text':
instance = instance + child.serialize()
child = child.next
mem = libxml2.debugMemory(1);
try:
doc = libxml2.parseDoc(instance)
except:
doc = None
if doc == None:
log.write("\nStrange: failed to parse incorrect instance:\n-----\n")
log.write(instance)
log.write("\n-----\n")
return
if debug:
print "instance line %d" % (node.lineNo())
try:
ctxt = schema.relaxNGNewValidCtxt()
ret = doc.relaxNGValidateDoc(ctxt)
del ctxt
except:
ret = -1
doc.freeDoc()
if mem != libxml2.debugMemory(1):
print "validating instance %d line %d leaks" % (
nb_instances_tests, node.lineNo())
if ret == 0:
log.write("\nFailed to detect validation problem in instance:\n-----\n")
log.write(instance)
log.write("\n-----\n")
nb_instances_failed = nb_instances_failed + 1
else:
nb_instances_success = nb_instances_success + 1
#
# handle an incorrect test
#
def handle_correct(node):
global log
global nb_schemas_success
global nb_schemas_failed
schema = ""
child = node.children
while child != None:
if child.type != 'text':
schema = schema + child.serialize()
child = child.next
try:
rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema))
rngs = rngp.relaxNGParse()
except:
rngs = None
if rngs == None:
log.write("\nFailed to compile correct schema:\n-----\n")
log.write(schema)
log.write("\n-----\n")
nb_schemas_failed = nb_schemas_failed + 1
else:
nb_schemas_success = nb_schemas_success + 1
return rngs
def handle_incorrect(node):
global log
global nb_schemas_success
global nb_schemas_failed
schema = ""
child = node.children
while child != None:
if child.type != 'text':
schema = schema + child.serialize()
child = child.next
try:
rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema))
rngs = rngp.relaxNGParse()
except:
rngs = None
if rngs != None:
log.write("\nFailed to detect schema error in:\n-----\n")
log.write(schema)
log.write("\n-----\n")
nb_schemas_failed = nb_schemas_failed + 1
else:
# log.write("\nSuccess detecting schema error in:\n-----\n")
# log.write(schema)
# log.write("\n-----\n")
nb_schemas_success = nb_schemas_success + 1
return None
#
# resource handling: keep a dictionary of URL->string mappings
#
def handle_resource(node, dir):
global resources
try:
name = node.prop('name')
except:
name = None
if name == None or name == '':
log.write("resource has no name")
return;
if dir != None:
# name = libxml2.buildURI(name, dir)
name = dir + '/' + name
res = ""
child = node.children
while child != None:
if child.type != 'text':
res = res + child.serialize()
child = child.next
resources[name] = res
#
# dir handling: pseudo directory resources
#
def handle_dir(node, dir):
try:
name = node.prop('name')
except:
name = None
if name == None or name == '':
log.write("resource has no name")
return;
if dir != None:
# name = libxml2.buildURI(name, dir)
name = dir + '/' + name
dirs = node.xpathEval('dir')
for dir in dirs:
handle_dir(dir, name)
res = node.xpathEval('resource')
for r in res:
handle_resource(r, name)
#
# handle a testCase element
#
def handle_testCase(node):
global nb_schemas_tests
global nb_instances_tests
global resources
sections = node.xpathEval('string(section)')
log.write("\n ======== test %d line %d section %s ==========\n" % (
nb_schemas_tests, node.lineNo(), sections))
resources = {}
if debug:
print "test %d line %d" % (nb_schemas_tests, node.lineNo())
dirs = node.xpathEval('dir')
for dir in dirs:
handle_dir(dir, None)
res = node.xpathEval('resource')
for r in res:
handle_resource(r, None)
tsts = node.xpathEval('incorrect')
if tsts != []:
if len(tsts) != 1:
print "warning test line %d has more than one <incorrect> example" %(node.lineNo())
schema = handle_incorrect(tsts[0])
else:
tsts = node.xpathEval('correct')
if tsts != []:
if len(tsts) != 1:
print "warning test line %d has more than one <correct> example"% (node.lineNo())
schema = handle_correct(tsts[0])
else:
print "warning <testCase> line %d has no <correct> nor <incorrect> child" % (node.lineNo())
nb_schemas_tests = nb_schemas_tests + 1;
valids = node.xpathEval('valid')
invalids = node.xpathEval('invalid')
nb_instances_tests = nb_instances_tests + len(valids) + len(invalids)
if schema != None:
for valid in valids:
handle_valid(valid, schema)
for invalid in invalids:
handle_invalid(invalid, schema)
#
# handle a testSuite element
#
def handle_testSuite(node, level = 0):
global nb_schemas_tests, nb_schemas_success, nb_schemas_failed
global nb_instances_tests, nb_instances_success, nb_instances_failed
if level >= 1:
old_schemas_tests = nb_schemas_tests
old_schemas_success = nb_schemas_success
old_schemas_failed = nb_schemas_failed
old_instances_tests = nb_instances_tests
old_instances_success = nb_instances_success
old_instances_failed = nb_instances_failed
docs = node.xpathEval('documentation')
authors = node.xpathEval('author')
if docs != []:
msg = ""
for doc in docs:
msg = msg + doc.content + " "
if authors != []:
msg = msg + "written by "
for author in authors:
msg = msg + author.content + " "
print msg
sections = node.xpathEval('section')
if sections != [] and level <= 0:
msg = ""
for section in sections:
msg = msg + section.content + " "
print "Tests for section %s" % (msg)
for test in node.xpathEval('testCase'):
handle_testCase(test)
for test in node.xpathEval('testSuite'):
handle_testSuite(test, level + 1)
if level >= 1 and sections != []:
msg = ""
for section in sections:
msg = msg + section.content + " "
print "Result of tests for section %s" % (msg)
if nb_schemas_tests != old_schemas_tests:
print "found %d test schemas: %d success %d failures" % (
nb_schemas_tests - old_schemas_tests,
nb_schemas_success - old_schemas_success,
nb_schemas_failed - old_schemas_failed)
if nb_instances_tests != old_instances_tests:
print "found %d test instances: %d success %d failures" % (
nb_instances_tests - old_instances_tests,
nb_instances_success - old_instances_success,
nb_instances_failed - old_instances_failed)
#
# Parse the conf file
#
libxml2.substituteEntitiesDefault(1);
testsuite = libxml2.parseFile(CONF)
#
# Error and warnng callbacks
#
def callback(ctx, str):
global log
log.write("%s%s" % (ctx, str))
libxml2.registerErrorHandler(callback, "")
libxml2.setEntityLoader(resolver)
root = testsuite.getRootElement()
if root.name != 'testSuite':
print "%s doesn't start with a testSuite element, aborting" % (CONF)
sys.exit(1)
print "Running Relax NG testsuite"
handle_testSuite(root)
print "\nTOTAL:\nfound %d test schemas: %d success %d failures" % (
nb_schemas_tests, nb_schemas_success, nb_schemas_failed)
print "found %d test instances: %d success %d failures" % (
nb_instances_tests, nb_instances_success, nb_instances_failed)
testsuite.freeDoc()
# Memory debug specific
libxml2.relaxNGCleanupTypes()
libxml2.cleanupParser()
if libxml2.debugMemory(1) == 0:
print "OK"
else:
print "Memory leak %d bytes" % (libxml2.debugMemory(1))
libxml2.dumpMemory()
......@@ -802,7 +802,7 @@ int xmlParseCtxtExternalEntity(xmlParserCtxtPtr ctx,
/*
* Parser contexts handling.
*/
void xmlInitParserCtxt (xmlParserCtxtPtr ctxt);
int xmlInitParserCtxt (xmlParserCtxtPtr ctxt);
void xmlClearParserCtxt (xmlParserCtxtPtr ctxt);
void xmlFreeParserCtxt (xmlParserCtxtPtr ctxt);
void xmlSetupParserForBuffer (xmlParserCtxtPtr ctxt,
......
......@@ -724,7 +724,7 @@ xmlNodePtr xmlAddNextSibling (xmlNodePtr cur,
void xmlUnlinkNode (xmlNodePtr cur);
xmlNodePtr xmlTextMerge (xmlNodePtr first,
xmlNodePtr second);
void xmlTextConcat (xmlNodePtr node,
int xmlTextConcat (xmlNodePtr node,
const xmlChar *content,
int len);
void xmlFreeNodeList (xmlNodePtr cur);
......
......@@ -1720,6 +1720,8 @@ xmlSplitQName(xmlParserCtxtPtr ctxt, const xmlChar *name, xmlChar **prefix) {
*prefix = NULL;
if (cur == NULL) return(NULL);
#ifndef XML_XML_NAMESPACE
/* xml: prefix is not really a namespace */
if ((cur[0] == 'x') && (cur[1] == 'm') &&
......@@ -1905,6 +1907,14 @@ xmlParseName(xmlParserCtxtPtr ctxt) {
ctxt->input->cur = in;
ctxt->nbChars += count;
ctxt->input->col += count;
if (ret == NULL) {
if ((ctxt->sax != NULL) && (ctxt->sax->error != NULL))
ctxt->sax->error(ctxt->userData,
"XML parser: out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
}
return(ret);
}
}
......@@ -4581,6 +4591,15 @@ xmlParseElementChildrenContentDecl
return(NULL);
}
cur = ret = xmlNewElementContent(elem, XML_ELEMENT_CONTENT_ELEMENT);
if (cur == NULL) {
if ((ctxt->sax != NULL) && (ctxt->sax->error != NULL))
ctxt->sax->error(ctxt->userData,
"xmlParseElementChildrenContentDecl : out of memory\n");
ctxt->errNo = XML_ERR_NO_MEMORY;
if (ctxt->recovery == 0) ctxt->disableSAX = 1;
xmlFree(elem);
return(NULL);
}
GROW;
if (RAW == '?') {
cur->ocur = XML_ELEMENT_CONTENT_OPT;
......@@ -6731,18 +6750,35 @@ xmlParseStartTag(xmlParserCtxtPtr ctxt) {
xmlGenericError(xmlGenericErrorContext,
"malloc of %ld byte failed\n",
maxatts * (long)sizeof(xmlChar *));
return(NULL);
if (attname != NULL)
xmlFree(attname);
if (attvalue != NULL)
xmlFree(attvalue);
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
goto failed;
}
} else if (nbatts + 4 > maxatts) {
const xmlChar **n;
maxatts *= 2;
atts = (const xmlChar **) xmlRealloc((void *) atts,
n = (const xmlChar **) xmlRealloc((void *) atts,
maxatts * sizeof(xmlChar *));
if (atts == NULL) {
if (n == NULL) {
xmlGenericError(xmlGenericErrorContext,
"realloc of %ld byte failed\n",
maxatts * (long)sizeof(xmlChar *));
return(NULL);
if (attname != NULL)
xmlFree(attname);
if (attvalue != NULL)
xmlFree(attvalue);
ctxt->errNo = XML_ERR_NO_MEMORY;
ctxt->instate = XML_PARSER_EOF;
ctxt->disableSAX = 1;
goto failed;
}
atts = n;
}
atts[nbatts++] = attname;
atts[nbatts++] = attvalue;
......@@ -6790,7 +6826,9 @@ failed:
ctxt->sax->startElement(ctxt->userData, name, atts);
if (atts != NULL) {
for (i = 0;i < nbatts;i++) xmlFree((xmlChar *) atts[i]);
for (i = 0;i < nbatts;i++)
if (atts[i] != NULL)
xmlFree((xmlChar *) atts[i]);
xmlFree((void *) atts);
}
return(name);
......@@ -8317,6 +8355,10 @@ xmlParseTryOrFinish(xmlParserCtxtPtr ctxt, int terminate) {
xmlParseGetLasts(ctxt, &lastlt, &lastgt);