adding xmlMemBlocks() work on generator of an automatic API regression

* xmlmemory.c include/libxml/xmlmemory.h: adding xmlMemBlocks()
* Makefile.am gentest.py testapi.c: work on generator of an
  automatic API regression test tool.
* SAX2.c nanoftp.c parser.c parserInternals.c tree.c xmlIO.c
  xmlstring.c: various API hardeing changes as a result of running
  teh first set of automatic API regression tests.
* test/slashdot16.xml: apparently missing from CVS, commited it
Daniel

ChangeLog

+Tue Nov  2 15:49:34 CET 2004 Daniel Veillard <daniel@veillard.com>
+
+	* xmlmemory.c include/libxml/xmlmemory.h: adding xmlMemBlocks()
+	* Makefile.am gentest.py testapi.c: work on generator of an
+	  automatic API regression test tool.
+	* SAX2.c nanoftp.c parser.c parserInternals.c tree.c xmlIO.c
+	  xmlstring.c: various API hardeing changes as a result of running
+	  teh first set of automatic API regression tests.
+	* test/slashdot16.xml: apparently missing from CVS, commited it
+
 Mon Nov  1 15:54:18 CET 2004 Daniel Veillard <daniel@veillard.com>
 
 	* xpath.c: fixed an UTF-8 parsing bug reported by Markus Bertheau

HTMLtree.c

@@ -1053,6 +1053,9 @@ htmlSaveFile(const char *filename, xmlDocPtr cur) {
     const char *encoding;
     int ret;
 
+    if ((cur == NULL) || (filename == NULL))
+        return(-1);
+       
     xmlInitParser();
 
     encoding = (const char *) htmlGetMetaEncoding(cur);
@@ -1113,6 +1116,9 @@ htmlSaveFileFormat(const char *filename, xmlDocPtr cur,
     xmlCharEncodingHandlerPtr handler = NULL;
     int ret;
 
+    if ((cur == NULL) || (filename == NULL))
+        return(-1);
+       
     xmlInitParser();
 
     if (encoding != NULL) {

Makefile.am

@@ -8,7 +8,7 @@ INCLUDES = -I$(top_builddir)/include -I@srcdir@/include @THREAD_CFLAGS@ @Z_CFLAG
 
 noinst_PROGRAMS=testSchemas testRelax testSAX testHTML testXPath testURI \
                 testThreads testC14N testAutomata testRegexp \
-		testReader
+		testReader testapi
 
 bin_PROGRAMS = xmllint xmlcatalog
 
@@ -113,6 +113,14 @@ testReader_LDFLAGS =
 testReader_DEPENDENCIES = $(DEPS)
 testReader_LDADD= $(LDADDS)
 
+testapi.c: gentest.py doc/libxml2-api.xml
+	-@(if [ "$(PYTHON)" != "" ] ; then $(PYTHON) gentest.py ; fi )
+
+testapi_SOURCES=testapi.c
+testapi_LDFLAGS = 
+testapi_DEPENDENCIES = $(DEPS)
+testapi_LDADD= $(LDADDS)
+
 #testOOM_SOURCES=testOOM.c testOOMlib.h testOOMlib.c
 #testOOM_LDFLAGS = 
 #testOOM_DEPENDENCIES = $(DEPS)

SAX2.c

@@ -890,7 +890,9 @@ xmlSAX2EndDocument(void *ctx)
 	ctxt->myDoc->encoding = ctxt->encoding;
 	ctxt->encoding = NULL;
     }
-    if ((ctxt->inputTab[0]->encoding != NULL) && (ctxt->myDoc != NULL) &&
+    if ((ctxt->inputTab != NULL) &&
+        (ctxt->inputNr > 0) && (ctxt->inputTab[0] != NULL) &&
+        (ctxt->inputTab[0]->encoding != NULL) && (ctxt->myDoc != NULL) &&
 	(ctxt->myDoc->encoding == NULL)) {
 	ctxt->myDoc->encoding = xmlStrdup(ctxt->inputTab[0]->encoding);
     }

gentest.py

+#!/usr/bin/python -u
+#
+# generate a tester program for the API
+#
+import sys
+import string
+try:
+    import libxml2
+except:
+    print "libxml2 python bindings not available, skipping testapi.c generation"
+    sys.exit(0)
+
+#
+# Modules we don't want skip in API test
+#
+skipped_modules = [ "SAX", "SAX2", "xlink", "threads", "globals",
+  "xpathInternals", "xmlunicode", "parserInternals", "xmlmemory",
+  "xmlversion", "debugXML" ]
+
+#
+# Some function really need to be skipped for the tests.
+#
+skipped_functions = [ "xmlFdRead", "xmlReadFd", "xmlCtxtReadFd",
+                      "xmlCleanupParser" ]
+
+#
+# Those functions have side effect on the global state
+# and hence generate errors on memory allocation tests
+#
+skipped_memcheck = [ "xmlLoadCatalog", "xmlAddEncodingAlias",
+   "xmlSchemaInitTypes", "xmlNanoFTPProxy", "xmlNanoFTPScanProxy",
+   "xmlNanoHTTPScanProxy", "xmlResetLastError", "xmlCatalogConvert",
+   "xmlCatalogRemove", "xmlLoadCatalogs", "xmlCleanupCharEncodingHandlers",
+   "xmlInitCharEncodingHandlers" ]
+
+modules = []
+
+def is_skipped_module(name):
+    for mod in skipped_modules:
+        if mod == name:
+	    return 1
+    return 0
+
+def is_skipped_function(name):
+    for fun in skipped_functions:
+        if fun == name:
+	    return 1
+    # Do not test destructors
+    if string.find(name, 'Free') != -1:
+        return 1
+    return 0
+
+def is_skipped_memcheck(name):
+    for fun in skipped_memcheck:
+        if fun == name:
+	    return 1
+    return 0
+
+missing_types = {}
+def add_missing_type(name, func):
+    try:
+        list = missing_types[name]
+	list.append(func)
+    except:
+        missing_types[name] = [func]
+
+#
+# Open the input API description and the C test program result
+#
+doc = libxml2.readFile('doc/libxml2-api.xml', None, 0)
+if doc == None:
+    print "Failed to load doc/libxml2-api.xml"
+    sys.exit(1)
+test = open('testapi.c', 'w')
+ctxt = doc.xpathNewContext()
+headers = ctxt.xpathEval("/api/files/file")
+
+#
+# Generate the test header
+#
+test.write("""/*
+ * testapi.c: libxml2 API tester program.
+ *
+ * Automatically generated by gentest.py from libxml2-api.xml
+ *
+ * See Copyright for the status of this software.
+ *
+ * daniel@veillard.com
+ */
+
+#include <stdio.h>
+#include <libxml/xmlerror.h>
+
+static int testlibxml2(void);
+
+static int generic_errors = 0;
+static int call_tests = 0;
+
+static void
+structured_errors(void *userData ATTRIBUTE_UNUSED,
+                  xmlErrorPtr error ATTRIBUTE_UNUSED) {
+    generic_errors++;
+}
+
+int main(void) {
+    int ret;
+    int blocks, mem;
+
+    LIBXML_TEST_VERSION
+
+    xmlSetStructuredErrorFunc(NULL, structured_errors);
+
+    ret = testlibxml2();
+
+    xmlCleanupParser();
+    blocks = xmlMemBlocks();
+    mem = xmlMemUsed();
+    if ((blocks != 0) || (mem != 0)) {
+        printf("testapi leaked %d bytes in %d blocks\\n", mem, blocks);
+    }
+    xmlMemoryDump();
+
+    return (ret != 0);
+}
+
+""");
+
+#
+# Load the interfaces
+# 
+for file in headers:
+    name = file.xpathEval('string(@name)')
+    if (name == None) or (name == ''):
+        continue
+
+    #
+    # do not test deprecated APIs
+    #
+    desc = file.xpathEval('string(description)')
+    if string.find(desc, 'DEPRECATED') != -1:
+        print "Skipping deprecated interface %s" % name
+	continue;
+
+    #
+    # Some module may be skipped because they don't really consists
+    # of user callable APIs
+    #
+    if is_skipped_module(name):
+        continue
+
+    test.write("#include <libxml/%s.h>\n" % name)
+    modules.append(name)
+        
+#
+# Generate the callers signatures
+# 
+for module in modules:
+    test.write("static int test_%s(void);\n" % module);
+
+#
+# Provide the type generators and destructors for the parameters
+#
+
+def type_convert(str, name, info, module, function):
+    res = string.replace(str, " *", "_ptr")
+    res = string.replace(res, " ", "_")
+    if res == 'const_char_ptr':
+        if string.find(name, "file") != -1 or \
+           string.find(name, "uri") != -1 or \
+           string.find(name, "URI") != -1 or \
+           string.find(info, "filename") != -1 or \
+           string.find(info, "URI") != -1 or \
+           string.find(info, "URL") != -1:
+	    if string.find(function, "Save") != -1:
+	        return('fileoutput')
+	    return('filepath')
+    if res == 'void_ptr':
+        if module == 'nanoftp' and name == 'ctx':
+	    return('xmlNanoFTPCtxtPtr')
+        if module == 'nanohttp' and name == 'ctx':
+	    return('xmlNanoHTTPCtxtPtr')
+        
+    return res
+
+known_param_types = [ "int", "const_char_ptr", "const_xmlChar_ptr",
+   "xmlParserCtxtPtr", "xmlDocPtr", "filepath", "fileoutput" ];
+
+def is_known_param_type(name):
+    for type in known_param_types:
+        if type == name:
+	    return 1
+    return 0
+
+test.write("""
+#define gen_nb_int 4
+
+static int gen_int(int no) {
+    if (no == 0) return(0);
+    if (no == 1) return(1);
+    if (no == 2) return(122);
+    return(-1);
+}
+
+static void des_int(int no ATTRIBUTE_UNUSED, int val ATTRIBUTE_UNUSED) {
+}
+
+#define gen_nb_const_char_ptr 4
+
+static const char *gen_const_char_ptr(int no) {
+    if (no == 0) return("foo");
+    if (no == 1) return("<foo/>");
+    if (no == 2) return("test/ent2");
+    return(NULL);
+}
+static void des_const_char_ptr(int no ATTRIBUTE_UNUSED, const char *val ATTRIBUTE_UNUSED) {
+}
+
+#define gen_nb_const_xmlChar_ptr 5
+
+static const xmlChar *gen_const_xmlChar_ptr(int no) {
+    if (no == 0) return((const xmlChar *) "foo");
+    if (no == 1) return((const xmlChar *) "<foo/>");
+    if (no == 2) return((const xmlChar *) "nne");
+    if (no == 3) return((const xmlChar *) " 2ab ");
+    return(NULL);
+}
+static void des_const_xmlChar_ptr(int no ATTRIBUTE_UNUSED, const xmlChar *val ATTRIBUTE_UNUSED) {
+}
+
+#define gen_nb_filepath 8
+
+static const char *gen_filepath(int no) {
+    if (no == 0) return("missing.xml");
+    if (no == 1) return("<foo/>");
+    if (no == 2) return("test/ent2");
+    if (no == 3) return("test/valid/REC-xml-19980210.xml");
+    if (no == 4) return("test/valid/xhtml1-strict.dtd");
+    if (no == 5) return("http://missing.example.org/");
+    if (no == 6) return("http://missing. example.org/");
+    return(NULL);
+}
+static void des_filepath(int no ATTRIBUTE_UNUSED, const char *val ATTRIBUTE_UNUSED) {
+}
+
+#define gen_nb_fileoutput 6
+
+static const char *gen_fileoutput(int no) {
+    if (no == 0) return("/missing.xml");
+    if (no == 1) return("<foo/>");
+    if (no == 2) return("ftp://missing.example.org/foo");
+    if (no == 3) return("http://missing.example.org/");
+    if (no == 4) return("http://missing. example.org/");
+    return(NULL);
+}
+static void des_fileoutput(int no ATTRIBUTE_UNUSED, const char *val ATTRIBUTE_UNUSED) {
+}
+
+#define gen_nb_xmlParserCtxtPtr 2
+static xmlParserCtxtPtr gen_xmlParserCtxtPtr(int no) {
+    if (no == 0) return(xmlNewParserCtxt());
+    return(NULL);
+}
+static void des_xmlParserCtxtPtr(int no ATTRIBUTE_UNUSED, xmlParserCtxtPtr val) {
+    if (val != NULL)
+        xmlFreeParserCtxt(val);
+}
+
+#define gen_nb_xmlDocPtr 2
+static xmlDocPtr gen_xmlDocPtr(int no) {
+    if (no == 0) return(xmlNewDoc(BAD_CAST "1.0"));
+    return(NULL);
+}
+static void des_xmlDocPtr(int no ATTRIBUTE_UNUSED, xmlDocPtr val) {
+    if (val != NULL)
+        xmlFreeDoc(val);
+}
+
+""");
+
+#
+# Provide the type destructors for the return values
+#
+
+known_return_types = [ "int", "const_char_ptr", "xmlDocPtr", "xmlNodePtr" ];
+
+def is_known_return_type(name):
+    for type in known_return_types:
+        if type == name:
+	    return 1
+    return 0
+
+test.write("""
+static void desret_int(int val ATTRIBUTE_UNUSED) {
+}
+static void desret_const_char_ptr(const char *val ATTRIBUTE_UNUSED) {
+}
+static void desret_xmlDocPtr(xmlDocPtr val) {
+    xmlFreeDoc(val);
+}
+static void desret_xmlNodePtr(xmlNodePtr val) {
+    xmlUnlinkNode(val);
+    xmlFreeNode(val);
+}
+""");
+
+#
+# Generate the top caller
+# 
+
+test.write("""
+/**
+ * testlibxml2:
+ *
+ * Main entry point of the tester for the full libxml2 module,
+ * it calls all the tester entry point for each module.
+ *
+ * Returns the number of error found
+ */
+static int
+testlibxml2(void)
+{
+    int ret = 0;
+
+""")
+
+for module in modules:
+    test.write("    ret += test_%s();\n" % module)
+
+test.write("""
+    printf("Total: %d tests, %d errors\\n", call_tests, ret);
+    return(ret);
+}
+
+""")
+
+#
+# How to handle a function
+# 
+nb_tests = 0
+
+def generate_test(module, node):
+    global test
+    global nb_tests
+    nb_cond = 0
+    no_gen = 0
+
+    name = node.xpathEval('string(@name)')
+    if is_skipped_function(name):
+        return
+
+    test.write("""
+static int
+test_%s(void) {
+    int ret = 0;
+
+""" % (name))
+
+    #
+    # check we know how to handle the args and return values
+    # and store the informations for the generation
+    #
+    try:
+	args = node.xpathEval("arg")
+    except:
+        args = []
+    t_args = []
+    for arg in args:
+        rtype = arg.xpathEval("string(@type)")
+	if rtype == 'void':
+	    break;
+	info = arg.xpathEval("string(@info)")
+	nam = arg.xpathEval("string(@name)")
+        type = type_convert(rtype, nam, info, module, name)
+	if is_known_param_type(type) == 0:
+	    add_missing_type(type, name);
+	    no_gen = 1
+	t_args.append((nam, type, rtype, info))
+    
+    try:
+	rets = node.xpathEval("return")
+    except:
+        rets = []
+    t_ret = None
+    for ret in rets:
+        rtype = ret.xpathEval("string(@type)")
+	info = ret.xpathEval("string(@info)")
+        type = type_convert(rtype, 'return', info, module, name)
+	if rtype == 'void':
+	    break
+	if is_known_return_type(type) == 0:
+	    add_missing_type(type, name);
+	    no_gen = 1
+	t_ret = (type, rtype, info)
+	break
+
+    if no_gen == 1:
+	test.write("""
+    /* missing type support */
+    return(ret);
+}
+
+""")
+        return
+
+    try:
+	conds = node.xpathEval("cond")
+	for cond in conds:
+	    test.write("#ifdef %s\n" % (cond.get_content()))
+	    nb_cond = nb_cond + 1
+    except:
+        pass
+    
+    # Declare the memory usage counter
+    no_mem = is_skipped_memcheck(name)
+    if no_mem == 0:
+	test.write("    int mem_base;\n");
+
+    # Declare the return value
+    if t_ret != None:
+        test.write("    %s ret_val;\n" % (t_ret[1]))
+
+    # Declare the arguments
+    for arg in t_args:
+        (nam, type, rtype, info) = arg;
+	# add declaration
+	test.write("    %s %s; /* %s */\n" % (rtype, nam, info))
+	test.write("    int n_%s;\n" % (nam))
+    test.write("\n")
+
+    # Cascade loop on of each argument list of values
+    for arg in t_args:
+        (nam, type, rtype, info) = arg;
+	#
+	test.write("    for (n_%s = 0;n_%s < gen_nb_%s;n_%s++) {\n" % (
+	           nam, nam, type, nam))
+    
+    # log the memory usage
+    if no_mem == 0:
+	test.write("        mem_base = xmlMemBlocks();\n");
+
+    # prepare the call
+    for arg in t_args:
+        (nam, type, rtype, info) = arg;
+	#
+	test.write("        %s = gen_%s(n_%s);\n" % (nam, type, nam))
+
+    # do the call, and clanup the result
+    if t_ret != None:
+	test.write("\n        ret_val = %s(" % (name))
+	need = 0
+	for arg in t_args:
+	    (nam, type, rtype, info) = arg
+	    if need:
+	        test.write(", ")
+	    else:
+	        need = 1
+	    test.write("%s" % nam);
+	test.write(");\n        desret_%s(ret_val);\n" % t_ret[0])
+    else:
+	test.write("\n        %s(" % (name));
+	need = 0;
+	for arg in t_args:
+	    (nam, type, rtype, info) = arg;
+	    if need:
+	        test.write(", ")
+	    else:
+	        need = 1
+	    test.write("%s" % nam)
+	test.write(");\n")
+    test.write("        call_tests++;\n");
+
+    # Free the arguments
+    for arg in t_args:
+        (nam, type, rtype, info) = arg;
+	#
+	test.write("        des_%s(n_%s, %s);\n" % (type, nam, nam))
+
+    test.write("        xmlResetLastError();\n");
+    # Check the memory usage
+    if no_mem == 0:
+	test.write("""        if (mem_base != xmlMemBlocks()) {
+            printf("Leak of %%d blocks found in %s\\n",
+	           xmlMemBlocks() - mem_base);
+	    ret++;
+        }
+""" % (name));
+
+    for arg in t_args:
+	test.write("    }\n")
+
+    #
+    # end of conditional
+    #
+    while nb_cond > 0:
+        test.write("#endif\n")
+	nb_cond = nb_cond -1
+
+    nb_tests = nb_tests + 1;
+
+    test.write("""
+    return(ret);
+}
+
+""")
+    
+#
+# Generate all module callers
+#
+for module in modules:
+    # gather all the functions exported by that module
+    try:
+	functions = ctxt.xpathEval("/api/symbols/function[@file='%s']" % (module))
+    except:
+        print "Failed to gather functions from module %s" % (module)
+	continue;
+
+    # iterate over all functions in the module generating the test
+    for function in functions:
+        generate_test(module, function);
+
+    # header
+    test.write("""static int
+test_%s(void) {
+    int ret = 0;
+
+    printf("Testing %s ...\\n");
+""" % (module, module))
+
+    # iterate over all functions in the module generating the call
+    for function in functions:
+        name = function.xpathEval('string(@name)')
+	if is_skipped_function(name):
+	    continue
+	test.write("    ret += test_%s();\n" % (name))
+
+    # footer
+    test.write("""
+    if (ret != 0)
+	printf("Module %s: %%d errors\\n", ret);
+    return(ret);
+}
+""" % (module))
+
+print "Generated test for %d modules and %d functions" %(len(modules), nb_tests)
+nr = 0
+miss = 'none'
+for missing in missing_types.keys():
+    n = len(missing_types[missing])
+    if n > nr:
+        miss = missing
+	nr = n
+
+if nr > 0:
+    print "most needed type support: %s %d times" % (miss, nr)
+
+

include/libxml/xmlmemory.h

@@ -139,6 +139,8 @@ XMLPUBFUN void XMLCALL
  */
 XMLPUBFUN int XMLCALL	
 	xmlMemUsed	(void);
+XMLPUBFUN int XMLCALL	
+	xmlMemBlocks	(void);
 XMLPUBFUN void XMLCALL	
 	xmlMemDisplay	(FILE *fp);
 XMLPUBFUN void XMLCALL	

nanoftp.c

@@ -260,12 +260,18 @@ xmlNanoFTPCleanup(void) {