fix the comment to describe the real return values lot of work on the

* encoding.c: fix the comment to describe the real return values
* pattern.c xpath.c include/libxml/pattern.h: lot of work on
  the patterns, pluggin in the XPath default evaluation, but
  disabled right now because it's not yet good enough for XSLT.
  pattern.h streaming API are likely to be changed to handle
  relative and absolute paths in the same expression.
Daniel

ChangeLog

+Wed Feb 16 01:19:27 CET 2005 Daniel Veillard <daniel@veillard.com>
+
+	* encoding.c: fix the comment to describe the real return values
+	* pattern.c xpath.c include/libxml/pattern.h: lot of work on
+	  the patterns, pluggin in the XPath default evaluation, but
+	  disabled right now because it's not yet good enough for XSLT.
+	  pattern.h streaming API are likely to be changed to handle
+	  relative and absolute paths in the same expression.
+
 Tue Feb 15 15:33:32 CET 2005 Kasimier Buchcik <libxml2-cvs@cazic.net>
 	* xmlschemas.c: Added IDC evaluation for attribute nodes.

encoding.c

@@ -223,7 +223,7 @@ UTF8Toascii(unsigned char* out, int *outlen,
  *
  * Take a block of ISO Latin 1 chars in and try to convert it to an UTF-8
  * block of chars out.
- * Returns 0 if success, or -1 otherwise
+ * Returns the number of bytes written if success, or -1 otherwise
  * The value of @inlen after return is the number of octets consumed
  *     if the return value is positive, else unpredictable.
  * The value of @outlen after return is the number of octets consumed.
@@ -311,7 +311,8 @@ UTF8ToUTF8(unsigned char* out, int *outlen,
  * Take a block of UTF-8 chars in and try to convert it to an ISO Latin 1
  * block of chars out.
  *
- * Returns 0 if success, -2 if the transcoding fails, or -1 otherwise
+ * Returns the number of bytes written if success, -2 if the transcoding fails,
+           or -1 otherwise
  * The value of @inlen after return is the number of octets consumed
  *     if the return value is positive, else unpredictable.
  * The value of @outlen after return is the number of octets consumed.

include/libxml/pattern.h

@@ -48,6 +48,12 @@ XMLPUBFUN int XMLCALL
 typedef struct _xmlStreamCtxt xmlStreamCtxt;
 typedef xmlStreamCtxt *xmlStreamCtxtPtr;
 
+XMLPUBFUN int XMLCALL
+			xmlPatternStreamable	(xmlPatternPtr comp);
+XMLPUBFUN int XMLCALL
+			xmlPatternMaxDepth	(xmlPatternPtr comp);
+XMLPUBFUN int XMLCALL
+			xmlPatternFromRoot	(xmlPatternPtr comp);
 XMLPUBFUN xmlStreamCtxtPtr XMLCALL
 			xmlPatternGetStreamCtxt	(xmlPatternPtr comp);
 XMLPUBFUN void XMLCALL

pattern.c

@@ -114,12 +114,16 @@ struct _xmlStepOp {
     const xmlChar *value2;
 };
 
+#define PAT_FROM_ROOT	1
+#define PAT_FROM_CUR	2
+
 struct _xmlPattern {
     void *data;    		/* the associated template */
     xmlDictPtr dict;		/* the optional dictionnary */
     struct _xmlPattern *next;	/* next pattern if | is used */
     const xmlChar *pattern;	/* the pattern */
 
+    int flags;			/* flags */
     int nbStep;
     int maxStep;
     xmlStepOpPtr steps;        /* ops for computation */
@@ -357,9 +361,19 @@ xsltSwapTopPattern(xmlPatternPtr comp) {
  */
 static int
 xmlReversePattern(xmlPatternPtr comp) {
-    int i = 0;
-    int j = comp->nbStep - 1;
-
+    int i, j;
+
+    /*
+     * remove the leading // for //a or .//a
+     */
+    if ((comp->nbStep > 0) && (comp->steps[0].op == XML_OP_ANCESTOR)) {
+        for (i = 0, j = 1;j < comp->nbStep;i++,j++) {
+	    comp->steps[i].value = comp->steps[j].value;
+	    comp->steps[i].value2 = comp->steps[j].value2;
+	    comp->steps[i].op = comp->steps[j].op;
+	}
+	comp->nbStep--;
+    }
     if (comp->nbStep >= comp->maxStep) {
         xmlStepOpPtr temp;
 	temp = (xmlStepOpPtr) xmlRealloc(comp->steps, comp->maxStep * 2 *
@@ -372,6 +386,8 @@ xmlReversePattern(xmlPatternPtr comp) {
 	comp->steps = temp;
 	comp->maxStep *= 2;
     }
+    i = 0;
+    j = comp->nbStep - 1;
     while (j > i) {
 	register const xmlChar *tmp;
 	register xmlPatOp op;
@@ -932,6 +948,7 @@ xmlCompileStepPattern(xmlPatParserContextPtr ctxt) {
 		    }
 		}
 		TODO
+		ctxt->error = 1;
 		/* URI = xsltGetQNameURI(ctxt->elem, &token); */
 		if (token == NULL) {
 		    ctxt->error = 1;
@@ -952,6 +969,7 @@ xmlCompileStepPattern(xmlPatParserContextPtr ctxt) {
 		    goto error;
 		}
 		TODO
+		ctxt->error = 1;
 		/* URI = xsltGetQNameURI(ctxt->elem, &token); */
 		if (token == NULL) {
 		    ctxt->error = 1;
@@ -1000,23 +1018,24 @@ error:
 static void
 xmlCompilePathPattern(xmlPatParserContextPtr ctxt) {
     SKIP_BLANKS;
+    if (CUR == '/') {
+        ctxt->comp->flags |= PAT_FROM_ROOT;
+    } else if (CUR == '.') {
+        ctxt->comp->flags |= PAT_FROM_CUR;
+    }
     if ((CUR == '/') && (NXT(1) == '/')) {
-	/*
-	 * since we reverse the query
-	 * a leading // can be safely ignored
-	 */
+	PUSH(XML_OP_ANCESTOR, NULL, NULL);
 	NEXT;
 	NEXT;
     } else if ((CUR == '.') && (NXT(1) == '/') && (NXT(2) == '/')) {
-	/*
-	 * a leading .// can be safely ignored
-	 */
+	PUSH(XML_OP_ANCESTOR, NULL, NULL);
 	NEXT;
 	NEXT;
 	NEXT;
     }
     if (CUR == '@') {
 	TODO
+	ctxt->error = 1;
     } else {
         if (CUR == '/') {
 	    PUSH(XML_OP_ROOT, NULL, NULL);
@@ -1041,6 +1060,11 @@ xmlCompilePathPattern(xmlPatParserContextPtr ctxt) {
 	    }
 	}
     }
+    if (CUR != 0) {
+	ERROR5(NULL, NULL, NULL,
+	       "Failed to compile pattern %s\n", ctxt->base);
+	ctxt->error = 1;
+    }
 error:
     return;
 }
@@ -1207,6 +1231,20 @@ xmlStreamCompile(xmlPatternPtr comp) {
 
     if ((comp == NULL) || (comp->steps == NULL))
         return(-1);
+    /*
+     * special case for .
+     */
+    if ((comp->nbStep == 1) &&
+        (comp->steps[0].op == XML_OP_ELEM) &&
+	(comp->steps[0].value == NULL) &&
+	(comp->steps[0].value2 == NULL)) {
+	stream = xmlNewStreamComp(0);
+	if (stream == NULL)
+	    return(-1);
+	comp->stream = stream;
+	return(0);
+    }
+
     stream = xmlNewStreamComp((comp->nbStep / 2) + 1);
     if (stream == NULL)
         return(-1);
@@ -1385,7 +1423,7 @@ xmlStreamPush(xmlStreamCtxtPtr stream,
 		tmp = xmlStreamCtxtAddState(stream, 0, 0);
 		if (tmp < 0)
 		    err++;
-		if (comp->steps[tmp].flags & XML_STREAM_STEP_FINAL)
+		if (comp->nbStep == 0)
 		    ret = 1;
 		stream = stream->next;
 		continue; /* while */
@@ -1592,6 +1630,8 @@ xmlPatterncompile(const xmlChar *pattern, xmlDict *dict,
 	ctxt->comp = cur;
 
 	xmlCompilePathPattern(ctxt);
+	if (ctxt->error != 0)
+	    goto error;
 	xmlFreePatParserContext(ctxt);
 
 
@@ -1675,4 +1715,75 @@ failed:
     return(NULL);
 }
 
+/**
+ * xmlPatternStreamable:
+ * @comp: the precompiled pattern
+ *
+ * Check if the pattern is streamable i.e. xmlPatternGetStreamCtxt()
+ * should work.
+ *
+ * Returns 1 if streamable, 0 if not and -1 in case of error.
+ */
+int
+xmlPatternStreamable(xmlPatternPtr comp) {
+    if (comp == NULL)
+        return(-1);
+    while (comp != NULL) {
+        if (comp->stream == NULL)
+	    return(0);
+	comp = comp->next;
+    }
+    return(1);
+}
+
+/**
+ * xmlPatternMaxDepth:
+ * @comp: the precompiled pattern
+ *
+ * Check the maximum depth reachable by a pattern
+ *
+ * Returns -2 if no limit (using //), otherwise the depth,
+ *         and -1 in case of error
+ */
+int
+xmlPatternMaxDepth(xmlPatternPtr comp) {
+    int ret = 0, i;
+    if (comp == NULL)
+        return(-1);
+    while (comp != NULL) {
+        if (comp->stream == NULL)
+	    return(-1);
+	for (i = 0;i < comp->stream->nbStep;i++)
+	    if (comp->stream->steps[i].flags & XML_STREAM_STEP_DESC)
+	        return(-2);
+	if (comp->stream->nbStep > ret)
+	    ret = comp->stream->nbStep;
+	comp = comp->next;
+    }
+    return(ret);
+
+}
+
+/**
+ * xmlPatternFromRoot:
+ * @comp: the precompiled pattern
+ *
+ * Check if the pattern must be looked at from the root.
+ *
+ * Returns 1 if true, 0 if false and -1 in case of error
+ */
+int
+xmlPatternFromRoot(xmlPatternPtr comp) {
+    if (comp == NULL)
+        return(-1);
+    while (comp != NULL) {
+        if (comp->stream == NULL)
+	    return(-1);
+	if (comp->flags & PAT_FROM_ROOT)
+	    return(1);
+	comp = comp->next;
+    }
+    return(0);
+
+}
 #endif /* LIBXML_PATTERN_ENABLED */

xpath.c

@@ -51,6 +51,22 @@
 #include <libxml/xmlerror.h>
 #include <libxml/threads.h>
 #include <libxml/globals.h>
+#ifdef LIBXML_PATTERN_ENABLED
+#include <libxml/pattern.h>
+#endif
+
+#ifdef LIBXML_PATTERN_ENABLED
+/* currently only in testing for DV as it's not yet solid enough for XSLT */
+/*
+ * TODO:
+ *  - fix the | where there is both relative and absolute expressions
+ *    probably need new pattens APIs to separate both e.g "//b | a "
+ *  - fix also the 0 depth trick used for "/" and "." when or'ed
+ *  - double check /. is a noop
+ *  - libxslt tests show a mem leak of a few bytes
+ */
+/* #define XPATH_STREAMING */
+#endif
 
 #define TODO 								\
     xmlGenericError(xmlGenericErrorContext,				\
@@ -436,6 +452,9 @@ struct _xmlXPathCompExpr {
     int nb;
     xmlChar *string;
 #endif
+#ifdef XPATH_STREAMING
+    xmlPatternPtr stream;
+#endif
 };
 
 /************************************************************************
@@ -521,6 +540,11 @@ xmlXPathFreeCompExpr(xmlXPathCompExprPtr comp)
     if (comp->string != NULL) {
         xmlFree(comp->string);
     }
+#endif
+#ifdef XPATH_STREAMING
+    if (comp->stream != NULL) {
+        xmlFreePatternList(comp->stream);
+    }
 #endif
     if (comp->expr != NULL) {
         xmlFree(comp->expr);
@@ -4042,8 +4066,15 @@ xmlXPathFreeParserContext(xmlXPathParserContextPtr ctxt) {
     if (ctxt->valueTab != NULL) {
         xmlFree(ctxt->valueTab);
     }
-    if (ctxt->comp)
+    if (ctxt->comp != NULL) {
+#ifdef XPATH_STREAMING
+	if (ctxt->comp->stream != NULL) {
+	    xmlFreePatternList(ctxt->comp->stream);
+	    ctxt->comp->stream = NULL;
+	}
+#endif
 	xmlXPathFreeCompExpr(ctxt->comp);
+    }
     xmlFree(ctxt);
 }
 
@@ -10904,6 +10935,163 @@ xmlXPathCompOpEval(xmlXPathParserContextPtr ctxt, xmlXPathStepOpPtr op)
     return (total);
 }
 
+#ifdef XPATH_STREAMING
+/**
+ * xmlXPathRunStreamEval:
+ * @ctxt:  the XPath parser context with the compiled expression
+ *
+ * Evaluate the Precompiled Streamable XPath expression in the given context.
+ */
+static xmlXPathObjectPtr
+xmlXPathRunStreamEval(xmlXPathContextPtr ctxt, xmlPatternPtr comp) {
+    int max_depth;
+    int from_root;
+    int ret, depth;
+    xmlNodePtr cur, limit;
+    xmlXPathObjectPtr retval;
+    xmlStreamCtxtPtr patstream;
+
+    int nb_nodes = 0;
+
+    if ((ctxt == NULL) || (comp == NULL))
+        return(NULL);
+    max_depth = xmlPatternMaxDepth(comp);
+    if (max_depth == -1)
+        return(NULL);
+    if (max_depth == -2)
+        max_depth = 10000;
+    from_root = xmlPatternFromRoot(comp);
+    if (from_root < 0)
+        return(NULL);
+/*    printf("stream eval: depth %d from root %d\n", max_depth, from_root); */
+
+    retval = xmlXPathNewNodeSet(NULL);
+    if (retval == NULL)
+        return(NULL);
+    
+    /* FIXME '. | /' */
+    if ((from_root) && (max_depth == 0)) {
+	xmlXPathNodeSetAddUnique(retval->nodesetval, (xmlNodePtr) ctxt->doc);
+	return(retval);
+    } else if (max_depth == 0) {
+	xmlXPathNodeSetAddUnique(retval->nodesetval, ctxt->node);
+	return(retval);
+    }
+    if (from_root) {
+        cur = ctxt->doc;
+	limit = NULL;
+    } else if (ctxt->node != NULL) {
+        switch (ctxt->node->type) {
+            case XML_ELEMENT_NODE:
+            case XML_DOCUMENT_NODE:
+            case XML_DOCUMENT_FRAG_NODE:
+            case XML_HTML_DOCUMENT_NODE:
+#ifdef LIBXML_DOCB_ENABLED
+            case XML_DOCB_DOCUMENT_NODE:
+#endif
+	        cur = ctxt->node;
+		break;
+            case XML_ATTRIBUTE_NODE:
+            case XML_TEXT_NODE:
+            case XML_CDATA_SECTION_NODE:
+            case XML_ENTITY_REF_NODE:
+            case XML_ENTITY_NODE:
+            case XML_PI_NODE:
+            case XML_COMMENT_NODE:
+            case XML_NOTATION_NODE:
+            case XML_DTD_NODE:
+            case XML_DOCUMENT_TYPE_NODE:
+            case XML_ELEMENT_DECL:
+            case XML_ATTRIBUTE_DECL:
+            case XML_ENTITY_DECL:
+            case XML_NAMESPACE_DECL:
+            case XML_XINCLUDE_START:
+            case XML_XINCLUDE_END:
+	        cur = NULL;
+		break;
+	}
+	limit = cur;
+    }
+    if (cur == NULL)
+        return(retval);
+
+    patstream = xmlPatternGetStreamCtxt(comp);
+    if (patstream == NULL) {
+        return(retval);
+    }
+
+    if (from_root) {
+	ret = xmlStreamPush(patstream, NULL, NULL);
+	if (ret < 0) {
+	} else if (ret == 1) {
+	    xmlXPathNodeSetAddUnique(retval->nodesetval, cur);
+	}
+    }
+
+    depth = 0;
+    goto scan_children;
+    do {
+next_node:
+        nb_nodes++;
+	if (cur->type == XML_ELEMENT_NODE) {
+	    ret = xmlStreamPush(patstream, cur->name,
+				(cur->ns ? cur->ns->href : NULL));
+	    if (ret < 0) {
+	    } else if (ret == 1) {
+		xmlXPathNodeSetAddUnique(retval->nodesetval, cur);
+	    }
+	    if ((cur->children == NULL) || (depth >= max_depth)) {
+		ret = xmlStreamPop(patstream);
+	    }
+	}
+        
+scan_children:
+	if ((cur->children != NULL) && (depth < max_depth)) {
+	    /*
+	     * Do not descend on entities declarations
+	     */
+	    if (cur->children->type != XML_ENTITY_DECL) {
+		cur = cur->children;
+		depth++;
+		/*
+		 * Skip DTDs
+		 */
+		if (cur->type != XML_DTD_NODE)
+		    continue;
+	    }
+	}
+
+	if (cur == limit)
+	    break;
+
+	while (cur->next != NULL) {
+	    cur = cur->next;
+	    if ((cur->type != XML_ENTITY_DECL) &&
+		(cur->type != XML_DTD_NODE))
+		goto next_node;
+	}
+	
+	do {
+	    ret = xmlStreamPop(patstream);
+	    cur = cur->parent;
+	    depth--;
+	    if ((cur == NULL) || (cur == limit))
+	        goto done;
+	    if (cur->next != NULL) {
+		cur = cur->next;
+		break;
+	    }
+	} while (cur != NULL);
+
+    } while ((cur != NULL) && (depth >= 0));
+done:
+/*    printf("stream eval: checked %d nodes selected %d\n",
+           nb_nodes, retval->nodesetval->nodeNr); */
+    xmlFreeStreamCtxt(patstream);
+    return(retval);
+}
+#endif /* XPATH_STREAMING */
+
 /**
  * xmlXPathRunEval:
  * @ctxt:  the XPath parser context with the compiled expression
@@ -10929,6 +11117,16 @@ xmlXPathRunEval(xmlXPathParserContextPtr ctxt) {
 	ctxt->valueMax = 10;
 	ctxt->value = NULL;
     }
+#ifdef XPATH_STREAMING
+    if (ctxt->comp->stream) {
+        xmlXPathObjectPtr ret;
+        ret = xmlXPathRunStreamEval(ctxt->context, ctxt->comp->stream);
+	if (ret != NULL) {
+	    valuePush(ctxt, ret);
+	    return;
+	}
+    }
+#endif
     comp = ctxt->comp;
     if(comp->last < 0) {
 	xmlGenericError(xmlGenericErrorContext,
@@ -11034,6 +11232,68 @@ xmlXPathEvaluatePredicateResult(xmlXPathParserContextPtr ctxt,
     return(0);
 }
 
+#ifdef XPATH_STREAMING
+/**
+ * xmlXPathTryStreamCompile:
+ * @ctxt: an XPath context
+ * @str:  the XPath expression
+ *
+ * Try to compile the XPath expression as a streamable subset.
+ *
+ * Returns the compiled expression or NULL if failed to compile.
+ */
+static xmlXPathCompExprPtr
+xmlXPathTryStreamCompile(xmlXPathContextPtr ctxt, const xmlChar *str) {
+    /*
+     * Optimization: use streaming patterns when the XPath expression can
+     * be compiled to a stream lookup
+     */
+    xmlPatternPtr stream;
+    xmlXPathCompExprPtr comp;
+    xmlDictPtr dict = NULL;
+    const xmlChar **namespaces = NULL;
+    xmlNsPtr ns;
+    int i, j;
+
+    if ((!xmlStrchr(str, '[')) && (!xmlStrchr(str, '(')) &&
+        (!xmlStrchr(str, '@'))) {
+	if (ctxt != NULL) {
+	    dict = ctxt->dict;
+	    if (ctxt->nsNr > 0) {
+		namespaces = xmlMalloc(2 * (ctxt->nsNr + 1));
+		if (namespaces == NULL) {
+		    xmlXPathErrMemory(ctxt, "allocating namespaces array\n");
+		    return(NULL);
+		}
+		for (i = 0, j = 0; (j < ctxt->nsNr); j++) {
+		    ns = ctxt->namespaces[j];
+		    namespaces[i++] = ns->href;
+		    namespaces[i++] = ns->prefix;
+		}
+		namespaces[i++] = NULL;
+		namespaces[i++] = NULL;
+	    }
+	}
+
+	stream = xmlPatterncompile(str, dict, 0, &namespaces[0]);
+	if ((stream != NULL) && (xmlPatternStreamable(stream) == 1)) {
+	    comp = xmlXPathNewCompExpr();
+	    if (comp == NULL) {
+		xmlXPathErrMemory(ctxt, "allocating streamable expression\n");
+		return(NULL);
+	    }
+	    comp->stream = stream;
+	    comp->dict = dict;
+	    if (comp->dict)
+		xmlDictReference(comp->dict);
+	    return(comp);
+	}
+	xmlFreePattern(stream);
+    }
+    return(NULL);
+}
+#endif /* XPATH_STREAMING */
+
 /**
  * xmlXPathCtxtCompile:
  * @ctxt: an XPath context
@@ -11049,6 +11309,12 @@ xmlXPathCtxtCompile(xmlXPathContextPtr ctxt, const xmlChar *str) {
     xmlXPathParserContextPtr pctxt;
     xmlXPathCompExprPtr comp;
 
+#ifdef XPATH_STREAMING
+    comp = xmlXPathTryStreamCompile(ctxt, str);
+    if (comp != NULL)
+        return(comp);
+#endif
+
     xmlXPathInit();
 
     pctxt = xmlXPathNewParserContext(str, ctxt);
@@ -11184,8 +11450,25 @@ xmlXPathCompiledEval(xmlXPathCompExprPtr comp, xmlXPathContextPtr ctx) {
  */
 void
 xmlXPathEvalExpr(xmlXPathParserContextPtr ctxt) {
+#ifdef XPATH_STREAMING
+    xmlXPathCompExprPtr comp;
+#endif
+
     if (ctxt == NULL) return;
-    xmlXPathCompileExpr(ctxt);
+    
+#ifdef XPATH_STREAMING
+    comp = xmlXPathTryStreamCompile(ctxt->context, ctxt->base);
+    if (comp != NULL) {
+        if (ctxt->comp != NULL)
+	    xmlXPathFreeCompExpr(ctxt->comp);
+        ctxt->comp = comp;
+	if (ctxt->cur != NULL)
+	    while (*ctxt->cur != 0) ctxt->cur++;
+    } else
+#endif
+    {
+	xmlXPathCompileExpr(ctxt);
+    }
     CHECK_ERROR;
     xmlXPathRunEval(ctxt);
 }
@@ -11217,7 +11500,11 @@ xmlXPathEval(const xmlChar *str, xmlXPathContextPtr ctx) {
 	xmlGenericError(xmlGenericErrorContext,
 		"xmlXPathEval: evaluation failed\n");
 	res = NULL;
-    } else if (*ctxt->cur != 0) {
+    } else if ((*ctxt->cur != 0) && (ctxt->comp != NULL) 
+#ifdef XPATH_STREAMING
+            && (ctxt->comp->stream == NULL)
+#endif
+	      ) {
 	xmlXPatherror(ctxt, __FILE__, __LINE__, XPATH_EXPR_ERROR);
 	res = NULL;
     } else {