JSPWIKI-1206 JSPWIKI-1221 Migrate from NekoHTML to htmlunit-neko

ChangeLog.md

@@ -39,6 +39,7 @@ under the License.
 
 * Fix for [JSPWIKI-922](https://issues.apache.org/jira/browse/JSPWIKI-922): the `Content-Disposition` header of an attachment download carried a name with non-ASCII characters as a raw byte, which is not valid UTF-8; such names get an ASCII fallback and a `filename*` parameter with the UTF-8 name now ([PR#571](https://github.com/apache/jspwiki/pull/571))
 * Fix for [JSPWIKI-501](https://issues.apache.org/jira/browse/JSPWIKI-501): wildcard and prefix searches that contain an underscore find the pages ([PR#543](https://github.com/apache/jspwiki/pull/543))
+* [JSPWIKI-1206](https://issues.apache.org/jira/browse/JSPWIKI-1206) and [JSPWIKI-1221](https://issues.apache.org/jira/browse/JSPWIKI-1221): NekoHTML is replaced by htmlunit-neko 5.7.0 in the HTML to wiki translator ([PR#548](https://github.com/apache/jspwiki/pull/548))
 * Fix for [JSPWIKI-1138](https://issues.apache.org/jira/browse/JSPWIKI-1138): Install.jsp echoed submitted values without escaping them; they are escaped now ([PR#323](https://github.com/apache/jspwiki/pull/323))
 * Fix for [JSPWIKI-1164](https://issues.apache.org/jira/browse/JSPWIKI-1164): a page reached from a search result showed `<` as `&lt;` when the query was highlighted ([PR#538](https://github.com/apache/jspwiki/pull/538))
 * Fix for [JSPWIKI-919](https://issues.apache.org/jira/browse/JSPWIKI-919): `VersioningFileProvider#pageExists()` ignored the first version of a page that has no history, so a page dropped into the page directory was reported as missing when the page cache is off ([PR#539](https://github.com/apache/jspwiki/pull/539))

LICENSE

@@ -270,7 +270,7 @@ lucene-queries-9.12.2.jar                   LICENSE
 lucene-queryparser-9.12.2.jar               LICENSE
 lucene-sandbox-9.12.2.jar                   LICENSE
 javax.mail-1.6.2.jar                        ./jspwiki-war/src/main/config/doc/LICENSE.cddl
-nekohtml-2.0.2.jar (codelibs fork)          LICENSE
+neko-htmlunit-5.7.0.jar                     LICENSE
 org.suigeneris.jrcs.diff-0.4.2              LICENSE
 oro-2.0.8.jar                               LICENSE
 sandler-0.5.jar                             ./jspwiki-war/src/main/config/doc/LICENSE.sandler

jspwiki-main/pom.xml

@@ -157,11 +157,6 @@
             <artifactId>lucene-queryparser</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>org.codelibs</groupId>
-            <artifactId>nekohtml</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>org.freshcookies</groupId>
             <artifactId>freshcookies-security</artifactId>

jspwiki-markdown/src/test/java/org/apache/wiki/htmltowiki/syntax/markdown/HtmlStringToMarkdownWikiTranslatorTest.java

@@ -159,9 +159,9 @@ public void testLists() throws Exception {
     public void testPre() throws Exception {
         Assertions.assertEquals( "\n``hallo``\n", html2wiki.translate( "<pre>hallo</pre>" ) );
         Assertions.assertEquals( "\n``Hallo\nWelt!\n\n``\n", html2wiki.translate( "<pre>Hallo<br>Welt!<br><br></pre>" ) );
-        Assertions.assertEquals( "\n``\n\n\n\nHallo\n\n\n\nWelt!\n\n\n\n``\n", html2wiki.translate( "\n\n\n\n<pre>\n\n\n\nHallo\n\n\n\nWelt!\n\n\n\n</pre>\n\n\n\n" ) );
+        Assertions.assertEquals( "\n``\n\n\nHallo\n\n\n\nWelt!\n\n\n\n``\n", html2wiki.translate( "\n\n\n\n<pre>\n\n\n\nHallo\n\n\n\nWelt!\n\n\n\n</pre>\n\n\n\n" ) );
 
-        Assertions.assertEquals( "\n``\n\n* Baltramon \n  lasdjfh\n\n``\n", html2wiki.translate( "<pre>\n\n* Baltramon \n  lasdjfh\n\n</pre>" ) );
+        Assertions.assertEquals( "\n``\n* Baltramon \n  lasdjfh\n\n``\n", html2wiki.translate( "<pre>\n\n* Baltramon \n  lasdjfh\n\n</pre>" ) );
 
         /*
          * // The style "font-family: courier new" is no longer translated as monospace text, so this test case is no longer needed. Assertions.assertEquals(

jspwiki-wysiwyg/pom.xml

@@ -53,8 +53,8 @@
     </dependency>
 
     <dependency>
-      <groupId>org.codelibs</groupId>
-      <artifactId>nekohtml</artifactId>
+      <groupId>org.htmlunit</groupId>
+      <artifactId>neko-htmlunit</artifactId>
     </dependency>
 
     <dependency>

jspwiki-wysiwyg/src/main/java/org/apache/wiki/htmltowiki/HtmlStringToWikiTranslator.java

@@ -20,13 +20,18 @@ Licensed to the Apache Software Foundation (ASF) under one
 
 import org.apache.wiki.api.core.Context;
 import org.apache.wiki.api.core.Engine;
-import org.codelibs.nekohtml.parsers.SAXParser;
+import org.htmlunit.cyberneko.parsers.SAXParser;
 import org.jdom2.Document;
 import org.jdom2.Element;
 import org.jdom2.JDOMException;
 import org.jdom2.input.SAXBuilder;
-import org.jdom2.input.sax.XMLReaderSAX2Factory;
+import org.jdom2.input.sax.XMLReaderJDOMFactory;
 import org.jdom2.output.XMLOutputter;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+import org.xml.sax.helpers.AttributesImpl;
+import org.xml.sax.helpers.XMLFilterImpl;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -38,7 +43,19 @@ Licensed to the Apache Software Foundation (ASF) under one
  */
 public class HtmlStringToWikiTranslator {
 
-    private static final String CYBERNEKO_PARSER = SAXParser.class.getName();
+    /** Hands JDOM the HTML parser directly, as it would set SAX features on it that the parser does not know about. */
+    private static final XMLReaderJDOMFactory HTML_READER_FACTORY = new XMLReaderJDOMFactory() {
+        @Override
+        public XMLReader createXMLReader() {
+            return new AttributeUriFilter( new SAXParser() );
+        }
+
+        @Override
+        public boolean isValidating() {
+            return false;
+        }
+    };
+
     private final Engine e;
 
     /**
@@ -97,12 +114,12 @@ public String translate( final String html, final XHtmlToWikiConfig config ) thr
      * Use NekoHtml to parse HTML like well-formed XHTML
      *
      * @param html HTML to parse.
-     * @return xhtml jdom root element (node "HTML")
+     * @return xhtml jdom root element (node "html")
      * @throws JDOMException when errors occur in parsing
      * @throws IOException when an I/O error prevents a document from being fully parsed
      */
     private Element htmlStringToElement( final String html ) throws JDOMException, IOException {
-        final SAXBuilder builder = new SAXBuilder( new XMLReaderSAX2Factory( true, CYBERNEKO_PARSER ), null, null );
+        final SAXBuilder builder = new SAXBuilder( HTML_READER_FACTORY, null, null );
         //builder.setProperty( XMLConstants.ACCESS_EXTERNAL_DTD, "" );
         //builder.setProperty( XMLConstants.ACCESS_EXTERNAL_SCHEMA, "" );
         final Document doc = builder.build( new StringReader( html ) );
@@ -121,4 +138,27 @@ public static String element2String( final Element element ) {
         return outputter.outputString( document );
     }
 
+    /**
+     *  The HTML parser reports {@code null} as the namespace URI of the attributes, where the SAX contract says it is the empty string,
+     *  and JDOM cannot handle that.
+     */
+    private static final class AttributeUriFilter extends XMLFilterImpl {
+
+        AttributeUriFilter( final XMLReader parent ) {
+            super( parent );
+        }
+
+        @Override
+        public void startElement( final String uri, final String localName, final String qName, final Attributes atts ) throws SAXException {
+            final AttributesImpl attributes = new AttributesImpl( atts );
+            for( int i = 0; i < attributes.getLength(); i++ ) {
+                if( attributes.getURI( i ) == null ) {
+                    attributes.setURI( i, "" );
+                }
+            }
+            super.startElement( uri, localName, qName, attributes );
+        }
+
+    }
+
 }

jspwiki-wysiwyg/src/main/java/org/apache/wiki/htmltowiki/XHtmlElementToWikiTranslator.java

@@ -130,14 +130,14 @@ public void translateText( final Text element ) {
     }
 
     public void translateImage( final Element base ) {
-        Element child = XmlUtil.getXPathElement( base, "TBODY/TR/TD/*" );
+        Element child = XmlUtil.getXPathElement( base, "tbody/tr/td/*" );
         if( child == null ) {
             child = base;
         }
         final Element img;
         final String href;
-        if( child.getName().equals( "A" ) ) {
-            img = child.getChild( "IMG" );
+        if( child.getName().equals( "a" ) ) {
+            img = child.getChild( "img" );
             href = child.getAttributeValue( "href" );
         } else {
             img = child;
@@ -156,7 +156,7 @@ public void translateImage( final Element base ) {
         putIfNotEmpty( imageAttrs, "height", img.getAttributeValue( "height" ) );
         putIfNotEmpty( imageAttrs, "width", img.getAttributeValue( "width" ) );
         putIfNotEmpty( imageAttrs, "alt", img.getAttributeValue( "alt" ) );
-        putIfNotEmpty( imageAttrs, "caption", emptyToNull( ( Element )XPathFactory.instance().compile(  "CAPTION" ).evaluateFirst( base ) ) );
+        putIfNotEmpty( imageAttrs, "caption", emptyToNull( ( Element )XPathFactory.instance().compile(  "caption" ).evaluateFirst( base ) ) );
         putIfNotEmpty( imageAttrs, "link", href );
         putIfNotEmpty( imageAttrs, "border", img.getAttributeValue( "border" ) );
         putIfNotEmpty( imageAttrs, "style", base.getAttributeValue( "style" ) );
@@ -288,7 +288,7 @@ public void translateChildren( final Element base ) throws JDOMException {
 
     void translateA( final Element e ) throws JDOMException {
         if( config.isNotIgnorableWikiMarkupLink( e ) ) {
-            if( e.getChild( "IMG" ) != null ) {
+            if( e.getChild( "img" ) != null ) {
                 translateImage( e );
             } else {
                 final String ref = config.trimLink( e.getAttributeValue( "href" ) );

jspwiki-wysiwyg/src/main/java/org/apache/wiki/htmltowiki/syntax/FormDecorator.java

@@ -46,7 +46,7 @@ protected FormDecorator( final PrintWriter out, final XHtmlElementToWikiTranslat
      */
     public void decorate( final Element e ) throws JDOMException {
         // remove the hidden input where name="formname" since a new one will be generated again when the xhtml is rendered.
-        final Element formName = XmlUtil.getXPathElement( e, "INPUT[@name='formname']" );
+        final Element formName = XmlUtil.getXPathElement( e, "input[@name='formname']" );
         if( formName != null ) {
             formName.detach();
         }

jspwiki-wysiwyg/src/test/java/org/apache/wiki/htmltowiki/HtmlStringToWikiTranslatorTest.java

@@ -161,10 +161,10 @@ public void testLists() throws Exception {
     public void testPre() throws Exception {
         Assertions.assertEquals( "\n{{{hallo}}}\n", html2wiki.translate( "<pre>hallo</pre>" ) );
         Assertions.assertEquals( "\n{{{Hallo\nWelt!\n\n}}}\n", html2wiki.translate( "<pre>Hallo<br>Welt!<br><br></pre>" ) );
-        Assertions.assertEquals( "\n{{{\n\n\n\nHallo\n\n\n\nWelt!\n\n\n\n}}}\n",
+        Assertions.assertEquals( "\n{{{\n\n\nHallo\n\n\n\nWelt!\n\n\n\n}}}\n",
                 html2wiki.translate( "\n\n\n\n<pre>\n\n\n\nHallo\n\n\n\nWelt!\n\n\n\n</pre>\n\n\n\n" ) );
 
-        Assertions.assertEquals( "\n{{{\n\n* Baltramon \n  lasdjfh\n\n}}}\n", html2wiki.translate( "<pre>\n\n* Baltramon \n  lasdjfh\n\n</pre>" ) );
+        Assertions.assertEquals( "\n{{{\n* Baltramon \n  lasdjfh\n\n}}}\n", html2wiki.translate( "<pre>\n\n* Baltramon \n  lasdjfh\n\n</pre>" ) );
 
         /*
          * // The style "font-family: courier new" is no longer translated as monospace text, so this test case is no longer needed. Assertions.assertEquals(

pom.xml

@@ -82,7 +82,7 @@
     <log4j2.version>2.26.1</log4j2.version>
     <lucene.version>9.12.2</lucene.version>
     <mockito.version>5.20.0</mockito.version>
-    <nekohtml.version>2.1.3</nekohtml.version>
+    <neko-htmlunit.version>5.7.0</neko-htmlunit.version>
     <oro.version>2.0.8</oro.version>
     <rome.version>2.1.0</rome.version>
     <sandler.version>0.5</sandler.version>
@@ -424,9 +424,9 @@
       </dependency>
 
       <dependency>
-        <groupId>org.codelibs</groupId>
-        <artifactId>nekohtml</artifactId>
-        <version>${nekohtml.version}</version>
+        <groupId>org.htmlunit</groupId>
+        <artifactId>neko-htmlunit</artifactId>
+        <version>${neko-htmlunit.version}</version>
       </dependency>
 
       <dependency>