Improved display of plain text messages and text to HTML conversion (#1488937)

Now instead of <pre> we use <div class="pre"> styled with monospace
font. We replace whitespace characters with non-breaking spaces where
needed. I.e. plain text is always unwrappable, until it uses format=flowed,
in such a case only flowed paragraphs are wrappable.

Also conversion of text to HTML in compose editor was modified in the same way.

8 files changed, 398 insertions, 123 deletions

diff --git a/program/js/app.js b/program/js/app.js
index 042ebb724..46d748062 100644
--- a/program/js/app.js
+++ b/program/js/app.js
@@ -3439,17 +3439,23 @@ function rcube_webmail()
   {
     this.stop_spellchecking();
 
-    if (props.mode == 'html') {
-      this.plain2html($('#'+props.id).val(), props.id);
-      tinyMCE.execCommand('mceAddControl', false, props.id);
+    var input = $('#' + props.id);
 
-      if (this.env.default_font)
-        setTimeout(function() {
-          $(tinyMCE.get(props.id).getBody()).css('font-family', rcmail.env.default_font);
-        }, 500);
-    }
-    else if (this.html2plain(tinyMCE.get(props.id).getContent(), props.id))
-      tinyMCE.execCommand('mceRemoveControl', false, props.id);
+    if (props.mode == 'html')
+      this.plain2html(input.val(), function(data) {
+        input.val(data);
+        tinyMCE.execCommand('mceAddControl', false, props.id);
+
+        if (ref.env.default_font)
+          setTimeout(function() {
+            $(tinyMCE.get(props.id).getBody()).css('font-family', ref.env.default_font);
+          }, 500);
+      });
+    else
+      this.html2plain(tinyMCE.get(props.id).getContent(), function(data) {
+        tinyMCE.execCommand('mceRemoveControl', false, props.id);
+        input.val(data);
+      });
 
     return true;
   };
@@ -6809,41 +6815,51 @@ function rcube_webmail()
   /*********  html to text conversion functions   *********/
   /********************************************************/
 
-  this.html2plain = function(htmlText, id)
+  this.html2plain = function(html, func)
+  {
+    return this.format_converter(html, 'html', func);
+  };
+
+  this.plain2html = function(plain, func)
+  {
+    return this.format_converter(plain, 'plain', func);
+  };
+
+  this.format_converter = function(text, format, func)
   {
     // warn the user (if converted content is not empty)
-    if (!htmlText || !(htmlText.replace(/<[^>]+>|&nbsp;|\s/g, '')).length) {
+    if (!text
+      || (format == 'html' && !(text.replace(/<[^>]+>|&nbsp;|\xC2\xA0|\s/g, '')).length)
+      || (format != 'html' && !(text.replace(/\xC2\xA0|\s/g, '')).length)
+    ) {
       // without setTimeout() here, textarea is filled with initial (onload) content
-      setTimeout(function() { $('#'+id).val(''); }, 50);
+      setTimeout(function() { if (func) func(''); }, 50);
       return true;
     }
 
-    if (!confirm(this.get_label('editorwarning')))
+    var confirmed = this.env.editor_warned || confirm(this.get_label('editorwarning'));
+
+    this.env.editor_warned = true;
+
+    if (!confirmed)
       return false;
 
-    var url = '?_task=utils&_action=html2text',
+    var url = '?_task=utils&_action=' + (format == 'html' ? 'html2text' : 'text2html'),
       lock = this.set_busy(true, 'converting');
 
     this.log('HTTP POST: ' + url);
 
-    $.ajax({ type: 'POST', url: url, data: htmlText, contentType: 'application/octet-stream',
+    $.ajax({ type: 'POST', url: url, data: text, contentType: 'application/octet-stream',
       error: function(o, status, err) { ref.http_error(o, status, err, lock); },
-      success: function(data) { ref.set_busy(false, null, lock); $('#'+id).val(data); ref.log(data); }
+      success: function(data) {
+        ref.set_busy(false, null, lock);
+        if (func) func(data);
+      }
     });
 
     return true;
   };
 
-  this.plain2html = function(plain, id)
-  {
-    var lock = this.set_busy(true, 'converting');
-
-    plain = plain.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
-    $('#'+id).val(plain ? '<pre>'+plain+'</pre>' : '');
-
-    this.set_busy(false, null, lock);
-  };
-
 
   /********************************************************/
   /*********        remote request methods        *********/
diff --git a/program/js/editor.js b/program/js/editor.js
index af877d7b5..d23d2b26f 100644
--- a/program/js/editor.js
+++ b/program/js/editor.js
@@ -37,7 +37,7 @@ function rcmail_editor_init(config)
       apply_source_formatting: true,
       theme: 'advanced',
       language: config.lang,
-      content_css: config.skin_path + '/editor_content.css',
+      content_css: config.skin_path + '/editor_content.css?v2',
       theme_advanced_toolbar_location: 'top',
       theme_advanced_toolbar_align: 'left',
       theme_advanced_buttons3: '',
diff --git a/program/lib/Roundcube/rcube_mime.php b/program/lib/Roundcube/rcube_mime.php
index 091b2fae8..370d5a8d5 100644
--- a/program/lib/Roundcube/rcube_mime.php
+++ b/program/lib/Roundcube/rcube_mime.php
@@ -480,15 +480,17 @@ class rcube_mime
     /**
      * Interpret a format=flowed message body according to RFC 2646
      *
-     * @param string  $text Raw body formatted as flowed text
+     * @param string $text Raw body formatted as flowed text
+     * @param string $mark Mark each flowed line with specified character
      *
      * @return string Interpreted text with unwrapped lines and stuffed space removed
      */
-    public static function unfold_flowed($text)
+    public static function unfold_flowed($text, $mark = null)
     {
         $text = preg_split('/\r?\n/', $text);
         $last = -1;
         $q_level = 0;
+        $marks = array();
 
         foreach ($text as $idx => $line) {
             if (preg_match('/^(>+)/', $line, $m)) {
@@ -508,6 +510,10 @@ class rcube_mime
                 ) {
                     $text[$last] .= $line;
                     unset($text[$idx]);
+
+                    if ($mark) {
+                        $marks[$last] = true;
+                    }
                 }
                 else {
                     $last = $idx;
@@ -520,7 +526,7 @@ class rcube_mime
                 }
                 else {
                     // remove space-stuffing
-                    $line = preg_replace('/^\s/', '', $line);
+                    $line = preg_replace('/^ /', '', $line);
 
                     if (isset($text[$last]) && $line
                         && $text[$last] != '-- '
@@ -528,6 +534,10 @@ class rcube_mime
                     ) {
                         $text[$last] .= $line;
                         unset($text[$idx]);
+
+                        if ($mark) {
+                            $marks[$last] = true;
+                        }
                     }
                     else {
                         $text[$idx] = $line;
@@ -538,6 +548,12 @@ class rcube_mime
             $q_level = $q;
         }
 
+        if (!empty($marks)) {
+            foreach (array_keys($marks) as $mk) {
+                $text[$mk] = $mark . $text[$mk];
+            }
+        }
+
         return implode("\r\n", $text);
     }
 
diff --git a/program/lib/Roundcube/rcube_text2html.php b/program/lib/Roundcube/rcube_text2html.php
new file mode 100644
index 000000000..5da771ac8
--- /dev/null
+++ b/program/lib/Roundcube/rcube_text2html.php
@@ -0,0 +1,277 @@
+<?php
+
+/**
+ +-----------------------------------------------------------------------+
+ | This file is part of the Roundcube Webmail client                     |
+ | Copyright (C) 2008-2014, The Roundcube Dev Team                       |
+ |                                                                       |
+ | Licensed under the GNU General Public License version 3 or            |
+ | any later version with exceptions for skins & plugins.                |
+ | See the README file for a full license statement.                     |
+ |                                                                       |
+ | PURPOSE:                                                              |
+ |   Converts plain text to HTML                                         |
+ +-----------------------------------------------------------------------+
+ | Author: Aleksander Machniak <alec@alec.pl>                            |
+ +-----------------------------------------------------------------------+
+ */
+
+/**
+ * Converts plain text to HTML
+ *
+ * @package    Framework
+ * @subpackage Utils
+ */
+class rcube_text2html
+{
+    /**
+     * Contains the HTML content after conversion.
+     *
+     * @var string $html
+     */
+    protected $html;
+
+    /**
+     * Contains the plain text.
+     *
+     * @var string $text
+     */
+    protected $text;
+
+    /**
+     * Configuration
+     *
+     * @var array $config
+     */
+    protected $config = array(
+        // non-breaking space
+        'space'  => "\xC2\xA0",
+        // enables format=flowed parser
+        'flowed' => false,
+        // enables wrapping for non-flowed text
+        'wrap'   => false,
+        // line-break tag
+        'break'  => "<br>\n",
+        // prefix and suffix (wrapper element)
+        'begin'  => '<div class="pre">',
+        'end'    => '</end>',
+        // enables links replacement
+        'links'  => true,
+    );
+
+
+    /**
+     * Constructor.
+     *
+     * If the plain text source string (or file) is supplied, the class
+     * will instantiate with that source propagated, all that has
+     * to be done it to call get_html().
+     *
+     * @param string  $source    Plain text
+     * @param boolean $from_file Indicates $source is a file to pull content from
+     * @param array   $config    Class configuration
+     */
+    function __construct($source = '', $from_file = false, $config = array())
+    {
+        if (!empty($source)) {
+            $this->set_text($source, $from_file);
+        }
+
+        if (!empty($config) && is_array($config)) {
+            $this->config = array_merge($this->config, $config);
+        }
+    }
+
+    /**
+     * Loads source text into memory, either from $source string or a file.
+     *
+     * @param string  $source    Plain text
+     * @param boolean $from_file Indicates $source is a file to pull content from
+     */
+    function set_text($source, $from_file = false)
+    {
+        if ($from_file && file_exists($source)) {
+            $this->text = file_get_contents($source);
+        }
+        else {
+            $this->text = $source;
+        }
+
+        $this->_converted = false;
+    }
+
+    /**
+     * Returns the HTML content.
+     *
+     * @return string HTML content
+     */
+    function get_html()
+    {
+        if (!$this->_converted) {
+            $this->_convert();
+        }
+
+        return $this->html;
+    }
+
+    /**
+     * Prints the HTML.
+     */
+    function print_html()
+    {
+        print $this->get_html();
+    }
+
+    /**
+     * Workhorse function that does actual conversion (calls _converter() method).
+     */
+    protected function _convert()
+    {
+        $text = stripslashes($this->text);
+
+        // Convert TXT to HTML
+        $this->html       = $this->_converter($text);
+        $this->_converted = true;
+    }
+
+    /**
+     * Workhorse function that does actual conversion.
+     *
+     * @param string Plain text
+     */
+    protected function _converter($text)
+    {
+        // make links and email-addresses clickable
+        $attribs  = array('link_attribs' => array('rel' => 'noreferrer', 'target' => '_blank'));
+        $replacer = new rcmail_string_replacer($attribs);
+
+        if ($this->config['flowed']) {
+            $flowed_char = 0x01;
+            $text        = rcube_mime::unfold_flowed($text, chr($flowed_char));
+        }
+
+        // search for patterns like links and e-mail addresses and replace with tokens
+        if ($this->config['links']) {
+            $text = $replacer->replace($text);
+        }
+
+        // split body into single lines
+        $text        = preg_split('/\r?\n/', $text);
+        $quote_level = 0;
+        $last        = -1;
+
+        // find/mark quoted lines...
+        for ($n=0, $cnt=count($text); $n < $cnt; $n++) {
+            $flowed = false;
+            if ($this->config['flowed'] && ord($text[0]) == $flowed_char) {
+                $flowed   = true;
+                $text[$n] = substr($text[$n], 1);
+            }
+
+            if ($text[$n][0] == '>' && preg_match('/^(>+ {0,1})+/', $text[$n], $regs)) {
+                $q        = substr_count($regs[0], '>');
+                $text[$n] = substr($text[$n], strlen($regs[0]));
+                $text[$n] = $this->_convert_line($text[$n], $flowed || $this->config['wrap']);
+
+                if ($q > $quote_level) {
+                    $text[$n] = $replacer->get_replacement($replacer->add(
+                        str_repeat('<blockquote>', $q - $quote_level))) . $text[$n];
+                    $last = $n;
+                }
+                else if ($q < $quote_level) {
+                    $text[$n] = $replacer->get_replacement($replacer->add(
+                        str_repeat('</blockquote>', $quote_level - $q))) . $text[$n];
+                    $last = $n;
+                }
+            }
+            else {
+                $text[$n] = $this->_convert_line($text[$n], $flowed || $this->config['wrap']);
+                $q = 0;
+
+                if ($quote_level > 0) {
+                    $text[$n] = $replacer->get_replacement($replacer->add(
+                        str_repeat('</blockquote>', $quote_level))) . $text[$n];
+                }
+            }
+
+            $quote_level = $q;
+        }
+
+        if ($quote_level > 0) {
+            $text[$n] = $replacer->get_replacement($replacer->add(
+                str_repeat('</blockquote>', $quote_level))) . $text[$n];
+        }
+
+        $text = join("\n", $text);
+
+        // colorize signature (up to <sig_max_lines> lines)
+        $len = strlen($text);
+        $sig_max_lines = rcube::get_instance()->config->get('sig_max_lines', 15);
+
+        while (($sp = strrpos($text, "-- \n", $sp ? -$len+$sp-1 : 0)) !== false) {
+            if ($sp == 0 || $text[$sp-1] == "\n") {
+                // do not touch blocks with more that X lines
+                if (substr_count($text, "\n", $sp) < $sig_max_lines) {
+                    $text = substr($text, 0, max(0, $sp))
+                        .'<span class="sig">'.substr($text, $sp).'</span>';
+                }
+
+                break;
+            }
+        }
+
+        // insert url/mailto links and citation tags
+        $text = $replacer->resolve($text);
+
+        // replace \n before </blockquote>
+        $text = str_replace("\n</blockquote>", "</blockquote>", $text);
+
+        // replace line breaks
+        $text = str_replace("\n", $this->config['break'], $text);
+
+        return $this->config['begin'] . $text . $this->config['end'];
+    }
+
+    /**
+     * Converts spaces in line of text
+     */
+    protected function _convert_line($text, $is_flowed)
+    {
+        static $table;
+
+        if (empty($table)) {
+            $table = get_html_translation_table(HTML_SPECIALCHARS);
+            unset($table['?']);
+        }
+
+        // skip signature separator
+        if ($text == '-- ') {
+            return $text;
+        }
+
+        // replace HTML special characters
+        $text = strtr($text, $table);
+
+        $nbsp = $this->config['space'];
+
+        // replace some whitespace characters
+        $text = str_replace(array("\r", "\t"), array('', '    '), $text);
+
+        // replace spaces with non-breaking spaces
+        if ($is_flowed) {
+            $text = preg_replace_callback('/(^|[^ ])( +)/', function($matches) {
+                    if (!strlen($matches[2])) {
+                        return str_repeat($nbsp, strlen($matches[2]));
+                    }
+                    else {
+                        return $matches[1] . ' ' . str_repeat($nbsp, strlen($matches[2])-1);
+                    }
+                }, $text);
+        }
+        else {
+            $text = str_replace(' ', $nbsp, $text);
+        }
+
+        return $text;
+    }
+}
diff --git a/program/steps/mail/compose.inc b/program/steps/mail/compose.inc
index 2b717d673..0ceb85db2 100644
--- a/program/steps/mail/compose.inc
+++ b/program/steps/mail/compose.inc
@@ -624,7 +624,8 @@ function rcmail_compose_header_from($attrib)
                 }
 
                 if (!$sql_arr['html_signature']) {
-                    $html = "<pre>" . $html . "</pre>";
+                    $t2h  = new rcube_text2html($sql_arr['signature'], false);
+                    $html = $t2h->get_html();
                 }
 
                 $a_signatures[$identity_id]['text'] = $text;
@@ -826,15 +827,8 @@ function rcmail_compose_part_body($part, $isHtml = false)
                 }
             }
 
-            if ($part->ctype_parameters['format'] == 'flowed') {
-                $body = rcube_mime::unfold_flowed($body);
-            }
-
             // add HTML formatting
-            $body = rcmail_plain_body($body);
-            if ($body) {
-                $body = '<pre>' . $body . '</pre>';
-            }
+            $body = rcmail_plain_body($body, $part->ctype_parameters['format'] == 'flowed');
         }
     }
     else {
diff --git a/program/steps/mail/func.inc b/program/steps/mail/func.inc
index 7270cf95a..ac0d7fc5f 100644
--- a/program/steps/mail/func.inc
+++ b/program/steps/mail/func.inc
@@ -854,95 +854,28 @@ function rcmail_print_body($part, $p = array())
 
     // plaintext postprocessing
     if ($part->ctype_secondary == 'plain') {
-        if ($part->ctype_secondary == 'plain' && $part->ctype_parameters['format'] == 'flowed') {
-            $body = rcube_mime::unfold_flowed($body);
-        }
-
-        $body = rcmail_plain_body($body);
+        $body = rcmail_plain_body($body, $part->ctype_parameters['format'] == 'flowed');
     }
 
     // allow post-processing of the message body
     $data = $RCMAIL->plugins->exec_hook('message_part_after',
         array('type' => $part->ctype_secondary, 'body' => $body, 'id' => $part->mime_id) + $data);
 
-    return $data['type'] == 'html' ? $data['body'] : html::tag('pre', array(), $data['body']);
+    return $data['body'];
 }
 
 /**
  * Handle links and citation marks in plain text message
  *
  * @param string  Plain text string
+ * @param boolean Set to True if the source text is in format=flowed
  *
  * @return string Formatted HTML string
  */
-function rcmail_plain_body($body)
+function rcmail_plain_body($body, $flowed = false)
 {
-    global $RCMAIL;
-
-    // make links and email-addresses clickable
-    $attribs  = array('link_attribs' => array('rel' => 'noreferrer', 'target' => '_blank'));
-    $replacer = new rcmail_string_replacer($attribs);
-
-    // search for patterns like links and e-mail addresses and replace with tokens
-    $body = $replacer->replace($body);
-
-    // split body into single lines
-    $body        = preg_split('/\r?\n/', $body);
-    $quote_level = 0;
-    $last        = -1;
-
-    // find/mark quoted lines...
-    for ($n=0, $cnt=count($body); $n < $cnt; $n++) {
-        if ($body[$n][0] == '>' && preg_match('/^(>+ {0,1})+/', $body[$n], $regs)) {
-            $q        = substr_count($regs[0], '>');
-            $body[$n] = substr($body[$n], strlen($regs[0]));
-
-            if ($q > $quote_level) {
-                $body[$n] = $replacer->get_replacement($replacer->add(
-                    str_repeat('<blockquote>', $q - $quote_level))) . $body[$n];
-                $last = $n;
-            }
-            else if ($q < $quote_level) {
-                $body[$n] = $replacer->get_replacement($replacer->add(
-                    str_repeat('</blockquote>', $quote_level - $q))) . $body[$n];
-                $last = $n;
-            }
-        }
-        else {
-            $q = 0;
-            if ($quote_level > 0)
-                $body[$n] = $replacer->get_replacement($replacer->add(
-                    str_repeat('</blockquote>', $quote_level))) . $body[$n];
-        }
-
-        $quote_level = $q;
-    }
-
-    $body = join("\n", $body);
-
-    // quote plain text (don't use rcube::Q() here, to display entities "as is")
-    $table = get_html_translation_table(HTML_SPECIALCHARS);
-    unset($table['?']);
-    $body = strtr($body, $table);
-
-    // colorize signature (up to <sig_max_lines> lines)
-    $len = strlen($body);
-    $sig_max_lines = $RCMAIL->config->get('sig_max_lines', 15);
-
-    while (($sp = strrpos($body, "-- \n", $sp ? -$len+$sp-1 : 0)) !== false) {
-        if ($sp == 0 || $body[$sp-1] == "\n") {
-            // do not touch blocks with more that X lines
-            if (substr_count($body, "\n", $sp) < $sig_max_lines) {
-                $body = substr($body, 0, max(0, $sp))
-                    . '<span class="sig">'.substr($body, $sp).'</span>';
-            }
-
-            break;
-        }
-    }
-
-    // insert url/mailto links and citation tags
-    $body = $replacer->resolve($body);
+    $text2html = new rcube_text2html($body, false, array('flowed' => $flowed));
+    $body      = $text2html->get_html();
 
     return $body;
 }
@@ -1272,8 +1205,8 @@ function rcmail_message_body($attrib)
             $plugin = $RCMAIL->plugins->exec_hook('message_body_prefix',
                 array('part' => $MESSAGE, 'prefix' => ''));
 
-            $out .= html::div('message-part', $plugin['prefix'] . html::tag('pre', array(),
-                rcmail_plain_body(rcube::Q($MESSAGE->body, 'strict', false))));
+            $out .= html::div('message-part',
+                $plugin['prefix'] . rcmail_plain_body($MESSAGE->body));
         }
     }
 
diff --git a/program/steps/mail/sendmail.inc b/program/steps/mail/sendmail.inc
index 04ba94f5e..baecbd118 100644
--- a/program/steps/mail/sendmail.inc
+++ b/program/steps/mail/sendmail.inc
@@ -280,13 +280,23 @@ if ($isHtml) {
 
 if (!$savedraft) {
     if ($isHtml) {
-        // remove signature's div ID
-        $message_body = preg_replace('/\s*id="_rc_sig"/', '', $message_body);
-
-        // add inline css for blockquotes
-        $bstyle       = 'padding-left:5px; border-left:#1010ff 2px solid; margin-left:5px';
-        $message_body = preg_replace('/<blockquote>/',
-            '<blockquote type="cite" style="'.$bstyle.'">', $message_body);
+        $b_style   = 'padding: 0 0.4em; border-left: #1010ff 2px solid; margin: 0';
+        $pre_style = 'margin: 0; padding: 0; font-family: monospace';
+
+        $message_body = preg_replace(
+            array(
+                // remove signature's div ID
+                '/\s*id="_rc_sig"/',
+                // add inline css for blockquotes and container
+                '/<blockquote>/',
+                '/<div class="pre">/'
+            ),
+            array(
+                '',
+                '<blockquote type="cite" style="'.$b_style.'">',
+                '<div class="pre" style="'.$pre_style.'">'
+            ),
+            $message_body);
     }
 
     // Check spelling before send
@@ -912,7 +922,8 @@ function rcmail_generic_message_footer($isHtml)
         if (!preg_match('/\.(php|ini|conf)$/', $file) && strpos($file, '/etc/') === false) {
             $footer = file_get_contents($file);
             if ($isHtml && !$html_footer) {
-                $footer = '<pre>' . $footer . '</pre>';
+                $t2h    = new rcube_text2html($footer, false);
+                $footer = $t2h->get_html();
             }
             return $footer;
         }
diff --git a/program/steps/utils/text2html.inc b/program/steps/utils/text2html.inc
new file mode 100644
index 000000000..167243694
--- /dev/null
+++ b/program/steps/utils/text2html.inc
@@ -0,0 +1,28 @@
+<?php
+
+/*
+ +-----------------------------------------------------------------------+
+ | program/steps/utils/text2html.inc                                     |
+ |                                                                       |
+ | This file is part of the Roundcube Webmail client                     |
+ | Copyright (C) 2005-2014, The Roundcube Dev Team                       |
+ |                                                                       |
+ | Licensed under the GNU General Public License version 3 or            |
+ | any later version with exceptions for skins & plugins.                |
+ | See the README file for a full license statement.                     |
+ |                                                                       |
+ | PURPOSE:                                                              |
+ |   Convert plain text to HTML                                          |
+ |                                                                       |
+ +-----------------------------------------------------------------------+
+ | Author: Thomas Bruederli <roundcube@gmail.com>                        |
+ +-----------------------------------------------------------------------+
+*/
+
+$text = stream_get_contents(fopen('php://input', 'r'));
+
+$converter = new rcube_text2html($text, false, array('wrap' => true));
+
+header('Content-Type: text/html; charset=' . RCUBE_CHARSET);
+print $converter->get_html();
+exit;