From 638ebf69c4bdc3717d8ae535ec3b1f4b753f5856 Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Fri, 9 May 2014 15:17:32 +0200 Subject: Fix/remove tests related to data-* attributes handling --- tests/Framework/Html.php | 9 --------- 1 file changed, 9 deletions(-) (limited to 'tests/Framework') diff --git a/tests/Framework/Html.php b/tests/Framework/Html.php index d9466e601..259d73e1a 100644 --- a/tests/Framework/Html.php +++ b/tests/Framework/Html.php @@ -54,15 +54,6 @@ class Framework_Html extends PHPUnit_Framework_TestCase array( array('data-test' => 'test'), null, ' data-test="test"', ), - array( - array('data-test' => 'test'), array('other'), '', - ), - array( - array('data-test' => 'test'), array('data-test'), ' data-test="test"', - ), - array( - array('data-test' => 'test'), array('data-*'), ' data-test="test"', - ), ); } -- cgit v1.2.3 From eda92ed4c0d2735144df8fa2136584de69634bdb Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Sun, 11 May 2014 11:03:45 +0200 Subject: Improved display of plain text messages and text to HTML conversion (#1488937) Now instead of
 we use 
styled with monospace font. We replace whitespace characters with non-breaking spaces where needed. I.e. plain text is always unwrappable, until it uses format=flowed, in such a case only flowed paragraphs are wrappable. Also conversion of text to HTML in compose editor was modified in the same way. --- plugins/hide_blockquote/hide_blockquote.js | 2 +- program/js/app.js | 70 +++++--- program/js/editor.js | 2 +- program/lib/Roundcube/rcube_mime.php | 22 ++- program/lib/Roundcube/rcube_text2html.php | 277 +++++++++++++++++++++++++++++ program/steps/mail/compose.inc | 12 +- program/steps/mail/func.inc | 83 +-------- program/steps/mail/sendmail.inc | 27 ++- program/steps/utils/text2html.inc | 28 +++ skins/classic/editor_content.css | 13 +- skins/classic/mail.css | 11 +- skins/larry/editor_content.css | 13 +- skins/larry/mail.css | 11 +- tests/Framework/Text2Html.php | 81 +++++++++ tests/phpunit.xml | 1 + 15 files changed, 497 insertions(+), 156 deletions(-) create mode 100644 program/lib/Roundcube/rcube_text2html.php create mode 100644 program/steps/utils/text2html.inc create mode 100644 tests/Framework/Text2Html.php (limited to 'tests/Framework') diff --git a/plugins/hide_blockquote/hide_blockquote.js b/plugins/hide_blockquote/hide_blockquote.js index 2d28076a1..964cc07a3 100644 --- a/plugins/hide_blockquote/hide_blockquote.js +++ b/plugins/hide_blockquote/hide_blockquote.js @@ -25,7 +25,7 @@ function hide_blockquote() if (limit <= 0) return; - $('pre > blockquote', $('#messagebody')).each(function() { + $('div.message-part div.pre > blockquote', $('#messagebody')).each(function() { var div, link, q = $(this), text = $.trim(q.text()), res = text.split(/\n/); diff --git a/program/js/app.js b/program/js/app.js index 042ebb724..46d748062 100644 --- a/program/js/app.js +++ b/program/js/app.js @@ -3439,17 +3439,23 @@ function rcube_webmail() { this.stop_spellchecking(); - if (props.mode == 'html') { - this.plain2html($('#'+props.id).val(), props.id); - tinyMCE.execCommand('mceAddControl', false, props.id); + var input = $('#' + props.id); - if (this.env.default_font) - setTimeout(function() { - $(tinyMCE.get(props.id).getBody()).css('font-family', rcmail.env.default_font); - }, 500); - } - else if (this.html2plain(tinyMCE.get(props.id).getContent(), props.id)) - tinyMCE.execCommand('mceRemoveControl', false, props.id); + if (props.mode == 'html') + this.plain2html(input.val(), function(data) { + input.val(data); + tinyMCE.execCommand('mceAddControl', false, props.id); + + if (ref.env.default_font) + setTimeout(function() { + $(tinyMCE.get(props.id).getBody()).css('font-family', ref.env.default_font); + }, 500); + }); + else + this.html2plain(tinyMCE.get(props.id).getContent(), function(data) { + tinyMCE.execCommand('mceRemoveControl', false, props.id); + input.val(data); + }); return true; }; @@ -6809,41 +6815,51 @@ function rcube_webmail() /********* html to text conversion functions *********/ /********************************************************/ - this.html2plain = function(htmlText, id) + this.html2plain = function(html, func) + { + return this.format_converter(html, 'html', func); + }; + + this.plain2html = function(plain, func) + { + return this.format_converter(plain, 'plain', func); + }; + + this.format_converter = function(text, format, func) { // warn the user (if converted content is not empty) - if (!htmlText || !(htmlText.replace(/<[^>]+>| |\s/g, '')).length) { + if (!text + || (format == 'html' && !(text.replace(/<[^>]+>| |\xC2\xA0|\s/g, '')).length) + || (format != 'html' && !(text.replace(/\xC2\xA0|\s/g, '')).length) + ) { // without setTimeout() here, textarea is filled with initial (onload) content - setTimeout(function() { $('#'+id).val(''); }, 50); + setTimeout(function() { if (func) func(''); }, 50); return true; } - if (!confirm(this.get_label('editorwarning'))) + var confirmed = this.env.editor_warned || confirm(this.get_label('editorwarning')); + + this.env.editor_warned = true; + + if (!confirmed) return false; - var url = '?_task=utils&_action=html2text', + var url = '?_task=utils&_action=' + (format == 'html' ? 'html2text' : 'text2html'), lock = this.set_busy(true, 'converting'); this.log('HTTP POST: ' + url); - $.ajax({ type: 'POST', url: url, data: htmlText, contentType: 'application/octet-stream', + $.ajax({ type: 'POST', url: url, data: text, contentType: 'application/octet-stream', error: function(o, status, err) { ref.http_error(o, status, err, lock); }, - success: function(data) { ref.set_busy(false, null, lock); $('#'+id).val(data); ref.log(data); } + success: function(data) { + ref.set_busy(false, null, lock); + if (func) func(data); + } }); return true; }; - this.plain2html = function(plain, id) - { - var lock = this.set_busy(true, 'converting'); - - plain = plain.replace(/&/g, '&').replace(//g, '>'); - $('#'+id).val(plain ? '
'+plain+'
' : ''); - - this.set_busy(false, null, lock); - }; - /********************************************************/ /********* remote request methods *********/ diff --git a/program/js/editor.js b/program/js/editor.js index af877d7b5..d23d2b26f 100644 --- a/program/js/editor.js +++ b/program/js/editor.js @@ -37,7 +37,7 @@ function rcmail_editor_init(config) apply_source_formatting: true, theme: 'advanced', language: config.lang, - content_css: config.skin_path + '/editor_content.css', + content_css: config.skin_path + '/editor_content.css?v2', theme_advanced_toolbar_location: 'top', theme_advanced_toolbar_align: 'left', theme_advanced_buttons3: '', diff --git a/program/lib/Roundcube/rcube_mime.php b/program/lib/Roundcube/rcube_mime.php index 091b2fae8..370d5a8d5 100644 --- a/program/lib/Roundcube/rcube_mime.php +++ b/program/lib/Roundcube/rcube_mime.php @@ -480,15 +480,17 @@ class rcube_mime /** * Interpret a format=flowed message body according to RFC 2646 * - * @param string $text Raw body formatted as flowed text + * @param string $text Raw body formatted as flowed text + * @param string $mark Mark each flowed line with specified character * * @return string Interpreted text with unwrapped lines and stuffed space removed */ - public static function unfold_flowed($text) + public static function unfold_flowed($text, $mark = null) { $text = preg_split('/\r?\n/', $text); $last = -1; $q_level = 0; + $marks = array(); foreach ($text as $idx => $line) { if (preg_match('/^(>+)/', $line, $m)) { @@ -508,6 +510,10 @@ class rcube_mime ) { $text[$last] .= $line; unset($text[$idx]); + + if ($mark) { + $marks[$last] = true; + } } else { $last = $idx; @@ -520,7 +526,7 @@ class rcube_mime } else { // remove space-stuffing - $line = preg_replace('/^\s/', '', $line); + $line = preg_replace('/^ /', '', $line); if (isset($text[$last]) && $line && $text[$last] != '-- ' @@ -528,6 +534,10 @@ class rcube_mime ) { $text[$last] .= $line; unset($text[$idx]); + + if ($mark) { + $marks[$last] = true; + } } else { $text[$idx] = $line; @@ -538,6 +548,12 @@ class rcube_mime $q_level = $q; } + if (!empty($marks)) { + foreach (array_keys($marks) as $mk) { + $text[$mk] = $mark . $text[$mk]; + } + } + return implode("\r\n", $text); } diff --git a/program/lib/Roundcube/rcube_text2html.php b/program/lib/Roundcube/rcube_text2html.php new file mode 100644 index 000000000..5da771ac8 --- /dev/null +++ b/program/lib/Roundcube/rcube_text2html.php @@ -0,0 +1,277 @@ + | + +-----------------------------------------------------------------------+ + */ + +/** + * Converts plain text to HTML + * + * @package Framework + * @subpackage Utils + */ +class rcube_text2html +{ + /** + * Contains the HTML content after conversion. + * + * @var string $html + */ + protected $html; + + /** + * Contains the plain text. + * + * @var string $text + */ + protected $text; + + /** + * Configuration + * + * @var array $config + */ + protected $config = array( + // non-breaking space + 'space' => "\xC2\xA0", + // enables format=flowed parser + 'flowed' => false, + // enables wrapping for non-flowed text + 'wrap' => false, + // line-break tag + 'break' => "
\n", + // prefix and suffix (wrapper element) + 'begin' => '
', + 'end' => '', + // enables links replacement + 'links' => true, + ); + + + /** + * Constructor. + * + * If the plain text source string (or file) is supplied, the class + * will instantiate with that source propagated, all that has + * to be done it to call get_html(). + * + * @param string $source Plain text + * @param boolean $from_file Indicates $source is a file to pull content from + * @param array $config Class configuration + */ + function __construct($source = '', $from_file = false, $config = array()) + { + if (!empty($source)) { + $this->set_text($source, $from_file); + } + + if (!empty($config) && is_array($config)) { + $this->config = array_merge($this->config, $config); + } + } + + /** + * Loads source text into memory, either from $source string or a file. + * + * @param string $source Plain text + * @param boolean $from_file Indicates $source is a file to pull content from + */ + function set_text($source, $from_file = false) + { + if ($from_file && file_exists($source)) { + $this->text = file_get_contents($source); + } + else { + $this->text = $source; + } + + $this->_converted = false; + } + + /** + * Returns the HTML content. + * + * @return string HTML content + */ + function get_html() + { + if (!$this->_converted) { + $this->_convert(); + } + + return $this->html; + } + + /** + * Prints the HTML. + */ + function print_html() + { + print $this->get_html(); + } + + /** + * Workhorse function that does actual conversion (calls _converter() method). + */ + protected function _convert() + { + $text = stripslashes($this->text); + + // Convert TXT to HTML + $this->html = $this->_converter($text); + $this->_converted = true; + } + + /** + * Workhorse function that does actual conversion. + * + * @param string Plain text + */ + protected function _converter($text) + { + // make links and email-addresses clickable + $attribs = array('link_attribs' => array('rel' => 'noreferrer', 'target' => '_blank')); + $replacer = new rcmail_string_replacer($attribs); + + if ($this->config['flowed']) { + $flowed_char = 0x01; + $text = rcube_mime::unfold_flowed($text, chr($flowed_char)); + } + + // search for patterns like links and e-mail addresses and replace with tokens + if ($this->config['links']) { + $text = $replacer->replace($text); + } + + // split body into single lines + $text = preg_split('/\r?\n/', $text); + $quote_level = 0; + $last = -1; + + // find/mark quoted lines... + for ($n=0, $cnt=count($text); $n < $cnt; $n++) { + $flowed = false; + if ($this->config['flowed'] && ord($text[0]) == $flowed_char) { + $flowed = true; + $text[$n] = substr($text[$n], 1); + } + + if ($text[$n][0] == '>' && preg_match('/^(>+ {0,1})+/', $text[$n], $regs)) { + $q = substr_count($regs[0], '>'); + $text[$n] = substr($text[$n], strlen($regs[0])); + $text[$n] = $this->_convert_line($text[$n], $flowed || $this->config['wrap']); + + if ($q > $quote_level) { + $text[$n] = $replacer->get_replacement($replacer->add( + str_repeat('
', $q - $quote_level))) . $text[$n]; + $last = $n; + } + else if ($q < $quote_level) { + $text[$n] = $replacer->get_replacement($replacer->add( + str_repeat('
', $quote_level - $q))) . $text[$n]; + $last = $n; + } + } + else { + $text[$n] = $this->_convert_line($text[$n], $flowed || $this->config['wrap']); + $q = 0; + + if ($quote_level > 0) { + $text[$n] = $replacer->get_replacement($replacer->add( + str_repeat('', $quote_level))) . $text[$n]; + } + } + + $quote_level = $q; + } + + if ($quote_level > 0) { + $text[$n] = $replacer->get_replacement($replacer->add( + str_repeat('', $quote_level))) . $text[$n]; + } + + $text = join("\n", $text); + + // colorize signature (up to lines) + $len = strlen($text); + $sig_max_lines = rcube::get_instance()->config->get('sig_max_lines', 15); + + while (($sp = strrpos($text, "-- \n", $sp ? -$len+$sp-1 : 0)) !== false) { + if ($sp == 0 || $text[$sp-1] == "\n") { + // do not touch blocks with more that X lines + if (substr_count($text, "\n", $sp) < $sig_max_lines) { + $text = substr($text, 0, max(0, $sp)) + .''.substr($text, $sp).''; + } + + break; + } + } + + // insert url/mailto links and citation tags + $text = $replacer->resolve($text); + + // replace \n before + $text = str_replace("\n", "", $text); + + // replace line breaks + $text = str_replace("\n", $this->config['break'], $text); + + return $this->config['begin'] . $text . $this->config['end']; + } + + /** + * Converts spaces in line of text + */ + protected function _convert_line($text, $is_flowed) + { + static $table; + + if (empty($table)) { + $table = get_html_translation_table(HTML_SPECIALCHARS); + unset($table['?']); + } + + // skip signature separator + if ($text == '-- ') { + return $text; + } + + // replace HTML special characters + $text = strtr($text, $table); + + $nbsp = $this->config['space']; + + // replace some whitespace characters + $text = str_replace(array("\r", "\t"), array('', ' '), $text); + + // replace spaces with non-breaking spaces + if ($is_flowed) { + $text = preg_replace_callback('/(^|[^ ])( +)/', function($matches) { + if (!strlen($matches[2])) { + return str_repeat($nbsp, strlen($matches[2])); + } + else { + return $matches[1] . ' ' . str_repeat($nbsp, strlen($matches[2])-1); + } + }, $text); + } + else { + $text = str_replace(' ', $nbsp, $text); + } + + return $text; + } +} diff --git a/program/steps/mail/compose.inc b/program/steps/mail/compose.inc index 2b717d673..0ceb85db2 100644 --- a/program/steps/mail/compose.inc +++ b/program/steps/mail/compose.inc @@ -624,7 +624,8 @@ function rcmail_compose_header_from($attrib) } if (!$sql_arr['html_signature']) { - $html = "
" . $html . "
"; + $t2h = new rcube_text2html($sql_arr['signature'], false); + $html = $t2h->get_html(); } $a_signatures[$identity_id]['text'] = $text; @@ -826,15 +827,8 @@ function rcmail_compose_part_body($part, $isHtml = false) } } - if ($part->ctype_parameters['format'] == 'flowed') { - $body = rcube_mime::unfold_flowed($body); - } - // add HTML formatting - $body = rcmail_plain_body($body); - if ($body) { - $body = '
' . $body . '
'; - } + $body = rcmail_plain_body($body, $part->ctype_parameters['format'] == 'flowed'); } } else { diff --git a/program/steps/mail/func.inc b/program/steps/mail/func.inc index 7270cf95a..ac0d7fc5f 100644 --- a/program/steps/mail/func.inc +++ b/program/steps/mail/func.inc @@ -854,95 +854,28 @@ function rcmail_print_body($part, $p = array()) // plaintext postprocessing if ($part->ctype_secondary == 'plain') { - if ($part->ctype_secondary == 'plain' && $part->ctype_parameters['format'] == 'flowed') { - $body = rcube_mime::unfold_flowed($body); - } - - $body = rcmail_plain_body($body); + $body = rcmail_plain_body($body, $part->ctype_parameters['format'] == 'flowed'); } // allow post-processing of the message body $data = $RCMAIL->plugins->exec_hook('message_part_after', array('type' => $part->ctype_secondary, 'body' => $body, 'id' => $part->mime_id) + $data); - return $data['type'] == 'html' ? $data['body'] : html::tag('pre', array(), $data['body']); + return $data['body']; } /** * Handle links and citation marks in plain text message * * @param string Plain text string + * @param boolean Set to True if the source text is in format=flowed * * @return string Formatted HTML string */ -function rcmail_plain_body($body) +function rcmail_plain_body($body, $flowed = false) { - global $RCMAIL; - - // make links and email-addresses clickable - $attribs = array('link_attribs' => array('rel' => 'noreferrer', 'target' => '_blank')); - $replacer = new rcmail_string_replacer($attribs); - - // search for patterns like links and e-mail addresses and replace with tokens - $body = $replacer->replace($body); - - // split body into single lines - $body = preg_split('/\r?\n/', $body); - $quote_level = 0; - $last = -1; - - // find/mark quoted lines... - for ($n=0, $cnt=count($body); $n < $cnt; $n++) { - if ($body[$n][0] == '>' && preg_match('/^(>+ {0,1})+/', $body[$n], $regs)) { - $q = substr_count($regs[0], '>'); - $body[$n] = substr($body[$n], strlen($regs[0])); - - if ($q > $quote_level) { - $body[$n] = $replacer->get_replacement($replacer->add( - str_repeat('
', $q - $quote_level))) . $body[$n]; - $last = $n; - } - else if ($q < $quote_level) { - $body[$n] = $replacer->get_replacement($replacer->add( - str_repeat('
', $quote_level - $q))) . $body[$n]; - $last = $n; - } - } - else { - $q = 0; - if ($quote_level > 0) - $body[$n] = $replacer->get_replacement($replacer->add( - str_repeat('', $quote_level))) . $body[$n]; - } - - $quote_level = $q; - } - - $body = join("\n", $body); - - // quote plain text (don't use rcube::Q() here, to display entities "as is") - $table = get_html_translation_table(HTML_SPECIALCHARS); - unset($table['?']); - $body = strtr($body, $table); - - // colorize signature (up to lines) - $len = strlen($body); - $sig_max_lines = $RCMAIL->config->get('sig_max_lines', 15); - - while (($sp = strrpos($body, "-- \n", $sp ? -$len+$sp-1 : 0)) !== false) { - if ($sp == 0 || $body[$sp-1] == "\n") { - // do not touch blocks with more that X lines - if (substr_count($body, "\n", $sp) < $sig_max_lines) { - $body = substr($body, 0, max(0, $sp)) - . ''.substr($body, $sp).''; - } - - break; - } - } - - // insert url/mailto links and citation tags - $body = $replacer->resolve($body); + $text2html = new rcube_text2html($body, false, array('flowed' => $flowed)); + $body = $text2html->get_html(); return $body; } @@ -1272,8 +1205,8 @@ function rcmail_message_body($attrib) $plugin = $RCMAIL->plugins->exec_hook('message_body_prefix', array('part' => $MESSAGE, 'prefix' => '')); - $out .= html::div('message-part', $plugin['prefix'] . html::tag('pre', array(), - rcmail_plain_body(rcube::Q($MESSAGE->body, 'strict', false)))); + $out .= html::div('message-part', + $plugin['prefix'] . rcmail_plain_body($MESSAGE->body)); } } diff --git a/program/steps/mail/sendmail.inc b/program/steps/mail/sendmail.inc index 04ba94f5e..baecbd118 100644 --- a/program/steps/mail/sendmail.inc +++ b/program/steps/mail/sendmail.inc @@ -280,13 +280,23 @@ if ($isHtml) { if (!$savedraft) { if ($isHtml) { - // remove signature's div ID - $message_body = preg_replace('/\s*id="_rc_sig"/', '', $message_body); - - // add inline css for blockquotes - $bstyle = 'padding-left:5px; border-left:#1010ff 2px solid; margin-left:5px'; - $message_body = preg_replace('/
/', - '
', $message_body); + $b_style = 'padding: 0 0.4em; border-left: #1010ff 2px solid; margin: 0'; + $pre_style = 'margin: 0; padding: 0; font-family: monospace'; + + $message_body = preg_replace( + array( + // remove signature's div ID + '/\s*id="_rc_sig"/', + // add inline css for blockquotes and container + '/
/', + '/
/' + ), + array( + '', + '
', + '
' + ), + $message_body); } // Check spelling before send @@ -912,7 +922,8 @@ function rcmail_generic_message_footer($isHtml) if (!preg_match('/\.(php|ini|conf)$/', $file) && strpos($file, '/etc/') === false) { $footer = file_get_contents($file); if ($isHtml && !$html_footer) { - $footer = '
' . $footer . '
'; + $t2h = new rcube_text2html($footer, false); + $footer = $t2h->get_html(); } return $footer; } diff --git a/program/steps/utils/text2html.inc b/program/steps/utils/text2html.inc new file mode 100644 index 000000000..167243694 --- /dev/null +++ b/program/steps/utils/text2html.inc @@ -0,0 +1,28 @@ + | + +-----------------------------------------------------------------------+ +*/ + +$text = stream_get_contents(fopen('php://input', 'r')); + +$converter = new rcube_text2html($text, false, array('wrap' => true)); + +header('Content-Type: text/html; charset=' . RCUBE_CHARSET); +print $converter->get_html(); +exit; diff --git a/skins/classic/editor_content.css b/skins/classic/editor_content.css index aabed07b5..67480ab77 100644 --- a/skins/classic/editor_content.css +++ b/skins/classic/editor_content.css @@ -12,20 +12,15 @@ body { margin-top: 2px; } -pre -{ +div.pre { margin: 0; padding: 0; - white-space: -moz-pre-wrap !important; - white-space: pre-wrap !important; - white-space: pre; - word-wrap: break-word; /* IE (and Safari) */ + font-family: monospace; } blockquote { - padding-left: 5px; border-left: #1010ff 2px solid; - margin-left: 5px; - width: 100%; + margin: 0; + padding: 0 0.4em; } diff --git a/skins/classic/mail.css b/skins/classic/mail.css index fc066e023..47faa29af 100644 --- a/skins/classic/mail.css +++ b/skins/classic/mail.css @@ -1306,17 +1306,12 @@ div.message-htmlpart a color: #0000CC; } -div.message-part pre, -div.message-htmlpart pre, div.message-part div.pre { margin: 0px; padding: 0px; font-family: monospace; font-size: 12px; - white-space: -moz-pre-wrap !important; - white-space: pre-wrap !important; - white-space: pre; } div.message-part span.sig @@ -1330,8 +1325,10 @@ div.message-part blockquote border-left: 2px solid blue; border-right: 2px solid blue; background-color: #F6F6F6; - margin: 2px 0px; - padding: 1px 8px 1px 10px; + margin: 0; + padding: 0 0.4em; + overflow: hidden; + text-overflow: ellipsis; } div.message-part blockquote blockquote diff --git a/skins/larry/editor_content.css b/skins/larry/editor_content.css index aabed07b5..67480ab77 100644 --- a/skins/larry/editor_content.css +++ b/skins/larry/editor_content.css @@ -12,20 +12,15 @@ body { margin-top: 2px; } -pre -{ +div.pre { margin: 0; padding: 0; - white-space: -moz-pre-wrap !important; - white-space: pre-wrap !important; - white-space: pre; - word-wrap: break-word; /* IE (and Safari) */ + font-family: monospace; } blockquote { - padding-left: 5px; border-left: #1010ff 2px solid; - margin-left: 5px; - width: 100%; + margin: 0; + padding: 0 0.4em; } diff --git a/skins/larry/mail.css b/skins/larry/mail.css index 3ec14568f..7afb14fba 100644 --- a/skins/larry/mail.css +++ b/skins/larry/mail.css @@ -1103,16 +1103,11 @@ div.message-partheaders { border-top: 0; } -div.message-part pre, -div.message-htmlpart pre, div.message-part div.pre { margin: 0; padding: 0; font-family: monospace; font-size: 12px; - white-space: -moz-pre-wrap !important; - white-space: pre-wrap !important; - white-space: pre; } div.message-part span.sig { @@ -1124,8 +1119,10 @@ div.message-part blockquote { border-left: 2px solid blue; border-right: 2px solid blue; background-color: #F6F6F6; - margin: 2px 0 2px 0; - padding: 1px 8px 1px 10px; + margin: 0; + padding: 0 0.4em; + overflow: hidden; + text-overflow: ellipsis; } div.message-part blockquote blockquote { diff --git a/tests/Framework/Text2Html.php b/tests/Framework/Text2Html.php new file mode 100644 index 000000000..91dabf2b7 --- /dev/null +++ b/tests/Framework/Text2Html.php @@ -0,0 +1,81 @@ + '', + 'end' => '', + 'break' => '
', + 'links' => false, + 'flowed' => false, + 'space' => '_', // replace UTF-8 non-breaking space for simpler testing + ); + + $data[] = array(" aaaa", "_aaaa", $options); + $data[] = array("aaaa aaaa", "aaaa_aaaa", $options); + $data[] = array("aaaa aaaa", "aaaa__aaaa", $options); + $data[] = array("aaaa aaaa", "aaaa___aaaa", $options); + $data[] = array("aaaa\taaaa", "aaaa____aaaa", $options); + $data[] = array("aaaa\naaaa", "aaaa
aaaa", $options); + $data[] = array("aaaa\n aaaa", "aaaa
_aaaa", $options); + $data[] = array("aaaa\n aaaa", "aaaa
__aaaa", $options); + $data[] = array("aaaa\n aaaa", "aaaa
___aaaa", $options); + $data[] = array("\taaaa", "____aaaa", $options); + $data[] = array("\naaaa", "
aaaa", $options); + $data[] = array("\n aaaa", "
_aaaa", $options); + $data[] = array("\n aaaa", "
__aaaa", $options); + $data[] = array("\n aaaa", "
___aaaa", $options); + $data[] = array("aaaa\n\nbbbb", "aaaa

bbbb", $options); + $data[] = array(">aaaa \n>aaaa", "
aaaa_
aaaa
", $options); + $data[] = array(">aaaa\n>aaaa", "
aaaa
aaaa
", $options); + $data[] = array(">aaaa \n>bbbb\ncccc dddd", "
aaaa_
bbbb
cccc_dddd", $options); + + $options['flowed'] = true; + + $data[] = array(" aaaa", "aaaa", $options); + $data[] = array("aaaa aaaa", "aaaa_aaaa", $options); + $data[] = array("aaaa aaaa", "aaaa__aaaa", $options); + $data[] = array("aaaa aaaa", "aaaa___aaaa", $options); + $data[] = array("aaaa\taaaa", "aaaa____aaaa", $options); + $data[] = array("aaaa\naaaa", "aaaa
aaaa", $options); + $data[] = array("aaaa\n aaaa", "aaaa
aaaa", $options); + $data[] = array("aaaa\n aaaa", "aaaa
_aaaa", $options); + $data[] = array("aaaa\n aaaa", "aaaa
__aaaa", $options); + $data[] = array("\taaaa", "____aaaa", $options); + $data[] = array("\naaaa", "
aaaa", $options); + $data[] = array("\n aaaa", "
aaaa", $options); + $data[] = array("\n aaaa", "
_aaaa", $options); + $data[] = array("\n aaaa", "
__aaaa", $options); + $data[] = array("aaaa\n\nbbbb", "aaaa

bbbb", $options); + $data[] = array(">aaaa \n>aaaa", "
aaaa aaaa
", $options); + $data[] = array(">aaaa\n>aaaa", "
aaaa
aaaa
", $options); + $data[] = array(">aaaa \n>bbbb\ncccc dddd", "
aaaa bbbb
cccc_dddd", $options); + + return $data; + } + + /** + * Test text to html conversion + * + * @dataProvider data_text2html + */ + function test_text2html($input, $output, $options) + { + $t2h = new rcube_text2html($input, false, $options); + + $html = $t2h->get_html(); + + $this->assertEquals($output, $html); + } +} diff --git a/tests/phpunit.xml b/tests/phpunit.xml index cee3434c1..f3df53521 100644 --- a/tests/phpunit.xml +++ b/tests/phpunit.xml @@ -40,6 +40,7 @@ Framework/SpellcheckPspell.php Framework/Spellchecker.php Framework/StringReplacer.php + Framework/Text2Html.php Framework/User.php Framework/Utils.php Framework/VCard.php -- cgit v1.2.3 From f0992426d9c5af5046c76a2da86183d0c3a40084 Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Tue, 13 May 2014 19:40:00 +0200 Subject: Bring back the old behaviour where text messages without format=flowed are auto-wrapped. Make it the default in text2html class. --- program/lib/Roundcube/rcube_text2html.php | 2 +- program/steps/mail/func.inc | 3 ++- tests/Framework/Text2Html.php | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) (limited to 'tests/Framework') diff --git a/program/lib/Roundcube/rcube_text2html.php b/program/lib/Roundcube/rcube_text2html.php index 60016fffd..8bcda301c 100644 --- a/program/lib/Roundcube/rcube_text2html.php +++ b/program/lib/Roundcube/rcube_text2html.php @@ -49,7 +49,7 @@ class rcube_text2html // enables format=flowed parser 'flowed' => false, // enables wrapping for non-flowed text - 'wrap' => false, + 'wrap' => true, // line-break tag 'break' => "
\n", // prefix and suffix (wrapper element) diff --git a/program/steps/mail/func.inc b/program/steps/mail/func.inc index ac0d7fc5f..a1d1a4163 100644 --- a/program/steps/mail/func.inc +++ b/program/steps/mail/func.inc @@ -874,7 +874,8 @@ function rcmail_print_body($part, $p = array()) */ function rcmail_plain_body($body, $flowed = false) { - $text2html = new rcube_text2html($body, false, array('flowed' => $flowed)); + $options = array('flowed' => $flowed, 'wrap' => !$flowed); + $text2html = new rcube_text2html($body, false, $options); $body = $text2html->get_html(); return $body; diff --git a/tests/Framework/Text2Html.php b/tests/Framework/Text2Html.php index 91dabf2b7..af2604d8e 100644 --- a/tests/Framework/Text2Html.php +++ b/tests/Framework/Text2Html.php @@ -19,6 +19,7 @@ class Framework_Text2Html extends PHPUnit_Framework_TestCase 'break' => '
', 'links' => false, 'flowed' => false, + 'wrap' => false, 'space' => '_', // replace UTF-8 non-breaking space for simpler testing ); -- cgit v1.2.3 From c0a5aa5f5ff38ac7b8a650b07c134b7b86deb27f Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Thu, 15 May 2014 10:41:35 +0200 Subject: Improved handling of new-lines in quoted paragraphs on text2html conversion --- program/lib/Roundcube/rcube_text2html.php | 65 +++++++++++++++++++++---------- skins/classic/mail.css | 2 +- skins/larry/mail.css | 2 +- tests/Framework/Text2Html.php | 11 ++++++ 4 files changed, 58 insertions(+), 22 deletions(-) (limited to 'tests/Framework') diff --git a/program/lib/Roundcube/rcube_text2html.php b/program/lib/Roundcube/rcube_text2html.php index 8bcda301c..363f1b21f 100644 --- a/program/lib/Roundcube/rcube_text2html.php +++ b/program/lib/Roundcube/rcube_text2html.php @@ -158,10 +158,10 @@ class rcube_text2html // split body into single lines $text = preg_split('/\r?\n/', $text); $quote_level = 0; - $last = -1; + $last = null; - // find/mark quoted lines... - for ($n=0, $cnt=count($text); $n < $cnt; $n++) { + // wrap quoted lines with
+ for ($n = 0, $cnt = count($text); $n < $cnt; $n++) { $flowed = false; if ($this->config['flowed'] && ord($text[$n][0]) == $flowed_char) { $flowed = true; @@ -172,43 +172,71 @@ class rcube_text2html $q = substr_count($regs[0], '>'); $text[$n] = substr($text[$n], strlen($regs[0])); $text[$n] = $this->_convert_line($text[$n], $flowed || $this->config['wrap']); + $_length = strlen(str_replace(' ', '', $text[$n])); if ($q > $quote_level) { - $text[$n] = $replacer->get_replacement($replacer->add( - str_repeat('
', $q - $quote_level))) . $text[$n]; - $last = $n; + if ($last !== null) { + $text[$last] .= (!$length ? "\n" : '') + . $replacer->get_replacement($replacer->add( + str_repeat('
', $q - $quote_level))) + . $text[$n]; + + unset($text[$n]); + } + else { + $text[$n] = $replacer->get_replacement($replacer->add( + str_repeat('
', $q - $quote_level))) . $text[$n]; + + $last = $n; + } } else if ($q < $quote_level) { - $text[$n] = $replacer->get_replacement($replacer->add( - str_repeat('
', $quote_level - $q))) . $text[$n]; + $text[$last] .= (!$length ? "\n" : '') + . $replacer->get_replacement($replacer->add( + str_repeat('
', $quote_level - $q))) + . $text[$n]; + + unset($text[$n]); + } + else { $last = $n; } } else { $text[$n] = $this->_convert_line($text[$n], $flowed || $this->config['wrap']); - $q = 0; + $q = 0; + $_length = strlen(str_replace(' ', '', $text[$n])); if ($quote_level > 0) { - $text[$n] = $replacer->get_replacement($replacer->add( - str_repeat('
', $quote_level))) . $text[$n]; + $text[$last] .= (!$length ? "\n" : '') + . $replacer->get_replacement($replacer->add( + str_repeat('
', $quote_level))) + . $text[$n]; + + unset($text[$n]); + } + else { + $last = $n; } } $quote_level = $q; + $length = $_length; } if ($quote_level > 0) { - $text[$n] = $replacer->get_replacement($replacer->add( - str_repeat('
', $quote_level))) . $text[$n]; + $text[$last] .= $replacer->get_replacement($replacer->add( + str_repeat('
', $quote_level))); } $text = join("\n", $text); // colorize signature (up to lines) - $len = strlen($text); + $len = strlen($text); + $sig_sep = "--" . $this->config['space'] . "\n"; $sig_max_lines = rcube::get_instance()->config->get('sig_max_lines', 15); - while (($sp = strrpos($text, "-- \n", $sp ? -$len+$sp-1 : 0)) !== false) { + while (($sp = strrpos($text, $sig_sep, $sp ? -$len+$sp-1 : 0)) !== false) { if ($sp == 0 || $text[$sp-1] == "\n") { // do not touch blocks with more that X lines if (substr_count($text, "\n", $sp) < $sig_max_lines) { @@ -223,9 +251,6 @@ class rcube_text2html // insert url/mailto links and citation tags $text = $replacer->resolve($text); - // replace \n before
- $text = str_replace("\n
", "", $text); - // replace line breaks $text = str_replace("\n", $this->config['break'], $text); @@ -246,7 +271,7 @@ class rcube_text2html // skip signature separator if ($text == '-- ') { - return $text; + return '--' . $this->config['space']; } // replace HTML special characters @@ -276,7 +301,7 @@ class rcube_text2html } else { // make the whole line non-breakable - $text = str_replace(array(' ', '-'), array($nbsp, '-⁠'), $text); + $text = str_replace(array(' ', '-', '/'), array($nbsp, '-⁠', '/⁠'), $text); } return $text; diff --git a/skins/classic/mail.css b/skins/classic/mail.css index 47faa29af..6409b6b9b 100644 --- a/skins/classic/mail.css +++ b/skins/classic/mail.css @@ -1325,7 +1325,7 @@ div.message-part blockquote border-left: 2px solid blue; border-right: 2px solid blue; background-color: #F6F6F6; - margin: 0; + margin: 2px 0; padding: 0 0.4em; overflow: hidden; text-overflow: ellipsis; diff --git a/skins/larry/mail.css b/skins/larry/mail.css index 7afb14fba..8306afd9f 100644 --- a/skins/larry/mail.css +++ b/skins/larry/mail.css @@ -1119,7 +1119,7 @@ div.message-part blockquote { border-left: 2px solid blue; border-right: 2px solid blue; background-color: #F6F6F6; - margin: 0; + margin: 2px 0; padding: 0 0.4em; overflow: hidden; text-overflow: ellipsis; diff --git a/tests/Framework/Text2Html.php b/tests/Framework/Text2Html.php index af2604d8e..8d091d5c9 100644 --- a/tests/Framework/Text2Html.php +++ b/tests/Framework/Text2Html.php @@ -41,6 +41,7 @@ class Framework_Text2Html extends PHPUnit_Framework_TestCase $data[] = array(">aaaa \n>aaaa", "
aaaa_
aaaa
", $options); $data[] = array(">aaaa\n>aaaa", "
aaaa
aaaa
", $options); $data[] = array(">aaaa \n>bbbb\ncccc dddd", "
aaaa_
bbbb
cccc_dddd", $options); + $data[] = array("aaaa-bbbb/cccc", "aaaa-⁠bbbb/⁠cccc", $options); $options['flowed'] = true; @@ -63,6 +64,16 @@ class Framework_Text2Html extends PHPUnit_Framework_TestCase $data[] = array(">aaaa\n>aaaa", "
aaaa
aaaa
", $options); $data[] = array(">aaaa \n>bbbb\ncccc dddd", "
aaaa bbbb
cccc_dddd", $options); + $options['flowed'] = false; + $options['wrap'] = true; + + $data[] = array(">>aaaa bbbb\n>>\n>>>\n>cccc\n\ndddd eeee", + "
aaaa bbbb


cccc

dddd eeee", $options); + $data[] = array("\n>>aaaa\n\ndddd", + "
aaaa

dddd", $options); + $data[] = array("aaaa\n>bbbb\n>cccc\n\ndddd\n>>test", + "aaaa
bbbb
cccc

dddd
test
", $options); + return $data; } -- cgit v1.2.3 From 001d337b3793857c5953bd814cd44b93a5f0d53d Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Fri, 16 May 2014 16:00:33 +0200 Subject: Fix handling unicode characters in links (#1489898) --- CHANGELOG | 1 + program/lib/Roundcube/rcube_string_replacer.php | 2 +- tests/Framework/StringReplacer.php | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) (limited to 'tests/Framework') diff --git a/CHANGELOG b/CHANGELOG index d38b97e64..d4cc0991e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -22,6 +22,7 @@ CHANGELOG Roundcube Webmail - Fix mbox files import - Fix unintentional draft autosave request if autosave is disabled (#1489882) - Fix malformed References: header in send/saved mail (#1489891) +- Fix handling unicode characters in links (#1489898) RELEASE 1.0.1 ------------- diff --git a/program/lib/Roundcube/rcube_string_replacer.php b/program/lib/Roundcube/rcube_string_replacer.php index 77b91d18b..ce61e5367 100644 --- a/program/lib/Roundcube/rcube_string_replacer.php +++ b/program/lib/Roundcube/rcube_string_replacer.php @@ -42,7 +42,7 @@ class rcube_string_replacer // Support unicode/punycode in top-level domain part $utf_domain = '[^?&@"\'\\/()<>\s\r\t\n]+\\.?([^\\x00-\\x2f\\x3b-\\x40\\x5b-\\x60\\x7b-\\x7f]{2,}|xn--[a-zA-Z0-9]{2,})'; $url1 = '.:;,'; - $url2 = 'a-zA-Z0-9%=#$@+?|!&\\/_~\\[\\]\\(\\){}\*-'; + $url2 = 'a-zA-Z0-9%=#$@+?|!&\\/_~\\[\\]\\(\\){}\*\x80-\xFE-'; $this->link_pattern = "/([\w]+:\/\/|\W[Ww][Ww][Ww]\.|^[Ww][Ww][Ww]\.)($utf_domain([$url1]*[$url2]+)*)/"; $this->mailto_pattern = "/(" diff --git a/tests/Framework/StringReplacer.php b/tests/Framework/StringReplacer.php index 0fa7fae34..7d9600a78 100644 --- a/tests/Framework/StringReplacer.php +++ b/tests/Framework/StringReplacer.php @@ -42,6 +42,10 @@ class Framework_StringReplacer extends PHPUnit_Framework_TestCase array('1@1.com www.domain.tld', '1@1.com www.domain.tld'), array(' www.domain.tld ', ' www.domain.tld '), array(' www.domain.tld/#!download|856p1|2 ', ' www.domain.tld/#!download|856p1|2 '), + // #1489898: allow some unicode characters + array('https://www.google.com/maps/place/New+York,+État+de+New+York/@40.7056308,-73.9780035,11z/data=!3m1!4b1!4m2!3m1!1s0x89c24fa5d33f083b:0xc80b8f06e177fe62', + 'https://www.google.com/maps/place/New+York,+État+de+New+York/@40.7056308,-73.9780035,11z/data=!3m1!4b1!4m2!3m1!1s0x89c24fa5d33f083b:0xc80b8f06e177fe62' + ), ); } -- cgit v1.2.3 From 59b765d83927cb9e81bf69656db3a7dbdc1b1b41 Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Sun, 18 May 2014 09:01:12 +0200 Subject: Don't remove links when html signature is converted to text (#1489621) Fix so when switching editor mode original version of signature is used (#1488849) --- CHANGELOG | 2 ++ program/js/app.js | 62 +++++++++++++++++++++++++++++++++++++----- program/steps/mail/compose.inc | 2 +- tests/Framework/Html2text.php | 5 ++++ tests/Framework/Text2Html.php | 1 + 5 files changed, 64 insertions(+), 8 deletions(-) (limited to 'tests/Framework') diff --git a/CHANGELOG b/CHANGELOG index db21586e7..27286e8e5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -20,6 +20,8 @@ CHANGELOG Roundcube Webmail - Add configurable LDAP_OPT_DEREF option (#1489864) - Optimize some framed pages content for better performance (#1489792) - Improve text messages display and conversion to HTML (#1488937) +- Don't remove links when html signature is converted to text (#1489621) +- Fix so when switching editor mode original version of signature is used (#1488849) - Fix mbox files import - Fix unintentional draft autosave request if autosave is disabled (#1489882) - Fix malformed References: header in send/saved mail (#1489891) diff --git a/program/js/app.js b/program/js/app.js index 97b8192b2..24d1f19e7 100644 --- a/program/js/app.js +++ b/program/js/app.js @@ -3439,10 +3439,27 @@ function rcube_webmail() { this.stop_spellchecking(); - var input = $('#' + props.id); + var ed, curr, content, result, + // these non-printable chars are not removed on text2html and html2text + // we can use them as temp signature replacement + sig_mark = "\u0002\u0003", + input = $('#' + props.id), + signature = this.env.identity ? this.env.signatures[this.env.identity] : null, + is_sig = signature && signature.text && signature.text.length > 1; + + if (props.mode == 'html') { + content = input.val(); + + // replace current text signature with temp mark + if (is_sig) + content = content.replace(signature.text, sig_mark); + + // convert to html + result = this.plain2html(content, function(data) { + // replace signature mark with html version of the signature + if (is_sig) + data = data.replace(sig_mark, '
' + signature.html + '
'); - if (props.mode == 'html') - this.plain2html(input.val(), function(data) { input.val(data); tinyMCE.execCommand('mceAddControl', false, props.id); @@ -3451,13 +3468,43 @@ function rcube_webmail() $(tinyMCE.get(props.id).getBody()).css('font-family', ref.env.default_font); }, 500); }); - else - this.html2plain(tinyMCE.get(props.id).getContent(), function(data) { + } + else { + ed = tinyMCE.get(props.id); + + if (is_sig) { + // get current version of signature, we'll need it in + // case of html2text conversion abort + if (curr = ed.dom.get('_rc_sig')) + curr = curr.innerHTML; + + // replace current signature with some non-printable characters + // we use non-printable characters, because this replacement + // is visible to the user + // doing this after getContent() would be hard + ed.dom.setHTML('_rc_sig', sig_mark); + } + + // get html content + content = ed.getContent(); + + // convert html to text + result = this.html2plain(content, function(data) { tinyMCE.execCommand('mceRemoveControl', false, props.id); + + // replace signture mark with text version of the signature + if (is_sig) + data = data.replace(sig_mark, "\n" + signature.text); + input.val(data); }); - return true; + // bring back current signature + if (!result && curr) + ed.dom.setHTML('_rc_sig', curr); + } + + return result; }; this.insert_response = function(key) @@ -6834,7 +6881,8 @@ function rcube_webmail() || (format != 'html' && !(text.replace(/\xC2\xA0|\s/g, '')).length) ) { // without setTimeout() here, textarea is filled with initial (onload) content - setTimeout(function() { if (func) func(''); }, 50); + if (func) + setTimeout(function() { func(''); }, 50); return true; } diff --git a/program/steps/mail/compose.inc b/program/steps/mail/compose.inc index 0ceb85db2..a3eb4b8a3 100644 --- a/program/steps/mail/compose.inc +++ b/program/steps/mail/compose.inc @@ -611,7 +611,7 @@ function rcmail_compose_header_from($attrib) $text = $html = $sql_arr['signature']; if ($sql_arr['html_signature']) { - $h2t = new rcube_html2text($sql_arr['signature'], false, false); + $h2t = new rcube_html2text($sql_arr['signature'], false, true); $text = trim($h2t->get_text()); } else { diff --git a/tests/Framework/Html2text.php b/tests/Framework/Html2text.php index 2c7759f7d..76b1f16cd 100644 --- a/tests/Framework/Html2text.php +++ b/tests/Framework/Html2text.php @@ -41,6 +41,11 @@ class rc_html2text extends PHPUnit_Framework_TestCase 'in' => 'ś', 'out' => 'Ś', ), + 6 => array( + 'title' => 'Don\'t remove non-printable chars', + 'in' => chr(0x002).chr(0x003), + 'out' => chr(0x002).chr(0x003), + ), ); } diff --git a/tests/Framework/Text2Html.php b/tests/Framework/Text2Html.php index 8d091d5c9..8d1325dee 100644 --- a/tests/Framework/Text2Html.php +++ b/tests/Framework/Text2Html.php @@ -63,6 +63,7 @@ class Framework_Text2Html extends PHPUnit_Framework_TestCase $data[] = array(">aaaa \n>aaaa", "
aaaa aaaa
", $options); $data[] = array(">aaaa\n>aaaa", "
aaaa
aaaa
", $options); $data[] = array(">aaaa \n>bbbb\ncccc dddd", "
aaaa bbbb
cccc_dddd", $options); + $data[] = array(chr(0x002).chr(0x003), chr(0x002).chr(0x003), $options); $options['flowed'] = false; $options['wrap'] = true; -- cgit v1.2.3 From 82ed256f6eeba8dce305f3953aa70681351c9bad Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Tue, 20 May 2014 19:25:45 +0200 Subject: Fix incorrect handling of HTML comments in messages sanitization code (#1489904) --- CHANGELOG | 1 + program/lib/Roundcube/rcube_washtml.php | 2 +- tests/Framework/Washtml.php | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) (limited to 'tests/Framework') diff --git a/CHANGELOG b/CHANGELOG index 27286e8e5..1b093a5a9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -26,6 +26,7 @@ CHANGELOG Roundcube Webmail - Fix unintentional draft autosave request if autosave is disabled (#1489882) - Fix malformed References: header in send/saved mail (#1489891) - Fix handling unicode characters in links (#1489898) +- Fix incorrect handling of HTML comments in messages sanitization code (#1489904) RELEASE 1.0.1 ------------- diff --git a/program/lib/Roundcube/rcube_washtml.php b/program/lib/Roundcube/rcube_washtml.php index e23e5b21d..5f40eecf4 100644 --- a/program/lib/Roundcube/rcube_washtml.php +++ b/program/lib/Roundcube/rcube_washtml.php @@ -456,7 +456,7 @@ class rcube_washtml // Remove invalid HTML comments (#1487759) // Don't remove valid conditional comments // Don't remove MSOutlook () conditional comments (#1489004) - $html = preg_replace('/

test

', $washed, "HTML invalid comments (#1487759)"); + + $html = "

para1

para2

"; + $washed = $washer->wash($html); + + $this->assertEquals('

para1

para2

', $washed, "HTML comments - simple comment"); + + $html = "

para1

para2

"; + $washed = $washer->wash($html); + + $this->assertEquals('

para1

para2

', $washed, "HTML comments - tags inside (#1489904)"); } /** -- cgit v1.2.3 From d921587f291a7effb34adab85d0feecec8663306 Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Tue, 27 May 2014 10:22:10 +0200 Subject: Add more tests for normalize_string() - some failing --- tests/Framework/Utils.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'tests/Framework') diff --git a/tests/Framework/Utils.php b/tests/Framework/Utils.php index 082aaea3b..82b8ebb73 100644 --- a/tests/Framework/Utils.php +++ b/tests/Framework/Utils.php @@ -325,8 +325,13 @@ class Framework_Utils extends PHPUnit_Framework_TestCase function test_normalize_string() { $test = array( - '' => '', + '' => '', 'abc def' => 'abc def', + 'ÇçäâàåéêëèïîìÅÉöôòüûùÿøØáíóúñÑÁÂÀãÃÊËÈÍÎÏÓÔõÕÚÛÙýÝ' => 'ccaaaaeeeeiiiaeooouuuyooaiounnaaaaaeeeiiioooouuuyy', + 'ß' => 'ss', + 'ae' => 'a', + 'oe' => 'o', + 'ue' => 'u', ); foreach ($test as $input => $output) { -- cgit v1.2.3 From 49dad5f669965ca6149c2d759ede2a91fa571149 Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Tue, 27 May 2014 11:39:31 +0200 Subject: Fix broken normalize_string(), add support for ISO-8859-2 --- program/lib/Roundcube/rcube_utils.php | 34 +++++++++++++++++++++++++--------- tests/Framework/Utils.php | 3 ++- 2 files changed, 27 insertions(+), 10 deletions(-) (limited to 'tests/Framework') diff --git a/program/lib/Roundcube/rcube_utils.php b/program/lib/Roundcube/rcube_utils.php index c2009cee0..00999ba50 100644 --- a/program/lib/Roundcube/rcube_utils.php +++ b/program/lib/Roundcube/rcube_utils.php @@ -928,7 +928,7 @@ class rcube_utils /** * Normalize the given string for fulltext search. - * Currently only optimized for Latin-1 characters; to be extended + * Currently only optimized for ISO-8859-1 and ISO-8859-2 characters; to be extended * * @param string Input string (UTF-8) * @param boolean True to return list of words as array @@ -949,15 +949,32 @@ class rcube_utils // split by words $arr = self::tokenize_string($str); + // detect character set + if (utf8_encode(utf8_decode($str)) == $str) { + // ISO-8859-1 (or ASCII) + preg_match_all('/./u', 'äâàåáãæçéêëèïîìíñöôòøõóüûùúýÿ', $keys); + preg_match_all('/./', 'aaaaaaaceeeeiiiinoooooouuuuyy', $values); + + $mapping = array_combine($keys[0], $values[0]); + $mapping = array_merge($mapping, array('ß' => 'ss', 'ae' => 'a', 'oe' => 'o', 'ue' => 'u')); + } + else if (rcube_charset::convert(rcube_charset::convert($str, 'UTF-8', 'ISO-8859-2'), 'ISO-8859-2', 'UTF-8') == $str) { + // ISO-8859-2 + preg_match_all('/./u', 'ąáâäćçčéęëěíîłľĺńňóôöŕřśšşťţůúűüźžżý', $keys); + preg_match_all('/./', 'aaaaccceeeeiilllnnooorrsssttuuuuzzzy', $values); + + $mapping = array_combine($keys[0], $values[0]); + $mapping = array_merge($mapping, array('ß' => 'ss', 'ae' => 'a', 'oe' => 'o', 'ue' => 'u')); + } + foreach ($arr as $i => $part) { - if (utf8_encode(utf8_decode($part)) == $part) { // is latin-1 ? - $arr[$i] = utf8_encode(strtr(strtolower(strtr(utf8_decode($part), - 'ÇçäâàåéêëèïîìÅÉöôòüûùÿøØáíóúñÑÁÂÀãÃÊËÈÍÎÏÓÔõÕÚÛÙýÝ', - 'ccaaaaeeeeiiiaeooouuuyooaiounnaaaaaeeeiiioooouuuyy')), - array('ß' => 'ss', 'ae' => 'a', 'oe' => 'o', 'ue' => 'u'))); + $part = mb_strtolower($part); + + if (!empty($mapping)) { + $part = strtr($part, $mapping); } - else - $arr[$i] = mb_strtolower($part); + + $arr[$i] = $part; } return $as_array ? $arr : join(" ", $arr); @@ -1039,7 +1056,6 @@ class rcube_utils } } - /** * Find out if the string content means true or false * diff --git a/tests/Framework/Utils.php b/tests/Framework/Utils.php index 82b8ebb73..560a8bde7 100644 --- a/tests/Framework/Utils.php +++ b/tests/Framework/Utils.php @@ -327,7 +327,8 @@ class Framework_Utils extends PHPUnit_Framework_TestCase $test = array( '' => '', 'abc def' => 'abc def', - 'ÇçäâàåéêëèïîìÅÉöôòüûùÿøØáíóúñÑÁÂÀãÃÊËÈÍÎÏÓÔõÕÚÛÙýÝ' => 'ccaaaaeeeeiiiaeooouuuyooaiounnaaaaaeeeiiioooouuuyy', + 'ÇçäâàåæéêëèïîìÅÉöôòüûùÿøØáíóúñÑÁÂÀãÃÊËÈÍÎÏÓÔõÕÚÛÙýÝ' => 'ccaaaaaeeeeiiiaeooouuuyooaiounnaaaaaeeeiiioooouuuyy', + 'ąáâäćçčéęëěíîłľĺńňóôöŕřśšşťţůúűüźžżýĄŚŻŹĆ' => 'aaaaccceeeeiilllnnooorrsssttuuuuzzzyaszzc', 'ß' => 'ss', 'ae' => 'a', 'oe' => 'o', -- cgit v1.2.3 From 5bf83d551e3c748b8ba37ad6459a505730ec9877 Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Tue, 27 May 2014 14:44:52 +0200 Subject: Fix unintentional line-height style modification in HTML messages (#1489917) --- CHANGELOG | 1 + program/lib/Roundcube/rcube_washtml.php | 2 +- tests/Framework/Washtml.php | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) (limited to 'tests/Framework') diff --git a/CHANGELOG b/CHANGELOG index 03d1d285a..af23c6ea5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -29,6 +29,7 @@ CHANGELOG Roundcube Webmail - Fix incorrect handling of HTML comments in messages sanitization code (#1489904) - Fix so current page is reset on list-mode change (#1489907) - Fix so responses menu hides on click in classic skin (#1489915) +- Fix unintentional line-height style modification in HTML messages (#1489917) RELEASE 1.0.1 ------------- diff --git a/program/lib/Roundcube/rcube_washtml.php b/program/lib/Roundcube/rcube_washtml.php index 5f40eecf4..984294376 100644 --- a/program/lib/Roundcube/rcube_washtml.php +++ b/program/lib/Roundcube/rcube_washtml.php @@ -206,7 +206,7 @@ class rcube_washtml $value .= ' ' . $val; // #1488535: Fix size units, so width:800 would be changed to width:800px - if (preg_match('/(left|right|top|bottom|width|height)/i', $cssid) + if (preg_match('/^(left|right|top|bottom|width|height)/i', $cssid) && preg_match('/^[0-9]+$/', $val) ) { $value .= 'px'; diff --git a/tests/Framework/Washtml.php b/tests/Framework/Washtml.php index 5c15c692c..f041504d7 100644 --- a/tests/Framework/Washtml.php +++ b/tests/Framework/Washtml.php @@ -169,4 +169,18 @@ class Framework_Washtml extends PHPUnit_Framework_TestCase $this->assertRegExp('|style="font-family: 新細明體; color: red"|', $washed, "Unicode chars in style attribute (#1489697)"); } + + /** + * Test style item fixes + */ + function test_style_wash() + { + $html = "

a

"; + + $washer = new rcube_washtml; + $washed = $washer->wash($html); + + $this->assertRegExp('|line-height: 1;|', $washed, "Untouched line-height (#1489917)"); + $this->assertRegExp('|; height: 10px|', $washed, "Fixed height units"); + } } -- cgit v1.2.3 From b231c8f6accb7c04461ab8364016d0abbe81f82e Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Mon, 7 Jul 2014 19:06:10 +0200 Subject: Fix images import from various vCard formats (#1489977) --- CHANGELOG | 1 + program/lib/Roundcube/rcube_vcard.php | 51 ++++++++++++++++++++++------------- tests/Framework/VCard.php | 16 +++++++++++ 3 files changed, 50 insertions(+), 18 deletions(-) (limited to 'tests/Framework') diff --git a/CHANGELOG b/CHANGELOG index 4a58d9ede..aa22fb0e3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -56,6 +56,7 @@ CHANGELOG Roundcube Webmail - Fix security issue in delete-response action - allow only ajax request - Fix Delete button state after deleting identity/response (#1489972) - Fix bug where contacts with no email address were listed on compose addressbook (#1489970) +- Fix images import from various vCard formats (#1489977) RELEASE 1.0.1 ------------- diff --git a/program/lib/Roundcube/rcube_vcard.php b/program/lib/Roundcube/rcube_vcard.php index fb8fdd525..4a2684f10 100644 --- a/program/lib/Roundcube/rcube_vcard.php +++ b/program/lib/Roundcube/rcube_vcard.php @@ -594,29 +594,34 @@ class rcube_vcard private static function vcard_decode($vcard) { // Perform RFC2425 line unfolding and split lines - $vcard = preg_replace(array("/\r/", "/\n\s+/"), '', $vcard); - $lines = explode("\n", $vcard); - $data = array(); + $vcard = preg_replace(array("/\r/", "/\n\s+/"), '', $vcard); + $lines = explode("\n", $vcard); + $result = array(); for ($i=0; $i < count($lines); $i++) { - if (!preg_match('/^([^:]+):(.+)$/', $lines[$i], $line)) + if (!($pos = strpos($lines[$i], ':'))) { continue; + } + + $prefix = substr($lines[$i], 0, $pos); + $data = substr($lines[$i], $pos+1); - if (preg_match('/^(BEGIN|END)$/i', $line[1])) + if (preg_match('/^(BEGIN|END)$/i', $prefix)) { continue; + } // convert 2.1-style "EMAIL;internet;home:" to 3.0-style "EMAIL;TYPE=internet;TYPE=home:" - if ($data['VERSION'][0] == "2.1" - && preg_match('/^([^;]+);([^:]+)/', $line[1], $regs2) + if ($result['VERSION'][0] == "2.1" + && preg_match('/^([^;]+);([^:]+)/', $prefix, $regs2) && !preg_match('/^TYPE=/i', $regs2[2]) ) { - $line[1] = $regs2[1]; + $prefix = $regs2[1]; foreach (explode(';', $regs2[2]) as $prop) { - $line[1] .= ';' . (strpos($prop, '=') ? $prop : 'TYPE='.$prop); + $prefix .= ';' . (strpos($prop, '=') ? $prop : 'TYPE='.$prop); } } - if (preg_match_all('/([^\\;]+);?/', $line[1], $regs2)) { + if (preg_match_all('/([^\\;]+);?/', $prefix, $regs2)) { $entry = array(); $field = strtoupper($regs2[1][0]); $enc = null; @@ -629,10 +634,10 @@ class rcube_vcard // add next line(s) to value string if QP line end detected if ($value == 'QUOTED-PRINTABLE') { while (preg_match('/=$/', $lines[$i])) { - $line[2] .= "\n" . $lines[++$i]; + $data .= "\n" . $lines[++$i]; } } - $enc = $value; + $enc = $value == 'BASE64' ? 'B' : $value; } else { $lc_key = strtolower($key); @@ -652,20 +657,30 @@ class rcube_vcard // should we use vCard 3.0 instead? // $entry['base64'] = true; } - $line[2] = self::decode_value($line[2], $enc ? $enc : 'base64'); + + $data = self::decode_value($data, $enc ? $enc : 'base64'); + } + else if ($field == 'PHOTO') { + // vCard 4.0 data URI, "PHOTO:data:image/jpeg;base64,..." + if (preg_match('/^data:[a-z\/_-]+;base64,/i', $data, $m)) { + $entry['encoding'] = $enc = 'B'; + $data = substr($data, strlen($m[0])); + $data = self::decode_value($data, 'base64'); + } } if ($enc != 'B' && empty($entry['base64'])) { - $line[2] = self::vcard_unquote($line[2]); + $data = self::vcard_unquote($data); } - $entry = array_merge($entry, (array) $line[2]); - $data[$field][] = $entry; + $entry = array_merge($entry, (array) $data); + $result[$field][] = $entry; } } - unset($data['VERSION']); - return $data; + unset($result['VERSION']); + + return $result; } /** diff --git a/tests/Framework/VCard.php b/tests/Framework/VCard.php index 3353b5b13..0a34fc51f 100644 --- a/tests/Framework/VCard.php +++ b/tests/Framework/VCard.php @@ -107,6 +107,22 @@ class Framework_VCard extends PHPUnit_Framework_TestCase // ENCODING=b case (#1488683) $this->assertEquals("/9j/4AAQSkZJRgABAQA", substr(base64_encode($vcard['photo']), 0, 19), "Photo decoding"); $this->assertEquals("Müller", $vcard['surname'], "Unicode characters"); + + $input = str_replace('ENCODING=b:', 'ENCODING=base64;jpeg:', $input); + + $vcards = rcube_vcard::import($input); + $vcard = $vcards[0]->get_assoc(); + + // ENCODING=base64 case (#1489977) + $this->assertEquals("/9j/4AAQSkZJRgABAQA", substr(base64_encode($vcard['photo']), 0, 19), "Photo decoding"); + + $input = str_replace('PHOTO;ENCODING=base64;jpeg:', 'PHOTO:data:image/jpeg;base64,', $input); + + $vcards = rcube_vcard::import($input); + $vcard = $vcards[0]->get_assoc(); + + // vcard4.0 "PHOTO:data:image/jpeg;base64," case (#1489977) + $this->assertEquals("/9j/4AAQSkZJRgABAQA", substr(base64_encode($vcard['photo']), 0, 19), "Photo decoding"); } function test_encodings() -- cgit v1.2.3 From fcb7d4fc034335d960917abd37254bd3997cf2f3 Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Fri, 1 Aug 2014 12:49:37 +0200 Subject: Fix various iCloud vCard issues, added fallback for external photos (#1489993) --- CHANGELOG | 1 + program/lib/Roundcube/html.php | 2 +- program/lib/Roundcube/rcube_vcard.php | 10 +++++----- program/steps/addressbook/func.inc | 10 +++++++--- program/steps/addressbook/photo.inc | 8 ++++++-- program/steps/mail/show.inc | 4 +++- tests/Framework/VCard.php | 19 +++++++++++++++++++ 7 files changed, 42 insertions(+), 12 deletions(-) (limited to 'tests/Framework') diff --git a/CHANGELOG b/CHANGELOG index a6ade5bfc..a2de91fa6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -41,6 +41,7 @@ CHANGELOG Roundcube Webmail - Fix some mime-type to extension mapping checks in Installer (#1489983) - Fix errors when using localStorage in Safari's private browsing mode (#1489996) - Fix bug where $Forwarded flag was being set even if server didn't support it (#1490000) +- Fix various iCloud vCard issues, added fallback for external photos (#1489993) RELEASE 1.0.2 ------------- diff --git a/program/lib/Roundcube/html.php b/program/lib/Roundcube/html.php index bcf89d7df..f18cad0bf 100644 --- a/program/lib/Roundcube/html.php +++ b/program/lib/Roundcube/html.php @@ -153,7 +153,7 @@ class html $attr = array('src' => $attr); } return self::tag('img', $attr + array('alt' => ''), null, array_merge(self::$common_attrib, - array('src','alt','width','height','border','usemap','onclick'))); + array('src','alt','width','height','border','usemap','onclick','onerror'))); } /** diff --git a/program/lib/Roundcube/rcube_vcard.php b/program/lib/Roundcube/rcube_vcard.php index 4a2684f10..96add110f 100644 --- a/program/lib/Roundcube/rcube_vcard.php +++ b/program/lib/Roundcube/rcube_vcard.php @@ -110,7 +110,7 @@ class rcube_vcard public function load($vcard, $charset = RCUBE_CHARSET, $detect = false) { self::$values_decoded = false; - $this->raw = self::vcard_decode($vcard); + $this->raw = self::vcard_decode(self::cleanup($vcard)); // resolve charset parameters if ($charset == null) { @@ -496,7 +496,7 @@ class rcube_vcard if (preg_match('/^END:VCARD$/i', $line)) { // parse vcard - $obj = new rcube_vcard(self::cleanup($vcard_block), $charset, true, self::$fieldmap); + $obj = new rcube_vcard($vcard_block, $charset, true, self::$fieldmap); // FN and N is required by vCard format (RFC 2426) // on import we can be less restrictive, let's addressbook decide if (!empty($obj->displayname) || !empty($obj->surname) || !empty($obj->firstname) || !empty($obj->email)) { @@ -532,9 +532,9 @@ class rcube_vcard // Cleanup $vcard = preg_replace(array( // convert special types (like Skype) to normal type='skype' classes with this simple regex ;) - '/item(\d+)\.(TEL|EMAIL|URL)([^:]*?):(.*?)item\1.X-ABLabel:(?:_\$!<)?([\w-() ]*)(?:>!\$_)?./s', - '/^item\d*\.X-AB.*$/m', // remove cruft like item1.X-AB* - '/^item\d*\./m', // remove item1.ADR instead of ADR + '/item(\d+)\.(TEL|EMAIL|URL)([^:]*?):(.*?)item\1.X-ABLabel:(?:_\$!<)?([\w-() ]*)(?:>!\$_)?./si', + '/^item\d*\.X-AB.*$/mi', // remove cruft like item1.X-AB* + '/^item\d*\./mi', // remove item1.ADR instead of ADR '/\n+/', // remove empty lines '/^(N:[^;\R]*)$/m', // if N doesn't have any semicolons, add some ), diff --git a/program/steps/addressbook/func.inc b/program/steps/addressbook/func.inc index d4c57cc9d..38de93d05 100644 --- a/program/steps/addressbook/func.inc +++ b/program/steps/addressbook/func.inc @@ -816,11 +816,15 @@ function rcmail_contact_photo($attrib) } $photo_img = $RCMAIL->url($url); } - else + else { $ff_value = '-del-'; // will disable delete-photo action + } - $img = html::img(array('src' => $photo_img, 'border' => 1, 'alt' => $RCMAIL->gettext('contactphoto'))); - $content = html::div($attrib, $img); + $content = html::div($attrib, html::img(array( + 'src' => $photo_img, + 'alt' => $RCMAIL->gettext('contactphoto'), + 'onerror' => 'this.src = rcmail.env.photo_placeholder', + ))); if ($CONTACT_COLTYPES['photo'] && ($RCMAIL->action == 'edit' || $RCMAIL->action == 'add')) { $RCMAIL->output->add_gui_object('contactphoto', $attrib['id']); diff --git a/program/steps/addressbook/photo.inc b/program/steps/addressbook/photo.inc index 482185735..30d09ffcc 100644 --- a/program/steps/addressbook/photo.inc +++ b/program/steps/addressbook/photo.inc @@ -72,8 +72,12 @@ $plugin = $RCMAIL->plugins->exec_hook('contact_photo', if ($plugin['url']) { $RCMAIL->output->redirect($plugin['url']); } -else { - $data = $plugin['data']; + +$data = $plugin['data']; + +// detect if photo data is an URL +if (strlen($data) < 1024 && filter_var($data, FILTER_VALIDATE_URL)) { + $RCMAIL->output->redirect($data); } // deliver alt image diff --git a/program/steps/mail/show.inc b/program/steps/mail/show.inc index 4b2d78d31..d4121fdd8 100644 --- a/program/steps/mail/show.inc +++ b/program/steps/mail/show.inc @@ -349,8 +349,10 @@ function rcmail_message_contactphoto($attrib) '_task' => 'addressbook', '_action' => 'photo', '_email' => $MESSAGE->sender['mailto'], - '_alt' => $placeholder + '_alt' => $placeholder, )); + + $attrib['onerror'] = "this.src = '" . ($placeholder ? $placeholder : 'program/resources/blank.gif') . "'"; } else { $photo_img = $placeholder ? $placeholder : 'program/resources/blank.gif'; diff --git a/tests/Framework/VCard.php b/tests/Framework/VCard.php index 0a34fc51f..c23dba844 100644 --- a/tests/Framework/VCard.php +++ b/tests/Framework/VCard.php @@ -79,6 +79,25 @@ class Framework_VCard extends PHPUnit_Framework_TestCase $this->assertEquals("http://domain.tld", $vcard['website:other'][0], "Decode dummy backslash character"); } + /** + * Some Apple vCard quirks (#1489993) + */ + function test_parse_six() + { + $vcard = new rcube_vcard("BEGIN:VCARD\n" + . "VERSION:3.0\n" + . "N:;;;;\n" + . "FN:Apple Computer AG\n" + . "ITEM1.ADR;type=WORK;type=pref:;;Birgistrasse 4a;Wallisellen-Zürich;;8304;Switzerland\n" + . "PHOTO;ENCODING=B:aHR0cDovL3Rlc3QuY29t\n" + . "END:VCARD" + ); + + $result = $vcard->get_assoc(); + + $this->assertCount(1, $result['address:work'], "ITEM1.-prefixed entry"); + } + function test_import() { $input = file_get_contents($this->_srcpath('apple.vcf')); -- cgit v1.2.3 From 5f17657e82abfae6bce0027e243e22a4183174a6 Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Thu, 14 Aug 2014 18:19:55 +0200 Subject: Support contacts import in GMail CSV format --- CHANGELOG | 1 + program/lib/Roundcube/rcube_csv2vcard.php | 175 ++++++++++++++++++++++++++++-- tests/Framework/Csv2vcard.php | 18 +++ tests/src/Csv2vcard/gmail.csv | Bin 0 -> 3760 bytes tests/src/Csv2vcard/gmail.vcf | 25 +++++ 5 files changed, 210 insertions(+), 9 deletions(-) create mode 100644 tests/src/Csv2vcard/gmail.csv create mode 100644 tests/src/Csv2vcard/gmail.vcf (limited to 'tests/Framework') diff --git a/CHANGELOG b/CHANGELOG index aecd0f422..b4269c68d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,7 @@ CHANGELOG Roundcube Webmail =========================== +- Support contacts import in GMail CSV format - Added namespace filter in Folder Manager - Added folder searching in Folder Manager - Added config option 'imap_log_session' to enable Roundcube <-> IMAP session ID logging diff --git a/program/lib/Roundcube/rcube_csv2vcard.php b/program/lib/Roundcube/rcube_csv2vcard.php index 06bc387d5..b7d159178 100644 --- a/program/lib/Roundcube/rcube_csv2vcard.php +++ b/program/lib/Roundcube/rcube_csv2vcard.php @@ -149,6 +149,13 @@ class rcube_csv2vcard // GMail 'groups' => 'groups', + 'group_membership' => 'groups', + 'given_name' => 'firstname', + 'additional_name' => 'middlename', + 'family_name' => 'surname', + 'name' => 'displayname', + 'name_prefix' => 'prefix', + 'name_suffix' => 'suffix', ); /** @@ -272,12 +279,95 @@ class rcube_csv2vcard 'work_mobile' => "Work Mobile", 'work_title' => "Work Title", 'work_zip' => "Work Zip", - 'groups' => "Group", + 'group' => "Group", + + // GMail + 'groups' => "Groups", + 'group_membership' => "Group Membership", + 'given_name' => "Given Name", + 'additional_name' => "Additional Name", + 'family_name' => "Family Name", + 'name' => "Name", + 'name_prefix' => "Name Prefix", + 'name_suffix' => "Name Suffix", + ); + + /** + * Special fields map for GMail format + * + * @var array + */ + protected $gmail_label_map = array( + 'E-mail' => array( + 'Value' => array( + 'home' => 'email:home', + 'work' => 'email:work', + ), + ), + 'Phone' => array( + 'Value' => array( + 'home' => 'phone:home', + 'homefax' => 'phone:homefax', + 'main' => 'phone:pref', + 'pager' => 'phone:pager', + 'mobile' => 'phone:cell', + 'work' => 'phone:work', + 'workfax' => 'phone:workfax', + ), + ), + 'Relation' => array( + 'Value' => array( + 'spouse' => 'spouse', + ), + ), + 'Website' => array( + 'Value' => array( + 'profile' => 'website:profile', + 'blog' => 'website:blog', + 'homepage' => 'website:homepage', + 'work' => 'website:work', + ), + ), + 'Address' => array( + 'Street' => array( + 'home' => 'street:home', + 'work' => 'street:work', + ), + 'City' => array( + 'home' => 'locality:home', + 'work' => 'locality:work', + ), + 'Region' => array( + 'home' => 'region:home', + 'work' => 'region:work', + ), + 'Postal Code' => array( + 'home' => 'zipcode:home', + 'work' => 'zipcode:work', + ), + 'Country' => array( + 'home' => 'country:home', + 'work' => 'country:work', + ), + ), + 'Organization' => array( + 'Name' => array( + '' => 'organization', + ), + 'Title' => array( + '' => 'jobtitle', + ), + 'Department' => array( + '' => 'department', + ), + ), ); + protected $local_label_map = array(); - protected $vcards = array(); - protected $map = array(); + protected $vcards = array(); + protected $map = array(); + protected $gmail_map = array(); /** @@ -308,16 +398,24 @@ class rcube_csv2vcard public function import($csv) { // convert to UTF-8 - $head = substr($csv, 0, 4096); - $charset = rcube_charset::detect($head, RCUBE_CHARSET); - $csv = rcube_charset::convert($csv, $charset); - $head = ''; + $head = substr($csv, 0, 4096); + $charset = rcube_charset::detect($head, RCUBE_CHARSET); + $csv = rcube_charset::convert($csv, $charset); + $csv = preg_replace(array('/^[\xFE\xFF]{2}/', '/^\xEF\xBB\xBF/', '/^\x00+/'), '', $csv); // also remove BOM + $head = ''; + $prev_line = false; - $this->map = array(); + $this->map = array(); + $this->gmail_map = array(); // Parse file foreach (preg_split("/[\r\n]+/", $csv) as $line) { + if (!empty($prev_line)) { + $line = '"' . $line; + } + $elements = $this->parse_line($line); + if (empty($elements)) { continue; } @@ -331,7 +429,28 @@ class rcube_csv2vcard } // Parse data row else { + // handle multiline elements (e.g. Gmail) + if (!empty($prev_line)) { + $first = array_shift($elements); + + if ($first[0] == '"') { + $prev_line[count($prev_line)-1] = '"' . $prev_line[count($prev_line)-1] . "\n" . substr($first, 1); + } + else { + $prev_line[count($prev_line)-1] .= "\n" . $first; + } + + $elements = array_merge($prev_line, $elements); + } + + $last_element = $elements[count($elements)-1]; + if ($last_element[0] == '"') { + $elements[count($elements)-1] = substr($last_element, 1); + $prev_line = $elements; + continue; + } $this->csv_to_vcard($elements); + $prev_line = false; } } } @@ -389,6 +508,7 @@ class rcube_csv2vcard $map1[$i] = $this->csv2vcard_map[$label]; } } + // check localized labels if (!empty($this->local_label_map)) { for ($i = 0; $i < $size; $i++) { @@ -406,6 +526,22 @@ class rcube_csv2vcard } $this->map = count($map1) >= count($map2) ? $map1 : $map2; + + // support special Gmail format + foreach ($this->gmail_label_map as $key => $items) { + $num = 1; + while (($_key = "$key $num - Type") && ($found = array_search($_key, $elements)) !== false) { + $this->gmail_map["$key:$num"] = array('_key' => $key, '_idx' => $found); + foreach (array_keys($items) as $item_key) { + $_key = "$key $num - $item_key"; + if (($found = array_search($_key, $elements)) !== false) { + $this->gmail_map["$key:$num"][$item_key] = $found; + } + } + + $num++; + } + } } /** @@ -421,6 +557,22 @@ class rcube_csv2vcard } } + // Gmail format support + foreach ($this->gmail_map as $idx => $item) { + $type = preg_replace('/[^a-z]/', '', strtolower($data[$item['_idx']])); + $key = $item['_key']; + + unset($item['_idx']); + unset($item['_key']); + + foreach ($item as $item_key => $item_idx) { + $value = $data[$item_idx]; + if ($value !== null && $value !== '' && ($data_idx = $this->gmail_label_map[$key][$item_key][$type])) { + $contact[$data_idx] = $value; + } + } + } + if (empty($contact)) { return; } @@ -430,9 +582,14 @@ class rcube_csv2vcard $contact['birthday'] = $contact['birthday-y'] .'-' .$contact['birthday-m'] . '-' . $contact['birthday-d']; } - // categories/groups separator in vCard is ',' not ';' if (!empty($contact['groups'])) { + // categories/groups separator in vCard is ',' not ';' $contact['groups'] = str_replace(';', ',', $contact['groups']); + + // remove "* " added by GMail + if (!empty($this->gmail_map)) { + $contact['groups'] = str_replace('* ', '', $contact['groups']); + } } // Empty dates, e.g. "0/0/00", "0000-00-00 00:00:00" diff --git a/tests/Framework/Csv2vcard.php b/tests/Framework/Csv2vcard.php index 5d52efc58..4f48dfaa2 100644 --- a/tests/Framework/Csv2vcard.php +++ b/tests/Framework/Csv2vcard.php @@ -55,4 +55,22 @@ class Framework_Csv2vcard extends PHPUnit_Framework_TestCase $vcard = trim(str_replace("\r\n", "\n", $vcard)); $this->assertEquals($vcf_text, $vcard); } + + function test_import_gmail() + { + $csv_text = file_get_contents(TESTS_DIR . '/src/Csv2vcard/gmail.csv'); + $vcf_text = file_get_contents(TESTS_DIR . '/src/Csv2vcard/gmail.vcf'); + + $csv = new rcube_csv2vcard; + $csv->import($csv_text); + $result = $csv->export(); + $vcard = $result[0]->export(false); + + $this->assertCount(1, $result); + + $vcf_text = trim(str_replace("\r\n", "\n", $vcf_text)); + $vcard = trim(str_replace("\r\n", "\n", $vcard)); + + $this->assertEquals($vcf_text, $vcard); + } } diff --git a/tests/src/Csv2vcard/gmail.csv b/tests/src/Csv2vcard/gmail.csv new file mode 100644 index 000000000..9f67fe9f5 Binary files /dev/null and b/tests/src/Csv2vcard/gmail.csv differ diff --git a/tests/src/Csv2vcard/gmail.vcf b/tests/src/Csv2vcard/gmail.vcf new file mode 100644 index 000000000..5337d7e63 --- /dev/null +++ b/tests/src/Csv2vcard/gmail.vcf @@ -0,0 +1,25 @@ +BEGIN:VCARD +VERSION:3.0 +FN:Prefix Firstname Middle Lastname Suffix +N:Lastname;Firstname;Middle;Prefix;Suffix +NICKNAME:nick +BDAY;VALUE=date:1975-12-12 +NOTE:note"note +CATEGORIES:My Contacts +EMAIL;TYPE=INTERNET;TYPE=HOME:home@aaa.pl +EMAIL;TYPE=INTERNET;TYPE=WORK:work@email.pl +TEL;TYPE=pager:pager +TEL;TYPE=pref:mainphone +TEL;TYPE=home:homephone +TEL;TYPE=homefax:homefax +TEL;TYPE=cell:mobile +TEL;TYPE=work:workphone +TEL;TYPE=workfax:workfax +X-SPOUSE:spouse +URL;TYPE=profile:test.com +URL;TYPE=homepage:home.page.com +ORG:company +TITLE:jobtitle +ADR;TYPE=home:;;home_street;home_city;home_state;home_zip;home_country +ADR;TYPE=work:;;work_street;work_city;;work_zip;work_country +END:VCARD -- cgit v1.2.3 From f01666a6229cfacee45d4131e7dbcb52e40abfea Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Thu, 21 Aug 2014 18:08:26 +0200 Subject: Fix handling of email addresses with quoted domain part (#1490040) --- CHANGELOG | 1 + program/lib/Roundcube/rcube_mime.php | 16 ++++++++++++++++ tests/Framework/Mime.php | 3 +++ 3 files changed, 20 insertions(+) (limited to 'tests/Framework') diff --git a/CHANGELOG b/CHANGELOG index 8486a9024..d6606eb16 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -56,6 +56,7 @@ CHANGELOG Roundcube Webmail - Fix errors when adding/updating contacts in active search (#1490015) - Fix incorrect thumbnail rotation with GD and exif orientation data (#1490029) - Fix contacts list update after adding/deleting/moving a contact (#1490028, #1490033) +- Fix handling of email addresses with quoted domain part (#1490040) RELEASE 1.0.2 ------------- diff --git a/program/lib/Roundcube/rcube_mime.php b/program/lib/Roundcube/rcube_mime.php index 14bc48336..f66cf1437 100644 --- a/program/lib/Roundcube/rcube_mime.php +++ b/program/lib/Roundcube/rcube_mime.php @@ -394,6 +394,7 @@ class rcube_mime } if ($address) { + $address = self::fix_email($address); $result[$key] = array('name' => $name, 'address' => $address); } } @@ -906,4 +907,19 @@ class rcube_mime return 'image/' . $type; } + /** + * Try to fix invalid email addresses + */ + public static function fix_email($email) + { + $parts = rcube_utils::explode_quoted_string('@', $email); + foreach ($parts as $idx => $part) { + // remove redundant quoting (#1490040) + if ($part[0] == '"' && preg_match('/^"([a-zA-Z0-9._+=-]+)"$/', $part, $m)) { + $parts[$idx] = $m[1]; + } + } + + return implode('@', $parts); + } } diff --git a/tests/Framework/Mime.php b/tests/Framework/Mime.php index d47eba896..43773f1a5 100644 --- a/tests/Framework/Mime.php +++ b/tests/Framework/Mime.php @@ -44,6 +44,8 @@ class Framework_Mime extends PHPUnit_Framework_TestCase 23 => '=?UTF-8?B?IlRlc3QsVGVzdCI=?= ', // invalid, but we do our best to parse correctly 24 => '"email@test.com" <>', + // valid with redundant quoting (#1490040) + 25 => '"user"@"domain.tld"', ); $results = array( @@ -73,6 +75,7 @@ class Framework_Mime extends PHPUnit_Framework_TestCase 22 => array(1, 'John Doe @ SomeBusinessName', 'MAILER-DAEMON'), 23 => array(1, 'Test,Test', 'test@domain.tld'), 24 => array(1, '', 'email@test.com'), + 25 => array(1, '', 'user@domain.tld'), ); foreach ($headers as $idx => $header) { -- cgit v1.2.3