- Improve messages display performance

3 files changed, 119 insertions, 32 deletions

diff --git a/program/include/rcube_imap.php b/program/include/rcube_imap.php
index f378f2c9f..36a346300 100644
--- a/program/include/rcube_imap.php
+++ b/program/include/rcube_imap.php
@@ -1047,9 +1047,10 @@ class rcube_imap
    * @param int     Message ID
    * @param string  Mailbox to read from 
    * @param boolean True if $id is the message UID
+   * @param boolean True if we need also BODYSTRUCTURE in headers
    * @return object Message headers representation
    */
-  function get_headers($id, $mbox_name=NULL, $is_uid=TRUE)
+  function get_headers($id, $mbox_name=NULL, $is_uid=TRUE, $bodystr=FALSE)
     {
     $mailbox = $mbox_name ? $this->_mod_mailbox($mbox_name) : $this->mailbox;
     $uid = $is_uid ? $id : $this->_id2uid($id);
@@ -1058,7 +1059,7 @@ class rcube_imap
     if ($uid && ($headers = &$this->get_cached_message($mailbox.'.msg', $uid)))
       return $headers;
 
-    $headers = iil_C_FetchHeader($this->conn, $mailbox, $id, $is_uid);
+    $headers = iil_C_FetchHeader($this->conn, $mailbox, $id, $is_uid, $bodystr);
 
     // write headers cache
     if ($headers)
@@ -1078,9 +1079,10 @@ class rcube_imap
    * an object structure similar to the one generated by PEAR::Mail_mimeDecode
    *
    * @param int Message UID to fetch
+   * @param string Message BODYSTRUCTURE string (optional)
    * @return object rcube_message_part Message part tree or False on failure
    */
-  function &get_structure($uid)
+  function &get_structure($uid, $structure_str='')
     {
     $cache_key = $this->mailbox.'.msg';
     $headers = &$this->get_cached_message($cache_key, $uid, true);
@@ -1095,7 +1097,8 @@ class rcube_imap
       return FALSE;
     }
 
-    $structure_str = iil_C_FetchStructureString($this->conn, $this->mailbox, $msg_id);
+    if (!$structure_str)
+      $structure_str = iil_C_FetchStructureString($this->conn, $this->mailbox, $msg_id);
     $structure = iml_GetRawStructureArray($structure_str);
     $struct = false;
 
@@ -1130,7 +1133,7 @@ class rcube_imap
    *
    * @access private
    */
-  function &_structure_part($part, $count=0, $parent='')
+  function &_structure_part($part, $count=0, $parent='', $raw_headers=null)
     {
     $struct = new rcube_message_part;
     $struct->mime_id = empty($parent) ? (string)$count : "$parent.$count";
@@ -1150,11 +1153,25 @@ class rcube_imap
           
       $struct->mimetype = 'multipart/'.$struct->ctype_secondary;
 
+      // build parts list for headers pre-fetching
+      for ($i=0, $count=0; $i<count($part); $i++)
+        if (is_array($part[$i]) && count($part[$i]) > 3)
+	  // fetch message headers if message/rfc822 or named part (could contain Content-Location header)
+	  if (strtolower($part[$i][0]) == 'message' ||
+	    (in_array('name', (array)$part[$i][2]) && (empty($part[$i][3]) || $part[$i][3]=='NIL'))) {
+	    $part_headers[] = $struct->mime_id ? $struct->mime_id.'.'.$i+1 : $i+1;
+	    }
+    
+      // pre-fetch headers of all parts (in one command for better performance)
+      if ($part_headers)
+        $part_headers = iil_C_FetchMIMEHeaders($this->conn, $this->mailbox, $this->_msg_id, $part_headers);
+
       $struct->parts = array();
       for ($i=0, $count=0; $i<count($part); $i++)
         if (is_array($part[$i]) && count($part[$i]) > 3)
-          $struct->parts[] = $this->_structure_part($part[$i], ++$count, $struct->mime_id);
-          
+          $struct->parts[] = $this->_structure_part($part[$i], ++$count, $struct->mime_id,
+		$part_headers[$struct->mime_id ? $struck->mime_id.'.'.$i+1 : $i+1]);
+
       return $struct;
       }
     
@@ -1219,8 +1236,9 @@ class rcube_imap
     
     // fetch message headers if message/rfc822 or named part (could contain Content-Location header)
     if ($struct->ctype_primary == 'message' || ($struct->ctype_parameters['name'] && !$struct->content_id)) {
-      $part_headers = iil_C_FetchPartHeader($this->conn, $this->mailbox, $this->_msg_id, $struct->mime_id);
-      $struct->headers = $this->_parse_headers($part_headers) + $struct->headers;
+      if (empty($raw_headers))
+        $raw_headers = iil_C_FetchPartHeader($this->conn, $this->mailbox, $this->_msg_id, $struct->mime_id);
+      $struct->headers = $this->_parse_headers($raw_headers) + $struct->headers;
     }
 
     if ($struct->ctype_primary=='message') {
@@ -2341,7 +2359,7 @@ class rcube_imap
     {
     if (empty($key) || !is_object($headers) || empty($headers->uid))
         return;
-    
+
     // add to internal (fast) cache
     $this->cache['__single_msg'][$headers->uid] = $headers;
     $this->cache['__single_msg'][$headers->uid]->structure = $struct;
diff --git a/program/include/rcube_message.php b/program/include/rcube_message.php
index 6f4963b61..f677fdcd0 100644
--- a/program/include/rcube_message.php
+++ b/program/include/rcube_message.php
@@ -65,19 +65,19 @@ class rcube_message
     $this->imap = $this->app->imap;
     
     $this->uid = $uid;
-    $this->headers = $this->imap->get_headers($uid);
+    $this->headers = $this->imap->get_headers($uid, NULL, true, true);
+
     $this->subject = rcube_imap::decode_mime_string($this->headers->subject, $this->headers->charset);
     list(, $this->sender) = each($this->imap->decode_address_list($this->headers->from));
     
     $this->set_safe((intval($_GET['_safe']) || $_SESSION['safe_messages'][$uid]));
-    
     $this->opt = array(
       'safe' => $this->is_safe,
       'prefer_html' => $this->app->config->get('prefer_html'),
       'get_url' => rcmail_url('get', array('_mbox' => $this->imap->get_mailbox_name(), '_uid' => $uid))
     );
 
-    if ($this->structure = $this->imap->get_structure($uid)) {
+    if ($this->structure = $this->imap->get_structure($uid, $this->headers->body_structure)) {
       $this->get_mime_numbers($this->structure);
       $this->parse_structure($this->structure);
     }
diff --git a/program/lib/imap.inc b/program/lib/imap.inc
index 047afdd00..1390653ed 100644
--- a/program/lib/imap.inc
+++ b/program/lib/imap.inc
@@ -76,6 +76,8 @@
 		- added 4th argument to iil_Connect()
 		- allow setting rootdir and delimiter before connect
 		- support multiquota result
+		- include BODYSTRUCTURE in iil_C_FetchHeaders()
+		- added iil_C_FetchMIMEHeaders() function
 
 ********************************************************/
 
@@ -163,6 +165,7 @@ class iilBasicHeader
 	var $flags;
 	var $timestamp;
 	var $f;
+	var $body_structure;
 	var $internaldate;
 	var $references;
 	var $priority;
@@ -1617,7 +1620,7 @@ function iil_IndexThreads(&$tree) {
 	return $t_index;
 }
 
-function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false)
+function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false, $bodystr=false)
 {
 	global $IMAP_USE_INTERNAL_DATE;
 	
@@ -1661,7 +1664,10 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false)
 	/* FETCH uid, size, flags and headers */
 	$key  	  = 'FH12';
 	$request  = $key . ($uidfetch ? ' UID' : '') . " FETCH $message_set ";
-	$request .= "(UID RFC822.SIZE FLAGS INTERNALDATE BODY.PEEK[HEADER.FIELDS ";
+	$request .= "(UID RFC822.SIZE FLAGS INTERNALDATE ";
+	if ($bodystr)
+		$request .= "BODYSTRUCTURE ";
+	$request .= "BODY.PEEK[HEADER.FIELDS ";
 	$request .= "(DATE FROM TO SUBJECT REPLY-TO IN-REPLY-TO CC BCC ";
 	$request .= "CONTENT-TRANSFER-ENCODING CONTENT-TYPE MESSAGE-ID ";
 	$request .= "REFERENCES DISPOSITION-NOTIFICATION-TO X-PRIORITY)])";
@@ -1670,7 +1676,9 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false)
 		return false;
 	}
 	do {
-		$line = chop(iil_ReadLine($fp, 1024));
+		$line = iil_ReadLine($fp, 1024);
+		$line = iil_MultLine($fp, $line);
+		
 		$a    = explode(' ', $line);
 		if (($line[0] == '*') && ($a[2] == 'FETCH')) {
 			$id = $a[1];
@@ -1680,20 +1688,23 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false)
 			$result[$id]->subject   = '';
 			$result[$id]->messageID = 'mid:' . $id;
 
+			$lines = array();
+			$ln = 0;
 			/*
 			    Sample reply line:
 			    * 321 FETCH (UID 2417 RFC822.SIZE 2730 FLAGS (\Seen)
-			    INTERNALDATE "16-Nov-2008 21:08:46 +0100" BODY[HEADER.FIELDS ...
+			    INTERNALDATE "16-Nov-2008 21:08:46 +0100" BODYSTRUCTURE (...)
+			    BODY[HEADER.FIELDS ...
 			*/
-			
-			if (preg_match('/^\* [0-9]+ FETCH \((.*) BODY\[HEADER/', $line, $matches)) {
+
+			if (preg_match('/^\* [0-9]+ FETCH \((.*) BODY/s', $line, $matches)) {
 				$str = $matches[1];
 
-				//swap parents with quotes, then explode
+				// swap parents with quotes, then explode
 				$str = eregi_replace("[()]", "\"", $str);
 				$a = iil_ExplodeQuotedString(' ', $str);
 
-				//did we get the right number of replies?
+				// did we get the right number of replies?
 				$parts_count = count($a);
 				if ($parts_count>=8) {
 					for ($i=0; $i<$parts_count; $i=$i+2) {
@@ -1761,6 +1772,27 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false)
 					$result[$id]->timestamp = $timestamp;
 					$result[$id]->date = $time_str;
 				}
+
+				// BODYSTRUCTURE 
+				if($bodystr) {
+					while (!preg_match('/ BODYSTRUCTURE (.*) BODY\[HEADER.FIELDS/s', $line, $m)) {
+						$line2 = iil_ReadLine($fp, 1024);
+						$line .= iil_MultLine($fp, $line2);
+					}
+					$result[$id]->body_structure = $m[1];
+				}
+
+				// the rest of the result
+				preg_match('/ BODY\[HEADER.FIELDS \(.*\)\]\s*(.*)/s', $line, $m);
+				$reslines = explode("\n", trim($m[1], '"'));
+				// re-parse (see below)
+				foreach ($reslines as $line) {
+					if (ord($line[0])<=32) {
+					    $lines[$ln] .= (empty($lines[$ln])?'':"\n").trim($line);
+					} else {
+						$lines[++$ln] = trim($line);
+					}
+				}
 			}
 
 			/*
@@ -1770,16 +1802,13 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false)
 				to the next valid header line.
 			*/
 	
-			$i     = 0;
-			$lines = array();
 			do {
 				$line = chop(iil_ReadLine($fp, 300), "\r\n");
 
 				if (ord($line[0])<=32) {
-				    $lines[$i] .= (empty($lines[$i])?'':"\n").trim($line);
+				    $lines[$ln] .= (empty($lines[$ln])?'':"\n").trim($line);
 				} else {
-					$i++;
-					$lines[$i] = trim($line);
+					$lines[++$ln] = trim($line);
 				}
 				/* 
 					The preg_match below works around communigate imap, which outputs " UID <number>)".
@@ -1798,8 +1827,8 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false)
 			} while (trim($line[0]) != ')' && strncmp($line, $key, strlen($key)));
 
 			if (strncmp($line, $key, strlen($key))) { 
-				//process header, fill iilBasicHeader obj.
-				//	initialize
+				// process header, fill iilBasicHeader obj.
+				// initialize
 				if (is_array($headers)) {
 					reset($headers);
 					while (list($k, $bar) = each($headers)) {
@@ -1807,7 +1836,7 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false)
 					}
 				}
 	
-				//	create array with header field:data
+				// create array with header field:data
 				while ( list($lines_key, $str) = each($lines) ) {
 					list($field, $string) = iil_SplitHeaderLine($str);
 					
@@ -1887,9 +1916,9 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false)
 	return $result;
 }
 
-function iil_C_FetchHeader(&$conn, $mailbox, $id, $uidfetch=false) {
+function iil_C_FetchHeader(&$conn, $mailbox, $id, $uidfetch=false, $bodystr=false) {
 
-	$a  = iil_C_FetchHeaders($conn, $mailbox, $id, $uidfetch);
+	$a  = iil_C_FetchHeaders($conn, $mailbox, $id, $uidfetch, $bodystr);
 	if (is_array($a)) {
 		return array_shift($a);
 	}
@@ -2366,11 +2395,51 @@ function iil_C_UnSubscribe(&$conn, $folder) {
 	return iil_ParseResult($line);
 }
 
+function iil_C_FetchMIMEHeaders(&$conn, $mailbox, $id, $parts) {
+	
+	$fp     = $conn->fp;
+
+	if (!iil_C_Select($conn, $mailbox)) {
+		return false;
+	}
+	
+	$result = false;
+	$parts = (array) $parts;
+	$key = 'fmh0';
+	$peeks = '';
+	$idx = 0;
+
+	// format request
+	foreach($parts as $part)
+		$peeks[] = "BODY[$part.MIME]";
+	
+	$request = "$key FETCH $id (" . implode(' ', $peeks) . ')';
+
+	// send request
+	if (!iil_PutLine($fp, $request)) {
+	    return false;
+	}
+        
+	do {
+        	$line = iil_ReadLine($fp, 1000);
+        	$line = iil_MultLine($fp, $line);
+
+		if (preg_match('/BODY\[([0-9\.]+)\.MIME\]/', $line, $matches)) {
+			$idx = $matches[1];
+			$result[$idx] = preg_replace('/^(\* '.$id.' FETCH \()?\s*BODY\['.$idx.'\.MIME\]\s+/', '', $line);
+			$result[$idx] = trim($result[$idx], '"');
+	    		$result[$idx] = rtrim($result[$idx], "\t\r\n\0\x0B");
+    		}
+	} while (!iil_StartsWith($line, $key, true));
+
+	return $result;
+}
+
 function iil_C_FetchPartHeader(&$conn, $mailbox, $id, $part) {
 
 	$part = empty($part) ? 'HEADER' : $part.'.MIME';
 
-	return iil_C_HandlePartBody($conn, $mailbox, $id, $part, 1);
+        return iil_C_HandlePartBody($conn, $mailbox, $id, $part, 1);
 }
 
 function iil_C_HandlePartBody(&$conn, $mailbox, $id, $part='', $mode=1, $file=NULL) {