Finalized new message parsing. Attention: changes in database schema

4 files changed, 112 insertions, 138 deletions

diff --git a/program/include/rcube_imap.inc b/program/include/rcube_imap.inc
index fe97a92f2..dbb00a5cf 100644
--- a/program/include/rcube_imap.inc
+++ b/program/include/rcube_imap.inc
@@ -35,7 +35,7 @@ require_once('lib/mime.inc');
  *
  * @package    RoundCube Webmail
  * @author     Thomas Bruederli <roundcube@gmail.com>
- * @version    1.31
+ * @version    1.34
  * @link       http://ilohamail.org
  */
 class rcube_imap
@@ -132,11 +132,10 @@ class rcube_imap
                        'message' => $GLOBALS['iil_error']), TRUE, FALSE);
       }
 
-    // get account namespace
+    // get server properties
     if ($this->conn)
       {
       $this->_parse_capability($this->conn->capability);
-      iil_C_NameSpace($this->conn);
       
       if (!empty($this->conn->delimiter))
         $this->delimiter = $this->conn->delimiter;
@@ -883,15 +882,19 @@ class rcube_imap
     $uid = $is_uid ? $id : $this->_id2uid($id);
 
     // get cached headers
-    if ($uid && ($headers = $this->get_cached_message($mailbox.'.msg', $uid)))
+    if ($uid && ($headers = &$this->get_cached_message($mailbox.'.msg', $uid)))
       return $headers;
 
-    $msg_id = $is_uid ? $this->_uid2id($id) : $id;
-    $headers = iil_C_FetchHeader($this->conn, $mailbox, $msg_id);
+    $headers = iil_C_FetchHeader($this->conn, $mailbox, $id, $is_uid);
 
     // write headers cache
     if ($headers)
-      $this->add_message_cache($mailbox.'.msg', $msg_id, $headers);
+      {
+      if ($is_uid)
+        $this->uid_id_map[$mbox_name][$uid] = $headers->id;
+
+      $this->add_message_cache($mailbox.'.msg', $headers->id, $headers);
+      }
 
     return $headers;
     }
@@ -906,6 +909,14 @@ class rcube_imap
    */
   function &get_structure($uid)
     {
+    $cache_key = $this->mailbox.'.msg';
+    $headers = &$this->get_cached_message($cache_key, $uid, true);
+
+    // return cached message structure
+    if (is_object($headers) && is_object($headers->structure))
+      return $headers->structure;
+    
+    // resolve message sequence number
     if (!($msg_id = $this->_uid2id($uid)))
       return FALSE;
 
@@ -929,6 +940,10 @@ class rcube_imap
         $struct->mimetype = strtolower($struct->headers['ctype']);
         list($struct->ctype_primary, $struct->ctype_secondary) = explode('/', $struct->mimetype);
         }
+
+      // write structure to cache
+      if ($this->caching_enabled)
+        $this->add_message_cache($cache_key, $msg_id, $headers, $struct);
       }
 	
 	return $struct;
@@ -1109,8 +1124,14 @@ class rcube_imap
         $body = $this->mime_decode($body, $o_part->encoding);
 
       // convert charset (if text or message part)
-      if (!empty($o_part->charset) && ($o_part->ctype_primary=='text' || $o_part->ctype_primary=='message') && !stristr($body, 'charset='))
+      if ($o_part->ctype_primary=='text' || $o_part->ctype_primary=='message')
+        {
+        // assume ISO-8859-1 if no charset specified
+        if (empty($o_part->charset))
+          $o_part->charset = 'ISO-8859-1';
+
         $body = rcube_charset_convert($body, $o_part->charset);
+        }
       }
 
     return $body;
@@ -1533,6 +1554,14 @@ class rcube_imap
     // make absolute path
     $mailbox = $this->_mod_mailbox($mbox_name);
     $abs_name = $this->_mod_mailbox($name);
+    
+    // check if mailbox is subscribed
+    $a_subscribed = $this->_list_mailboxes();
+    $subscribed = in_array($mailbox, $a_subscribed);
+    
+    // unsubscribe folder
+    if ($subscribed)
+      iil_C_UnSubscribe($this->conn, $mailbox);
 
     if (strlen($abs_name))
       $result = iil_C_RenameFolder($this->conn, $mailbox, $abs_name);
@@ -1541,11 +1570,12 @@ class rcube_imap
     if ($result)
       {
       $this->clear_message_cache($mailbox.'.msg');
-      $this->clear_cache('mailboxes');
+      $this->clear_cache('mailboxes');      
       }
-      
+
     // try to subscribe it
-    $this->subscribe($name);
+    if ($result && $subscribed)
+      iil_C_Subscribe($this->conn, $abs_name);
 
     return $result ? $name : FALSE;
     }
@@ -1842,18 +1872,16 @@ class rcube_imap
     }
 
 
-  function &get_cached_message($key, $uid, $body=FALSE)
+  function &get_cached_message($key, $uid, $struct=false)
     {
     if (!$this->caching_enabled)
       return FALSE;
 
     $internal_key = '__single_msg';
-    if ($this->caching_enabled && (!isset($this->cache[$internal_key][$uid]) || $body))
+    if ($this->caching_enabled && (!isset($this->cache[$internal_key][$uid]) ||
+        ($struct && empty($this->cache[$internal_key][$uid]->structure))))
       {
-      $sql_select = "idx, uid, headers";
-      if ($body)
-        $sql_select .= ", body";
-      
+      $sql_select = "idx, uid, headers" . ($struct ? ", structure" : '');
       $sql_result = $this->db->query(
         "SELECT $sql_select
          FROM ".get_table_name('messages')."
@@ -1863,14 +1891,12 @@ class rcube_imap
         $_SESSION['user_id'],
         $key,
         $uid);
-      
+
       if ($sql_arr = $this->db->fetch_assoc($sql_result))
         {
-        $headers = unserialize($sql_arr['headers']);
-        if (is_object($headers) && !empty($sql_arr['body']))
-          $headers->body = $sql_arr['body'];
-
-        $this->cache[$internal_key][$uid] = $headers;
+        $this->cache[$internal_key][$uid] = unserialize($sql_arr['headers']);
+        if (is_object($this->cache[$internal_key][$uid]) && !empty($sql_arr['structure']))
+          $this->cache[$internal_key][$uid]->structure = unserialize($sql_arr['structure']);
         }
       }
 
@@ -1906,25 +1932,55 @@ class rcube_imap
     }
 
 
-  function add_message_cache($key, $index, $headers)
+  function add_message_cache($key, $index, $headers, $struct=null)
     {
-    if (!$key || !is_object($headers) || empty($headers->uid))
+    if (empty($key) || !is_object($headers) || empty($headers->uid))
       return;
+      
+    // check for an existing record (probly headers are cached but structure not)
+    $sql_result = $this->db->query(
+        "SELECT message_id
+         FROM ".get_table_name('messages')."
+         WHERE  user_id=?
+         AND    cache_key=?
+         AND    uid=?
+         AND    del<>1",
+        $_SESSION['user_id'],
+        $key,
+        $headers->uid);
 
-    $this->db->query(
-      "INSERT INTO ".get_table_name('messages')."
-       (user_id, del, cache_key, created, idx, uid, subject, ".$this->db->quoteIdentifier('from').", ".$this->db->quoteIdentifier('to').", cc, date, size, headers)
-       VALUES (?, 0, ?, now(), ?, ?, ?, ?, ?, ?, ".$this->db->fromunixtime($headers->timestamp).", ?, ?)",
-      $_SESSION['user_id'],
-      $key,
-      $index,
-      $headers->uid,
-      (string)substr($this->decode_header($headers->subject, TRUE), 0, 128),
-      (string)substr($this->decode_header($headers->from, TRUE), 0, 128),
-      (string)substr($this->decode_header($headers->to, TRUE), 0, 128),
-      (string)substr($this->decode_header($headers->cc, TRUE), 0, 128),
-      (int)$headers->size,
-      serialize($headers));
+    // update cache record
+    if ($sql_arr = $this->db->fetch_assoc($sql_result))
+      {
+      $this->db->query(
+        "UPDATE ".get_table_name('messages')."
+         SET   idx=?, headers=?, structure=?
+         WHERE message_id=?",
+        $index,
+        serialize($headers),
+        is_object($struct) ? serialize($struct) : NULL,
+        $sql_arr['message_id']
+        );
+      }
+    else  // insert new record
+      {
+      $this->db->query(
+        "INSERT INTO ".get_table_name('messages')."
+         (user_id, del, cache_key, created, idx, uid, subject, ".$this->db->quoteIdentifier('from').", ".$this->db->quoteIdentifier('to').", cc, date, size, headers, structure)
+         VALUES (?, 0, ?, now(), ?, ?, ?, ?, ?, ?, ".$this->db->fromunixtime($headers->timestamp).", ?, ?, ?)",
+        $_SESSION['user_id'],
+        $key,
+        $index,
+        $headers->uid,
+        (string)substr($this->decode_header($headers->subject, TRUE), 0, 128),
+        (string)substr($this->decode_header($headers->from, TRUE), 0, 128),
+        (string)substr($this->decode_header($headers->to, TRUE), 0, 128),
+        (string)substr($this->decode_header($headers->cc, TRUE), 0, 128),
+        (int)$headers->size,
+        serialize($headers),
+        is_object($struct) ? serialize($struct) : NULL
+        );
+      }
     }
     
     
diff --git a/program/lib/imap.inc b/program/lib/imap.inc
index 2c07f3409..b2e1d43c0 100644
--- a/program/lib/imap.inc
+++ b/program/lib/imap.inc
@@ -46,6 +46,7 @@
 		- Casting date parts in iil_StrToTime() to avoid mktime() warnings
 		- Also acceppt LIST responses in iil_C_ListSubscribed()
 		- Sanity check of $message_set in iil_C_FetchHeaders(), iil_C_FetchHeaderIndex(), iil_C_FetchThreadHeaders()
+		- Implemented UID FETCH in iil_C_FetchHeaders()
 		- Removed some debuggers (echo ...)
 
 ********************************************************/
@@ -1195,7 +1196,7 @@ function iil_IndexThreads(&$tree){
 	return $t_index;
 }
 
-function iil_C_FetchHeaders(&$conn, $mailbox, $message_set){
+function iil_C_FetchHeaders(&$conn, $mailbox, $message_set, $uidfetch=false){
 	global $IMAP_USE_INTERNAL_DATE;
 	
 	$c=0;
@@ -1231,7 +1232,8 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set){
 
 	/* FETCH date,from,subject headers */
 	$key="fh".($c++);
-	$request=$key." FETCH $message_set (BODY.PEEK[HEADER.FIELDS (DATE FROM TO SUBJECT REPLY-TO IN-REPLY-TO CC BCC CONTENT-TRANSFER-ENCODING CONTENT-TYPE MESSAGE-ID REFERENCE)])\r\n";
+	$prefix=$uidfetch?" UID":"";
+	$request=$key.$prefix." FETCH $message_set (BODY.PEEK[HEADER.FIELDS (DATE FROM TO SUBJECT REPLY-TO IN-REPLY-TO CC BCC CONTENT-TRANSFER-ENCODING CONTENT-TYPE MESSAGE-ID REFERENCE)])\r\n";
 
 	if (!fputs($fp, $request)) return false;
 	do{
@@ -1309,7 +1311,7 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set){
 		Sample reply line: "* 3 FETCH (UID 2417 RFC822.SIZE 2730 FLAGS (\Seen \Deleted))"
 	*/
 	$command_key="fh".($c++);
-	$request= $command_key." FETCH $message_set (UID RFC822.SIZE FLAGS INTERNALDATE)\r\n";
+	$request= $command_key.$prefix." FETCH $message_set (UID RFC822.SIZE FLAGS INTERNALDATE)\r\n";
 	if (!fputs($fp, $request)) return false;
 	do{
 		$line=chop(iil_ReadLine($fp, 200));
@@ -1401,10 +1403,10 @@ function iil_C_FetchHeaders(&$conn, $mailbox, $message_set){
 }
 
 
-function iil_C_FetchHeader(&$conn, $mailbox, $id){
+function iil_C_FetchHeader(&$conn, $mailbox, $id, $uidfetch=false){
 	$fp = $conn->fp;
-	$a=iil_C_FetchHeaders($conn, $mailbox, $id);
-	if (is_array($a)) return $a[$id];
+	$a=iil_C_FetchHeaders($conn, $mailbox, $id, $uidfetch);
+	if (is_array($a)) return array_shift($a);
 	else return false;
 }
 
diff --git a/program/steps/mail/func.inc b/program/steps/mail/func.inc
index cbe455934..a2f2e07c2 100644
--- a/program/steps/mail/func.inc
+++ b/program/steps/mail/func.inc
@@ -671,6 +671,9 @@ function rcmail_print_body($part, $safe=FALSE, $plain=FALSE)
   // text/html
   if ($part->ctype_secondary=='html')
     {
+    // remove charset specification in HTML message
+    $body = preg_replace('/charset=[a-z0-9\-]+/i', '', $body);
+
     if (!$safe)  // remove remote images and scripts
       {
       $remote_patterns = array('/(src|background)=(["\']?)([hftps]{3,5}:\/{2}[^"\'\s]+)(\2|\s|>)/Ui',
@@ -1247,94 +1250,6 @@ function rcmail_first_text_part($message_struct)
   }
 
 
-// get source code of a specific message and cache it
-// deprecated
-function rcmail_message_source($uid)
-  {
-  global $IMAP, $DB, $CONFIG;
-
-  // get message ID if uid is given
-  $cache_key = $IMAP->mailbox.'.msg';
-  $cached = $IMAP->get_cached_message($cache_key, $uid, FALSE);
-  
-  // message is cached in database
-  if ($cached && !empty($cached->body))
-    return $cached->body;
-
-  if (!$cached)
-    $headers = $IMAP->get_headers($uid);
-  else
-    $headers = &$cached;
-
-  // create unique identifier based on message_id
-  if (!empty($headers->messageID))
-    $message_id = md5($headers->messageID);
-  else
-    $message_id = md5($headers->uid.'@'.$_SESSION['imap_host']);
-  
-  $temp_dir = $CONFIG['temp_dir'].(!eregi('\/$', $CONFIG['temp_dir']) ? '/' : '');
-  $cache_dir = $temp_dir.$_SESSION['client_id'];
-  $cache_path = $cache_dir.'/'.$message_id;
-
-  // message is cached in temp dir
-  if ($CONFIG['enable_caching'] && is_dir($cache_dir) && is_file($cache_path))
-    {
-    if ($fp = fopen($cache_path, 'r'))
-      {
-      $msg_source = fread($fp, filesize($cache_path));
-      fclose($fp);
-      return $msg_source;
-      }
-    }
-
-
-  // get message from server
-  $msg_source = $IMAP->get_raw_body($uid);
-  
-  // return message source without caching
-  if (!$CONFIG['enable_caching'])
-    return $msg_source;
-
-
-  // let's cache the message body within the database
-  if ($cached && ($CONFIG['db_max_length'] -300) > $headers->size)
-    {
-    $DB->query("UPDATE ".get_table_name('messages')."
-                SET    body=?
-                WHERE  user_id=?
-                AND    cache_key=?
-                AND    uid=?",
-               $msg_source,
-               $_SESSION['user_id'],
-               $cache_key,
-               $uid);
-
-    return $msg_source;
-    }
-
-
-  // create dir for caching
-  if (!is_dir($cache_dir))
-    $dir = mkdir($cache_dir);
-  else
-    $dir = true;
-
-  // attempt to write a file with the message body    
-  if ($dir && ($fp = fopen($cache_path, 'w')))
-    {
-    fwrite($fp, $msg_source);
-    fclose($fp);
-    }
-  else
-    {
-    raise_error(array('code' => 403, 'type' => 'php', 'line' => __LINE__, 'file' => __FILE__, 
-                      'message' => "Failed to write to temp dir"), TRUE, FALSE);
-    }
-
-  return $msg_source;
-  }
-
-
 // decode address string and re-format it as HTML links
 function rcmail_address_string($input, $max=NULL, $addicon=NULL)
   {
@@ -1412,7 +1327,7 @@ function rcmail_message_part_controls()
     {
     $out .= sprintf('<tr><td class="title">%s</td><td>%s</td><td>[<a href="./?%s">%s</a>]</tr>'."\n",
                     rcube_label('filename'),
-                    rep_specialchars_output($filename),
+                    rep_specialchars_output(rcube_imap::decode_mime_string($filename)),
                     str_replace('_frame=', '_download=', $_SERVER['QUERY_STRING']),
                     rcube_label('download'));
     }
diff --git a/program/steps/mail/get.inc b/program/steps/mail/get.inc
index ded6a0022..f05bbbd94 100644
--- a/program/steps/mail/get.inc
+++ b/program/steps/mail/get.inc
@@ -69,12 +69,12 @@ else if ($pid = get_input_value('_part', RCUBE_INPUT_GET))
       // send download headers
       header("Content-Type: application/octet-stream");
       header(sprintf('Content-Disposition: attachment; filename="%s"',
-                     $filename ? $filename : "roundcube.$ctype_secondary"));
+                     $filename ? rcube_imap::decode_mime_string($filename) : "roundcube.$ctype_secondary"));
       }
     else
       {
       header("Content-Type: $mimetype");
-      header(sprintf('Content-Disposition: inline; filename="%s"', $filename));
+      header(sprintf('Content-Disposition: inline; filename="%s"', rcube_imap::decode_mime_string($filename)));
       }
 
     // We need to set the following headers to make downloads work using IE in HTTPS mode.
@@ -89,14 +89,15 @@ else if ($pid = get_input_value('_part', RCUBE_INPUT_GET))
       {
       // get part body if not available
       if (!$part->body)
-        $part->body = $IMAP->get_message_part($MESSAGE['UID'], $part->mime_id, $part);        
+        $part->body = $IMAP->get_message_part($MESSAGE['UID'], $part->mime_id, $part);      
 
       list($MESSAGE['parts']) = rcmail_parse_message($part,
                                                      array('safe' => (bool)$_GET['_safe'],
                                                            'prefer_html' => TRUE,
                                                            'get_url' => $GET_URL.'&_part=%s'));
 
-      print rcmail_print_body($MESSAGE['parts'][0], (bool)$_GET['_safe']);
+      $OUTPUT = new rcube_html_page();
+      $OUTPUT->write(rcmail_print_body($MESSAGE['parts'][0], (bool)$_GET['_safe']));
       }
     else
       {