Index: cyrus-imapd-2.3.9/imap/mailbox.c =================================================================== --- cyrus-imapd-2.3.9.orig/imap/mailbox.c 2007-09-24 08:00:35.000000000 -0400 +++ cyrus-imapd-2.3.9/imap/mailbox.c 2007-09-24 08:01:57.000000000 -0400 @@ -1905,6 +1905,7 @@ * For two-phase, we should sort by UID. */ *((bit32 *)(buf+OFFSET_LAST_UPDATED)) = htonl(now); + *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(0); n = retry_write(expunge_fd, buf, mailbox->record_size); if (n != mailbox->record_size) { syslog(LOG_ERR, @@ -2318,11 +2319,12 @@ so we delete all records from cyrus.expunge. */ if (decideproc == expungenone) decideproc = expungeall; - /* Copy over records for nonpurged messages */ + /* Copy over records for nonpurged messages, don't put them in the + * new cyrus.cache file */ r = process_records(mailbox, newexpungeindex, expunge_index_base, expunge_exists, deleted, &numdeleted, "adeleted, &numansweredflag, &numdeletedflag, - &numflaggedflag, newcache, &new_cache_total_size, + &numflaggedflag, NULL, NULL, -1, 0, decideproc, deciderock, EXPUNGE_CLEANUP); if (r) goto fail; expunge_exists -= (numdeleted - new_deleted); Index: cyrus-imapd-2.3.9/imap/reconstruct.c =================================================================== --- cyrus-imapd-2.3.9.orig/imap/reconstruct.c 2007-09-24 08:00:35.000000000 -0400 +++ cyrus-imapd-2.3.9/imap/reconstruct.c 2007-09-24 08:01:57.000000000 -0400 @@ -649,7 +649,7 @@ } return (0); -} +} char * getmailname (char * mailboxname) @@ -966,8 +966,7 @@ break; } } - if ( expunge_found == 0 ) { - + if (expunge_found) continue; /* Find old index record, if it exists */ while (old_msg < mailbox.exists && old_index.uid < uid[msg]) { @@ -1006,7 +1005,6 @@ mailbox.highestmodseq = message_index.modseq; } - } if (((r = message_parse_file(msgfile, NULL, NULL, &body)) != 0) || ((r = message_create_record(&mailbox, &message_index, body)) != 0)) { fclose(msgfile); @@ -1021,7 +1019,6 @@ fclose(msgfile); if (body) message_free_body(body); - if (expunge_found == 0) { /* if internaldate didn't get updated the body parse, get the old one * or fall back on the mtime (should be accurate since we set it * everywhere now */ @@ -1055,7 +1052,6 @@ if (message_index.system_flags & FLAG_DELETED) new_deleted++; new_quota += message_index.size; } - } if (expuid_num) { free (expuid); } Index: cyrus-imapd-2.3.9/imap/unexpunge.c =================================================================== --- cyrus-imapd-2.3.9.orig/imap/unexpunge.c 2007-09-24 08:00:35.000000000 -0400 +++ cyrus-imapd-2.3.9/imap/unexpunge.c 2007-09-24 08:01:57.000000000 -0400 @@ -68,6 +68,7 @@ #include "lock.h" #include "map.h" #include "mailbox.h" +#include "message.h" #include "mboxlist.h" #include "util.h" #include "xmalloc.h" @@ -142,41 +143,52 @@ printf("\tRecv: %s", ctime(&internaldate)); printf("\tExpg: %s", ctime(&last_updated)); - cacheitem = mailbox->cache_base + cache_offset; - cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */ - cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body structure */ - cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body */ - cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip binary body */ - cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip cached headers */ - - printf("\tFrom: %s\n", cacheitem + CACHE_ITEM_SIZE_SKIP); - cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip from */ - printf("\tTo : %s\n", cacheitem + CACHE_ITEM_SIZE_SKIP); - cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip to */ - printf("\tCc : %s\n", cacheitem + CACHE_ITEM_SIZE_SKIP); - cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip cc */ - printf("\tBcc : %s\n", cacheitem + CACHE_ITEM_SIZE_SKIP); - cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bcc */ - printf("\tSubj: %s\n\n", cacheitem + CACHE_ITEM_SIZE_SKIP); + if (cache_offset) { + cacheitem = mailbox->cache_base + cache_offset; + cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */ + cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body structure */ + cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body */ + cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip binary body */ + cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip cached headers */ + + printf("\tFrom: %s\n", cacheitem + CACHE_ITEM_SIZE_SKIP); + cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip from */ + printf("\tTo : %s\n", cacheitem + CACHE_ITEM_SIZE_SKIP); + cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip to */ + printf("\tCc : %s\n", cacheitem + CACHE_ITEM_SIZE_SKIP); + cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip cc */ + printf("\tBcc : %s\n", cacheitem + CACHE_ITEM_SIZE_SKIP); + cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bcc */ + printf("\tSubj: %s\n\n", cacheitem + CACHE_ITEM_SIZE_SKIP); + } } } int restore_expunged(struct mailbox *mailbox, struct msg *msgs, unsigned long eexists, const char *expunge_index_base, - unsigned *numrestored, int unsetdeleted) + unsigned *numrestored, int unsetdeleted, + int updateuidvalidity) { int r = 0; const char *irec; char buf[INDEX_HEADER_SIZE > INDEX_RECORD_SIZE ? INDEX_HEADER_SIZE : INDEX_RECORD_SIZE]; + char ibuf[INDEX_HEADER_SIZE > INDEX_RECORD_SIZE ? + INDEX_HEADER_SIZE : INDEX_RECORD_SIZE]; char *path, fnamebuf[MAX_MAILBOX_PATH+1], fnamebufnew[MAX_MAILBOX_PATH+1]; - FILE *newindex = NULL, *newexpungeindex = NULL; - unsigned emsgno, imsgno; + FILE *newindex = NULL, *newexpungeindex = NULL, *newcache; + bit32 sysflags; + unsigned emsgno, imsgno, newcache_offset; + unsigned cache_len, cache_offset; unsigned long iexists, euid, iuid; - uquota_t quotarestored = 0, newquotaused; - unsigned numansweredflag = 0, numdeletedflag = 0, numflaggedflag = 0; - unsigned newexists, newexpunged, newdeleted, newanswered, newflagged; + unsigned oldquotaused; + unsigned newexists = 0; + unsigned newquotaused = 0; + unsigned newexpunged = 0; + unsigned newdeleted = 0; + unsigned newanswered = 0; + unsigned newflagged = 0; time_t now = time(NULL); struct txn *tid = NULL; @@ -204,21 +216,48 @@ strlcat(fnamebufnew, ".NEW", sizeof(fnamebufnew)); newexpungeindex = fopen(fnamebufnew, "w+"); - if (!newindex) { + if (!newexpungeindex) { + syslog(LOG_ERR, "IOERROR: creating %s: %m", fnamebufnew); + fclose(newindex); + return IMAP_IOERROR; + } + + path = (mailbox->mpath && + (config_metapartition_files & IMAP_ENUM_METAPARTITION_FILES_CACHE)) ? + mailbox->mpath : mailbox->path; + + strlcpy(fnamebufnew, path, sizeof(fnamebufnew)); + strlcat(fnamebufnew, FNAME_CACHE, sizeof(fnamebufnew)); + strlcat(fnamebufnew, ".NEW", sizeof(fnamebufnew)); + + newcache = fopen(fnamebufnew, "w+"); + if (!newcache) { syslog(LOG_ERR, "IOERROR: creating %s: %m", fnamebufnew); + fclose(newexpungeindex); fclose(newindex); return IMAP_IOERROR; } - /* Copy over index/expunge headers - * - * XXX do we want/need to bump the generation number? + /* Copy over index/expunge/cache headers */ - fwrite(mailbox->index_base, 1, mailbox->start_offset, newindex); - fwrite(expunge_index_base, 1, mailbox->start_offset, newexpungeindex); + memcpy(buf, mailbox->index_base, mailbox->start_offset); + /* Update Generation Number */ + *((bit32 *)buf+OFFSET_GENERATION_NO) = htonl(mailbox->generation_no+1); + fwrite(buf, 1, mailbox->start_offset, newindex); + fwrite(buf, 1, mailbox->start_offset, newexpungeindex); + /* Write generation number to cache file */ + fwrite(buf, 1, sizeof(bit32), newcache); + +#ifdef HAVE_LONG_LONG_INT + oldquotaused = ntohll(*((bit64 *)(buf+OFFSET_QUOTA_MAILBOX_USED64))); +#else + oldquotaused = ntohl(*((bit32 *)(buf+OFFSET_QUOTA_MAILBOX_USED))); +#endif iexists = ntohl(*((bit32 *)(mailbox->index_base+OFFSET_EXISTS))); + newcache_offset = 4; + for (imsgno = 0, emsgno = 0; emsgno < eexists; emsgno++) { /* Copy expunge index record for this message */ memcpy(buf, @@ -231,18 +270,47 @@ /* Write all cyrus.index records w/ iuid < euid to cyrus.index */ for (; imsgno < iexists; imsgno++) { /* Jump to index record for this message */ - irec = mailbox->index_base + mailbox->start_offset + - imsgno * mailbox->record_size; - - iuid = ntohl(*((bit32 *)(irec+OFFSET_UID))); + memcpy(ibuf, mailbox->index_base + mailbox->start_offset + + imsgno * mailbox->record_size, + mailbox->record_size); + iuid = ntohl(*((bit32 *)(ibuf+OFFSET_UID))); if (iuid > euid) break; - fwrite(irec, 1, mailbox->record_size, newindex); + /* Update counts */ + newexists++; + newquotaused += ntohl(*((bit32 *)(ibuf+OFFSET_SIZE))); + sysflags = ntohl(*((bit32 *)(ibuf+OFFSET_SYSTEM_FLAGS))); + if (sysflags & FLAG_ANSWERED) newanswered++; + if (sysflags & FLAG_FLAGGED) newflagged++; + if (sysflags & FLAG_DELETED) newdeleted++; + + /* copy the old cache record */ + cache_offset = ntohl(*((bit32 *)(ibuf+OFFSET_CACHE_OFFSET))); + cache_len = mailbox_cache_size(mailbox, imsgno); + fwrite(mailbox->cache_base + cache_offset, 1, cache_len, newcache); + + /* update the cache location */ + *((bit32 *)(ibuf+OFFSET_CACHE_OFFSET)) = htonl(newcache_offset); + newcache_offset += cache_len; + + /* write the new index record */ + fwrite(ibuf, 1, mailbox->record_size, newindex); } if (msgs[emsgno].restore) { - bit32 sysflags = ntohl(*((bit32 *)(buf+OFFSET_SYSTEM_FLAGS))); + struct body *body = NULL; + FILE *msgfile; + char msgfname[MAILBOX_FNAME_LEN+1]; + mailbox_message_get_fname(mailbox, msgs[emsgno].uid, msgfname, sizeof(msgfname)); + msgfile = fopen(msgfname, "r"); + if (!msgfile) { + fprintf(stderr, "unexpunge: fopen() failed for '%s' [error=%d] -- skipping.\n", + msgfname, errno); + continue; + } + + sysflags = ntohl(*((bit32 *)(buf+OFFSET_SYSTEM_FLAGS))); if (verbose) { printf("\trestoring UID %ld\n", msgs[emsgno].uid); @@ -251,67 +319,85 @@ } /* Update counts */ - (*numrestored)++; - quotarestored += ntohl(*((bit32 *)(buf+OFFSET_SIZE))); - if (sysflags & FLAG_ANSWERED) numansweredflag++; - if (sysflags & FLAG_FLAGGED) numflaggedflag++; + newexists++; + newquotaused += ntohl(*((bit32 *)(buf+OFFSET_SIZE))); + if (sysflags & FLAG_ANSWERED) newanswered++; + if (sysflags & FLAG_FLAGGED) newflagged++; if (unsetdeleted) { sysflags &= ~FLAG_DELETED; *((bit32 *)(buf+OFFSET_SYSTEM_FLAGS)) = htonl(sysflags); } - else if (sysflags & FLAG_DELETED) numdeletedflag++; + else if (sysflags & FLAG_DELETED) newdeleted++; - /* Write record to cyrus.index */ + /* parse the body and add cache record */ + message_parse_file(msgfile, NULL, NULL, &body); + cache_len = message_write_cache(newcache, body); + message_free_body(body); + fclose(msgfile); + + /* update the next index record */ *((bit32 *)(buf+OFFSET_LAST_UPDATED)) = htonl(now); + *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(newcache_offset); + newcache_offset += cache_len; + + /* Write record to cyrus.index */ fwrite(buf, 1, mailbox->record_size, newindex); } else { /* Write record to cyrus.expunge */ + newexpunged++; fwrite(buf, 1, mailbox->record_size, newexpungeindex); } } /* Write all remaining cyrus.index records to cyrus.index */ - if (imsgno < iexists) { + for (; imsgno < iexists; imsgno++) { /* Jump to index record for next message */ - irec = mailbox->index_base + mailbox->start_offset + - imsgno * mailbox->record_size; + memcpy(ibuf, mailbox->index_base + mailbox->start_offset + + imsgno * mailbox->record_size, + mailbox->record_size); + + /* Update counts */ + newexists++; + newquotaused += ntohl(*((bit32 *)(ibuf+OFFSET_SIZE))); + sysflags = ntohl(*((bit32 *)(ibuf+OFFSET_SYSTEM_FLAGS))); + if (sysflags & FLAG_ANSWERED) newanswered++; + if (sysflags & FLAG_FLAGGED) newflagged++; + if (sysflags & FLAG_DELETED) newdeleted++; + + /* copy the old cache record */ + cache_offset = ntohl(*((bit32 *)(ibuf+OFFSET_CACHE_OFFSET))); + cache_len = mailbox_cache_size(mailbox, imsgno); + fwrite(mailbox->cache_base + cache_offset, 1, cache_len, newcache); + + /* update the index record */ + *((bit32 *)(ibuf+OFFSET_CACHE_OFFSET)) = htonl(newcache_offset); + newcache_offset += cache_len; - fwrite(irec, 1, (iexists - imsgno) * mailbox->record_size, newindex); + /* and write it to cyrus.index */ + fwrite(ibuf, 1, mailbox->record_size, newindex); } /* Fix up information in index header */ memcpy(buf, mailbox->index_base, mailbox->start_offset); /* Update uidvalidity */ - *((bit32 *)(buf+OFFSET_UIDVALIDITY)) = now; + if (updateuidvalidity) + *((bit32 *)(buf+OFFSET_UIDVALIDITY)) = now; - /* Fix up exists */ - newexists = ntohl(*((bit32 *)(buf+OFFSET_EXISTS))) + *numrestored; + /* Fix up counts */ *((bit32 *)(buf+OFFSET_EXISTS)) = htonl(newexists); - - /* Fix up expunged count */ - newexpunged = ntohl(*((bit32 *)(buf+OFFSET_LEAKED_CACHE))) - *numrestored; - *((bit32 *)(buf+OFFSET_LEAKED_CACHE)) = htonl(newexpunged); - - /* Fix up other counts */ - newanswered = ntohl(*((bit32 *)(buf+OFFSET_ANSWERED))) + numansweredflag; + *((bit32 *)(buf+OFFSET_LEAKED_CACHE)) = htonl(0); *((bit32 *)(buf+OFFSET_ANSWERED)) = htonl(newanswered); - newdeleted = ntohl(*((bit32 *)(buf+OFFSET_DELETED))) + numdeletedflag; *((bit32 *)(buf+OFFSET_DELETED)) = htonl(newdeleted); - newflagged = ntohl(*((bit32 *)(buf+OFFSET_FLAGGED))) + numflaggedflag; *((bit32 *)(buf+OFFSET_FLAGGED)) = htonl(newflagged); /* Fix up quota_mailbox_used */ #ifdef HAVE_LONG_LONG_INT - newquotaused = - ntohll(*((bit64 *)(buf+OFFSET_QUOTA_MAILBOX_USED64))) + quotarestored; *((bit64 *)(buf+OFFSET_QUOTA_MAILBOX_USED64)) = htonll(newquotaused); #else /* Zero the unused 32bits */ *((bit32 *)(buf+OFFSET_QUOTA_MAILBOX_USED64)) = htonl(0); - newquotaused = - ntohl(*((bit32 *)(buf+OFFSET_QUOTA_MAILBOX_USED))) + quotarestored; *((bit32 *)(buf+OFFSET_QUOTA_MAILBOX_USED)) = htonl(newquotaused); #endif @@ -330,34 +416,9 @@ *((bit32 *)(buf+OFFSET_UIDVALIDITY)) = now; /* Fix up exists */ - newexists = ntohl(*((bit32 *)(buf+OFFSET_EXISTS))) - *numrestored; - *((bit32 *)(buf+OFFSET_EXISTS)) = htonl(newexists); - - /* Fix up other counts */ - newanswered = ntohl(*((bit32 *)(buf+OFFSET_ANSWERED))) - numansweredflag; - *((bit32 *)(buf+OFFSET_ANSWERED)) = htonl(newanswered); - /* XXX we use the numrestored count here because we may have unset - * the \Deleted flag when we copied the record to cyrus.index, - * but we know that any message that has to be restored had the - * \Deleted set in cyrus.expunge in the first place - */ - newdeleted = ntohl(*((bit32 *)(buf+OFFSET_DELETED))) - *numrestored; - *((bit32 *)(buf+OFFSET_DELETED)) = htonl(newdeleted); - newflagged = ntohl(*((bit32 *)(buf+OFFSET_FLAGGED))) - numflaggedflag; - *((bit32 *)(buf+OFFSET_FLAGGED)) = htonl(newflagged); + *((bit32 *)(buf+OFFSET_EXISTS)) = htonl(newexpunged); - /* Fix up quota_mailbox_used */ -#ifdef HAVE_LONG_LONG_INT - newquotaused = - ntohll(*((bit64 *)(buf+OFFSET_QUOTA_MAILBOX_USED64))) - quotarestored; - *((bit64 *)(buf+OFFSET_QUOTA_MAILBOX_USED64)) = htonll(newquotaused); -#else - /* Zero the unused 32bits */ - *((bit32 *)(buf+OFFSET_QUOTA_MAILBOX_USED64)) = htonl(0); - newquotaused = - ntohl(*((bit32 *)(buf+OFFSET_QUOTA_MAILBOX_USED))) - quotarestored; - *((bit32 *)(buf+OFFSET_QUOTA_MAILBOX_USED)) = htonl(newquotaused); -#endif + /* Don't care about other counts in a deleted file */ /* Write out new expunge index header */ rewind(newexpungeindex); @@ -400,16 +461,32 @@ return IMAP_IOERROR; } + path = (mailbox->mpath && + (config_metapartition_files & IMAP_ENUM_METAPARTITION_FILES_CACHE)) ? + mailbox->mpath : mailbox->path; + + strlcpy(fnamebuf, path, sizeof(fnamebuf)); + strlcat(fnamebuf, FNAME_CACHE, sizeof(fnamebuf)); + + strlcpy(fnamebufnew, fnamebuf, sizeof(fnamebufnew)); + strlcat(fnamebufnew, ".NEW", sizeof(fnamebufnew)); + + if (rename(fnamebufnew, fnamebuf)) { + syslog(LOG_ERR, "IOERROR: renaming cache file for %s: %m", + mailbox->name); + return IMAP_IOERROR; + } + /* Record quota restore */ r = quota_read(&mailbox->quota, &tid, 1); if (!r) { - mailbox->quota.used += quotarestored; + mailbox->quota.used += newquotaused - oldquotaused; r = quota_write(&mailbox->quota, &tid); if (!r) quota_commit(&tid); else { syslog(LOG_ERR, "LOSTQUOTA: unable to record restore of " UQUOTA_T_FMT " bytes in quota %s", - quotarestored, mailbox->quota.root); + newquotaused - oldquotaused, mailbox->quota.root); } } else if (r == IMAP_QUOTAROOT_NONEXISTENT) r = 0; @@ -427,6 +504,7 @@ int doclose = 0, mode = MODE_UNKNOWN, unsetdeleted = 0; char expunge_fname[MAX_MAILBOX_PATH+1]; int expunge_fd = -1; + int updateuidvalidity = 1; struct stat sbuf; const char *lockfailaction; struct msg *msgs; @@ -436,7 +514,7 @@ fatal("must run as the Cyrus user", EC_USAGE); } - while ((opt = getopt(argc, argv, "C:laudv")) != EOF) { + while ((opt = getopt(argc, argv, "C:laudvn")) != EOF) { switch (opt) { case 'C': /* alt config file */ alt_config = optarg; @@ -465,6 +543,10 @@ verbose = 1; break; + case 'n': + updateuidvalidity = 0; + break; + default: usage(); break; @@ -597,7 +679,7 @@ mode == MODE_ALL ? "all " : "", mailbox.name); r = restore_expunged(&mailbox, msgs, exists, expunge_index_base, - &numrestored, unsetdeleted); + &numrestored, unsetdeleted, updateuidvalidity); if (!r) { printf("restored %u out of %lu expunged messages\n", numrestored, exists); Index: cyrus-imapd-2.3.9/man/unexpunge.8 =================================================================== --- cyrus-imapd-2.3.9.orig/man/unexpunge.8 2007-09-24 08:00:35.000000000 -0400 +++ cyrus-imapd-2.3.9/man/unexpunge.8 2007-09-24 08:01:57.000000000 -0400 @@ -63,6 +63,9 @@ [ .B \-v ] +[ +.B \-n +] .I mailbox .br .B unexpunge @@ -109,6 +112,12 @@ .TP .B \-v Enable verbose output/logging. +.TP +.B \-n +Disable UIDVALIDITY change. This may confuse your clients since messages +that were already expunged have magically re-appeared. On the other hand +if your MUA has tons of messages cached and you just accidentally deleted +a message some other way, it can be handy not to have to resync the lot. .SH FILES .TP .B /etc/imapd.conf Index: cyrus-imapd-2.3.9/imap/message.c =================================================================== --- cyrus-imapd-2.3.9.orig/imap/message.c 2007-09-24 08:00:35.000000000 -0400 +++ cyrus-imapd-2.3.9/imap/message.c 2007-09-24 08:01:57.000000000 -0400 @@ -198,8 +198,6 @@ static int message_pendingboundary P((const char *s, char **boundaries, int *boundaryct)); -static int message_write_cache P((int outfd, struct body *body)); - static void message_write_envelope P((struct ibuf *ibuf, struct body *body)); static void message_write_body P((struct ibuf *ibuf, struct body *body, int newformat)); @@ -2052,7 +2050,7 @@ * Write the cache information for the message parsed to 'body' * to 'outfile'. */ -static int +int message_write_cache(outfd, body) int outfd; struct body *body; Index: cyrus-imapd-2.3.9/imap/message.h =================================================================== --- cyrus-imapd-2.3.9.orig/imap/message.h 2007-09-24 08:00:35.000000000 -0400 +++ cyrus-imapd-2.3.9/imap/message.h 2007-09-24 08:01:57.000000000 -0400 @@ -97,6 +97,7 @@ struct index_record *message_index, struct body *body)); extern void message_free_body P((struct body *body)); +extern int message_write_cache P((int outfd, struct body *body)); extern int message_parse_mapped_async P((const char *msg_base, unsigned long msg_len,