courier-imap: implement UTF8 mode in the POP3 server.

6 files changed, 99 insertions, 76 deletions

diff --git a/imap/ChangeLog b/imap/ChangeLog
index ba51db1..4c10f8e 100644
--- a/imap/ChangeLog
+++ b/imap/ChangeLog
@@ -1,3 +1,8 @@
+2018-07-21  Sam Varshavchik  <mrsam@courier-mta.com>
+
+	* pop3dserver.c: update Courier-IMAP to support UTF8 POP3. Update
+	version of the courierpop3dsizelist cache file.
+
 2018-07-17  Sam Varshavchik  <mrsam@courier-mta.com>
 
 	* courier-imap, sqwebmail: update Courier-IMAP to support UTF8 IMAP.
diff --git a/imap/Makefile.am b/imap/Makefile.am
index 7a7f4f5..7d8f5ff 100644
--- a/imap/Makefile.am
+++ b/imap/Makefile.am
@@ -85,14 +85,16 @@ pop3login_DEPENDENCIES=../tcpd/libspipe.la ../tcpd/libspipe.la libpop3d.la
 pop3login_LDADD=../tcpd/libtlsclient.la ../tcpd/libspipe.la libpop3d.la ../tcpd/libspipe.la @LDAUTH@ -lcourierauth -lcourierauthsasl @NETLIBS@
 
 libpop3d_la_SOURCES=externalauth.c
-libpop3d_la_LIBADD=../maildir/libmaildir.la ../rfc822/librfc822.la \
+libpop3d_la_LIBADD=../maildir/libmaildir.la \
+	../rfc2045/librfc2045.la \
+	../rfc822/librfc822.la \
 	../numlib/libnumlib.la
 libpop3d_la_DEPENDENCIES=$(libpop3d_la_LIBADD)
 libpop3d_la_LDFLAGS=-static
 
 pop3d_SOURCES=pop3dserver.c pop3dcapa.c
 pop3d_DEPENDENCIES=libpop3d.la
-pop3d_LDADD=libpop3d.la @LDAUTH@ -lcourierauth
+pop3d_LDADD=libpop3d.la @LDAUTH@ -lcourierauth -lcourier-unicode
 
 HTML2TXT=links -dump -no-numbering
 
diff --git a/imap/fetch.c b/imap/fetch.c
index 1eb52dd..fd25265 100644
--- a/imap/fetch.c
+++ b/imap/fetch.c
@@ -287,8 +287,8 @@ int do_fetch(unsigned long n, int byuid, void *p)
 			open_err=1;
 			cannot_open_because=
 				" because it is a Unicode message and your"
-				" mail client did not enable Unicode support."
-				" Please use a mail client that supports"
+				" E-mail reader did not enable Unicode support."
+				" Please use an E-mail reader that supports"
 				" IMAP with UTF-8 (see"
 				" https://tools.ietf.org/html/rfc6855.html)";
 		}
diff --git a/imap/pop3dcapa.c b/imap/pop3dcapa.c
index 5d0eef4..ca9e12e 100644
--- a/imap/pop3dcapa.c
+++ b/imap/pop3dcapa.c
@@ -1,5 +1,5 @@
 /*
-** Copyright 1998 - 2008 Double Precision, Inc.
+** Copyright 1998 - 2018 Double Precision, Inc.
 ** See COPYING for distribution information.
 */
 
@@ -78,10 +78,13 @@ void pop3dcapa()
 		printf("SASL %s%s%s\r\n", p, *p && *external ? " ":"",
 		       *external ? "EXTERNAL":"");
 	}
-		
+
 	if (have_starttls())
 		printf("STLS\r\n");
 
-	printf("TOP\r\nUSER\r\nLOGIN-DELAY 10\r\nPIPELINING\r\nUIDL\r\nIMPLEMENTATION Courier Mail Server\r\n.\r\n");
+	printf("TOP\r\nUSER\r\nLOGIN-DELAY 10\r\n"
+	       "PIPELINING\r\nUIDL\r\n"
+	       "UTF8 USER\r\n"
+	       "IMPLEMENTATION Courier Mail Server\r\n.\r\n");
 	fflush(stdout);
 }
diff --git a/imap/pop3dserver.c b/imap/pop3dserver.c
index 13aced9..beca04a 100644
--- a/imap/pop3dserver.c
+++ b/imap/pop3dserver.c
@@ -1,5 +1,5 @@
 /*
-** Copyright 1998 - 2011 Double Precision, Inc.
+** Copyright 1998 - 2018 Double Precision, Inc.
 ** See COPYING for distribution information.
 **
 */
@@ -54,12 +54,20 @@
 #include	"maildir/maildirgetquota.h"
 #include	"maildir/maildircreate.h"
 #include	"maildir/loginexec.h"
+#include	"rfc2045/rfc2045.h"
 #include	"courierauth.h"
 
 #define POP3DLIST "courierpop3dsizelist"
+#define LISTVERSION 3
 
 extern void pop3dcapa();
 static void acctout(const char *disc);
+void rfc2045_error(const char *p)
+{
+	if (write(2, p, strlen(p)) < 0)
+		_exit(1);
+	_exit(0);
+}
 
 static const char *authaddr, *remoteip, *remoteport;
 
@@ -68,13 +76,14 @@ struct msglist {
 	char *filename;
 	int isdeleted;
 	int isnew;
+	int isutf8;
 	off_t size;
 
 	struct {
 		unsigned long uidv;
 		unsigned long n;
 	} uid;
-	} ;
+} ;
 
 static struct msglist *msglist_l;
 static struct msglist **msglist_a;
@@ -82,6 +91,7 @@ static unsigned msglist_cnt;
 
 static struct stat enomem_stat;
 static int enomem_1msg;
+int utf8_enabled;
 
 /*
 ** When a disk error occurs while saving an updated courierpop3dsizelist
@@ -109,7 +119,6 @@ static unsigned long bytes_sent_count=0;
 static unsigned long bytes_received_count=0;
 
 static unsigned long uidv=0;
-static int convert_v0=0;
 
 static time_t start_time;
 
@@ -120,29 +129,18 @@ static time_t start_time;
 
 static void calcsize(struct msglist *m)
 {
-FILE	*f=fopen(m->filename, "r");
-int	c, lastc;
+	FILE	*f=fopen(m->filename, "r");
+	struct rfc2045 *p=rfc2045_fromfp(f);
 
-	m->size=0;
-	if (f == 0)
-	{
-		perror("calcsize fopen");
-		return;
-	}
-	lastc='\n';
-	while ((c=getc(f)) >= 0)
-	{
-		if (c == '\n')	++m->size;
-		++m->size;
-		lastc=c;
-	}
-	if (lastc != '\n')	m->size += 2;
+	m->size=p->nlines + p->endpos;
 
-	if (ferror(f))
-	{
-		perror("calcsize ferror");
-		return;
-	}
+	clearerr(f);
+	if (m->size > 0 && fseek(f, -1, SEEK_SET) == 0 && getc(f) != '\n')
+		m->size+=2; /* We'll add an extra CRLF ourselves */
+
+	if (p->rfcviolation & RFC2045_ERR8BITHEADER)
+		m->isutf8=1;
+	rfc2045_free(p);
 	fclose(f);
 }
 
@@ -189,27 +187,15 @@ static struct msglist **readpop3dlist(unsigned long *uid)
 
 	uidv=time(NULL);
 
-	convert_v0=0;
-
 	if (fp == NULL ||
 	    fgets(linebuf, sizeof(linebuf)-1, fp) == NULL ||
 	    linebuf[0] != '/' || sscanf(linebuf+1, "%d %lu %lu", &vernum,
 					uid, &uidv)
-	    < 2 || (vernum != 1 && vernum != 2))
+	    < 2 || vernum != LISTVERSION)
 	{
-		if (fp == NULL)
-			convert_v0=1;
-
-		if (vernum == 0 && fp && fseek(fp, 0L, SEEK_SET) >= 0)
-		{
-			/* Old version 0 format courierpop3dsizelist file */
-		}
-		else
-		{
-			if (fp)
-				fclose(fp);
-			fp=NULL;
-		}
+		if (fp)
+			fclose(fp);
+		fp=NULL;
 	}
 
 	if (fp)
@@ -223,8 +209,6 @@ static struct msglist **readpop3dlist(unsigned long *uid)
 
 		while ((ch=getc(fp)) != EOF)
 		{
-			unsigned long sz;
-
 			if (ch != '\n')
 			{
 				if (n < sizeof(linebuf)-3)
@@ -234,10 +218,6 @@ static struct msglist **readpop3dlist(unsigned long *uid)
 			linebuf[n]=0;
 			n=0;
 
-			if (vernum == 0)
-				strcat(linebuf, " 0");
-			/* Convert version 0 to version 1 format - PRESTO! */
-
 			if ((p=strrchr(linebuf, ' ')) == NULL)
 				continue;
 			*p=0;
@@ -263,18 +243,16 @@ static struct msglist **readpop3dlist(unsigned long *uid)
 				exit(1);
 			}
 
-			switch (sscanf(p, "%lu %lu:%lu", &sz,
-				       &m->uid.n, &m->uid.uidv)) {
-			case 2:
-				m->uid.uidv=0;
-				/* FALLTHROUGH */
-			case 3:
-				m->size=sz;
+			if (sscanf(p, "%lu %lu:%lu:%d", &m->size,
+				   &m->uid.n, &m->uid.uidv,
+				   &m->isutf8) == 4)
+			{
 				m->next=list;
 				list=m;
 				++mcnt;
-				break;
-			default:
+			}
+			else
+			{
 				free(m->filename);
 				free(m);
 			}
@@ -316,7 +294,7 @@ static int savepop3dlist(struct msglist **a, size_t cnt,
 		return -1;
 	}
 
-	fprintf(fp, "/2 %lu %lu\n", uid, uidv);
+	fprintf(fp, "/%d %lu %lu\n", LISTVERSION, uid, uidv);
 
 	for (i=0; i<cnt; i++)
 	{
@@ -326,8 +304,8 @@ static int savepop3dlist(struct msglist **a, size_t cnt,
 		if ((q=strrchr(p, '/')) != NULL)
 			p=q+1;
 
-		fprintf(fp, "%s %lu %lu:%lu\n", p, (unsigned long)a[i]->size,
-			a[i]->uid.n, a[i]->uid.uidv);
+		fprintf(fp, "%s %lu %lu:%lu:%d\n", p, (unsigned long)a[i]->size,
+			a[i]->uid.n, a[i]->uid.uidv, a[i]->isutf8);
 	}
 
 	if (fflush(fp) || ferror(fp))
@@ -411,7 +389,7 @@ static int cmpmsgs(const void *a, const void *b)
 		++bp;
 	else
 		bp=bname;
- 
+
 	na=atol(ap);
 	nb=atol(bp);
 
@@ -480,6 +458,7 @@ static void sortmsgs()
 		{
 			msglist_a[i]->size=prev_list[n]->size;
 			msglist_a[i]->uid=prev_list[n]->uid;
+			msglist_a[i]->isutf8=prev_list[n]->isutf8;
 			n++;
 		}
 		else
@@ -487,12 +466,10 @@ static void sortmsgs()
 			msglist_a[i]->uid.n=nextuid++;
 			msglist_a[i]->uid.uidv=uidv;
 			msglist_a[i]->isnew=1;
-			if (convert_v0)
-				msglist_a[i]->uid.n=0;
 
 			calcsize(msglist_a[i]);
 			savesizes=1;
-		}      
+		}
 	}
 
 	if (prev_list[n])
@@ -662,12 +639,27 @@ char	buf[NUMBUFSIZE];
 
 static void do_retr(unsigned i, unsigned *lptr)
 {
-FILE	*f=fopen(msglist_a[i]->filename, "r");
-char	*p;
-int	c, lastc='\n';
-int	inheader=1;
-char	buf[NUMBUFSIZE];
-unsigned long *cntr;
+	FILE	*f;
+	char	*p;
+	int	c, lastc='\n';
+	int	inheader=1;
+	char	buf[NUMBUFSIZE];
+	unsigned long *cntr;
+
+	if (msglist_a[i]->isutf8 && !utf8_enabled)
+	{
+		printed(printf("-ERR Cannot open message %u because it is"
+			       " a Unicode message and your E-mail reader"
+			       " did not enable Unicode support. Please"
+			       " use an E-mail reader that supports POP3"
+			       " with UTF-8 (see"
+			       " https://tools.ietf.org/html/rfc6856.html)\r\n",
+			       i+1));
+		fflush(stdout);
+		return;
+	}
+
+	f=fopen(msglist_a[i]->filename, "r");
 
 	if (!f)
 	{
@@ -1045,6 +1037,14 @@ char	*p;
 			authaddr="nobody";
 	}
 
+	utf8_enabled=1; /* Until proven otherwise */
+
+	{
+		const char *utf8=getenv("UTF8");
+
+		if (utf8)
+			utf8_enabled=atoi(utf8);
+	}
 	if ((remoteip=getenv("TCPREMOTEIP")) == NULL)
 		remoteip="127.0.0.1";
 
@@ -1078,7 +1078,7 @@ char	*p;
 		fflush(stdout);
 		exit(1);
 	}
-	
+
 	maildir_loginexec();
 
 	if (auth_getoptionenvint("disablepop3"))
diff --git a/imap/pop3login.c b/imap/pop3login.c
index d541254..be96d0e 100644
--- a/imap/pop3login.c
+++ b/imap/pop3login.c
@@ -37,6 +37,7 @@ extern const char *pop3_externalauth();
 
 static const char *pop3d;
 static const char *defaultmaildir;
+int utf8_enabled=0;
 
 static const char *safe_getenv(const char *p)
 {
@@ -223,6 +224,10 @@ static int login_callback(struct authinfo *ainfo, void *dummy)
 				       ainfo->address);
 				putenv(p);
 
+				if (utf8_enabled)
+					putenv("UTF8=1");
+				else
+					putenv("UTF8=0");
 				alarm(0);
 				execl(pop3d, pop3d,
 				      ainfo->maildir ?
@@ -306,6 +311,14 @@ char *q ;
 				break;
 			}
 
+			if ( strcmp(p, "UTF8") == 0)
+			{
+				printf("+OK UTF8 enabled\r\n");
+				fflush(stdout);
+				utf8_enabled=1;
+				continue;
+			}
+
 			if ( strcmp(p, "USER") == 0)
 			{
 				if (tls_required())