courier-imap: implement literal8 element and APPEND UTF8 option.

4 files changed, 192 insertions, 70 deletions

diff --git a/imap/imapd.c b/imap/imapd.c
index cf31af4..78a689c 100644
--- a/imap/imapd.c
+++ b/imap/imapd.c
@@ -85,6 +85,7 @@
 #include	"maildir/maildirinfo.h"
 #include	"maildir/loginexec.h"
 #include	"rfc822/rfc822.h"
+#include	"rfc2045/rfc2045.h"
 
 #include	<courier-unicode.h>
 #include	"maildir/maildirkeywords.h"
@@ -711,19 +712,23 @@ static int store_mailbox(const char *tag, const char *mailbox,
 			 struct	imapflags *flags,
 			 struct libmail_kwMessage *keywords,
 			 time_t	timestamp,
-			 unsigned long nbytes,
+			 struct imaptoken *curtoken,
 			 unsigned long *new_uidv,
 			 unsigned long *new_uid)
 {
-char	*tmpname;
-char	*newname;
-char	*p;
-char    *e;
-FILE	*fp;
-unsigned long n;
-static const char nowrite[]=" NO [ALERT] Cannot create message - no write permission or out of disk space.\r\n";
-int	lastnl;
-int     rb;
+	unsigned long nbytes=curtoken->tokennum;
+	char	*tmpname;
+	char	*newname;
+	char	*p;
+	char    *e;
+	FILE	*fp;
+	unsigned long n;
+	static const char nowrite[]=" NO [ALERT] Cannot create message - no write permission or out of disk space.\r\n";
+	int	lastnl;
+	int     rb;
+	int	errflag;
+	struct rfc2045 *rfc2045_parser;
+	const char *errmsg=nowrite;
 
 	fp=maildir_mkfilename(mailbox, flags, 0, &tmpname, &newname);
 
@@ -741,6 +746,8 @@ int     rb;
 	current_temp_fd=fileno(fp);
 	current_temp_fn=tmpname;
 
+	rfc2045_parser=rfc2045_alloc();
+
 	while (nbytes)
 	{
 		read_string(&p, &n, nbytes);
@@ -757,31 +764,53 @@ int     rb;
 			}
 			else if (e)
 			{
+				rfc2045_parse(rfc2045_parser, p, e-p);
 				rb = fwrite(p, 1, e-p, fp);
 			}
 			else
 			{
+				rfc2045_parse(rfc2045_parser, p, n);
 				rb = fwrite(p, 1, n, fp);
 			}
 			n -= rb;
 			p += rb;
 		}
 	}
-	if (!lastnl) putc('\n', fp);
+	if (!lastnl)
+	{
+		putc('\n', fp);
+		rfc2045_parse(rfc2045_parser, "\n", 1);
+	}
 
 	current_temp_fd=-1;
 	current_temp_fn=NULL;
+	errflag=0;
 
 	if (fflush(fp) || ferror(fp))
 	{
 		fprintf(stderr,
                         "ERR: error storing a message, user=%s, errno=%d\n",
                                 getenv("AUTHENTICATED"), errno);
+		errflag=1;
+	}
+
+	if ((rfc2045_parser->rfcviolation & RFC2045_ERR8BITHEADER) &&
+	    curtoken->tokentype != IT_LITERAL8_STRING_START)
+	{
+		errmsg=" NO [ALERT] Your IMAP client does not appear to "
+			"correctly implement Unicode messages, "
+			"see https://tools.ietf.org/html/rfc6855.html\r\n";
+		errflag=1;
+	}
+
+	rfc2045_free(rfc2045_parser);
 
+	if (errflag)
+	{
 		fclose(fp);
 		unlink(tmpname);
 		writes(tag);
-		writes(nowrite);
+		writes(errmsg);
 		free(tmpname);
 		free(newname);
 		return (-1);
@@ -3881,6 +3910,7 @@ static int append(const char *tag, const char *mailbox, const char *path)
 	unsigned long new_uidv, new_uid;
 	char access_rights[8];
 	struct imaptoken *curtoken;
+	int need_rparen;
 
 	if (access(path, 0))
 	{
@@ -3960,9 +3990,35 @@ static int append(const char *tag, const char *mailbox, const char *path)
 	else if (curtoken->tokentype == IT_NIL)
 		curtoken=nexttoken_noparseliteral();
 
-	if (curtoken->tokentype != IT_LITERAL_STRING_START)
+	need_rparen=0;
+
+	if (enabled_utf8 && curtoken->tokentype == IT_ATOM &&
+	    strcmp(curtoken->tokenbuf, "UTF8") == 0)
+	{
+		curtoken=nexttoken();
+		if (curtoken->tokentype != IT_LPAREN)
+		{
+			libmail_kwmDestroy(keywords);
+			return (-1);
+		}
+
+		curtoken=nexttoken_noparseliteral();
+		if (curtoken->tokentype != IT_LITERAL8_STRING_START)
+		{
+			libmail_kwmDestroy(keywords);
+
+			/* Don't break the protocol level */
+			convert_literal_tokens(curtoken);
+			return (-1);
+		}
+		need_rparen=1;
+	}
+	else if (curtoken->tokentype != IT_LITERAL_STRING_START)
 	{
 		libmail_kwmDestroy(keywords);
+
+		/* Don't break the protocol level */
+		convert_literal_tokens(curtoken);
 		return (-1);
 	}
 
@@ -3972,7 +4028,7 @@ static int append(const char *tag, const char *mailbox, const char *path)
 			  acl_flags_adjust(access_rights, &flags)
 			  ? NULL:keywords,
 			  timestamp,
-			  curtoken->tokennum, &new_uidv, &new_uid))
+			  curtoken, &new_uidv, &new_uid))
 	{
 		libmail_kwmDestroy(keywords);
 		unread('\n');
@@ -3980,6 +4036,14 @@ static int append(const char *tag, const char *mailbox, const char *path)
 	}
 	libmail_kwmDestroy(keywords);
 
+	if (need_rparen)
+	{
+		if (nexttoken()->tokentype != IT_RPAREN)
+		{
+			return (-1);
+		}
+	}
+
 	if (nexttoken()->tokentype != IT_EOL)
 	{
 		return (-1);
@@ -4185,6 +4249,13 @@ static int validate_charset(const char *tag, char **charset)
 	if (*charset == NULL)
 		*charset=my_strdup("UTF-8");
 
+	if (enabled_utf8 && strcmp(*charset, "UTF-8"))
+	{
+		writes(tag);
+		writes(" BAD [BADCHARSET] Only UTF-8 charset is valid after enabling RFC 6855 support\r\n");
+		return (-1);
+	}
+
 	conv=unicode_convert_tou_init(*charset, &ucptr, &ucsize, 1);
 
 	if (!conv)
diff --git a/imap/imaptoken.c b/imap/imaptoken.c
index 82dda7c..a11310a 100644
--- a/imap/imaptoken.c
+++ b/imap/imaptoken.c
@@ -229,8 +229,54 @@ static int ignore_output_func(const char *ptr, size_t cnt, void *ignore)
 	return 0;
 }
 
+static struct imaptoken *do_readtoken_nolog(int touc);
+
 static struct imaptoken *do_readtoken(int touc)
 {
+	struct imaptoken *tok=do_readtoken_nolog(touc);
+
+	if (debugfile)
+	{
+		char	*p=0;
+
+		fprintf(debugfile, "READ: ");
+		switch (tok->tokentype) {
+		case IT_ATOM:
+			p=curtoken.tokenbuf; fprintf(debugfile, "ATOM"); break;
+		case IT_NUMBER:
+			p=curtoken.tokenbuf; fprintf(debugfile, "NUMBER"); break;
+		case IT_QUOTED_STRING:
+			p=curtoken.tokenbuf; fprintf(debugfile, "QUOTED_STRING"); break;
+		case IT_LPAREN:
+			fprintf(debugfile, "LPAREN"); break;
+		case IT_RPAREN:
+			fprintf(debugfile, "RPAREN"); break;
+		case IT_NIL:
+			fprintf(debugfile, "NIL"); break;
+		case IT_ERROR:
+			fprintf(debugfile, "ERROR"); break;
+		case IT_EOL:
+			fprintf(debugfile, "EOL"); break;
+		case IT_LBRACKET:
+			fprintf(debugfile, "LBRACKET"); break;
+		case IT_RBRACKET:
+			fprintf(debugfile, "RBRACKET"); break;
+		case IT_LITERAL_STRING_START:
+			fprintf(debugfile, "{%lu}", tok->tokennum); break;
+		case IT_LITERAL8_STRING_START:
+			fprintf(debugfile, "~{%lu}", tok->tokennum); break;
+		}
+
+		if (p)
+			fprintf(debugfile, ": %s", p);
+		fprintf(debugfile, "\n");
+		fflush(debugfile);
+	}
+	return tok;
+}
+
+static struct imaptoken *do_readtoken_nolog(int touc)
+{
 int	c=0;
 unsigned l;
 
@@ -289,11 +335,9 @@ unsigned l;
 				curtoken.tokentype=IT_ERROR;
 				return (&curtoken);
 			}
-			if (l >= 8192)
+			if (l >= BUFSIZ)
 			{
-				fprintf(stderr, "ERR: Quoted string literal "
-					"overflow, ip=[%s]\n",
-					getenv("TCPREMOTEIP"));
+				writes("* NO [ALERT] IMAP command too long.\r\n");
 				curtoken.tokentype=IT_ERROR;
 			}
 			if (c & 0x80)
@@ -331,6 +375,22 @@ unsigned l;
 		return (&curtoken);
 	}
 
+	/*
+	** Parse LITERAL or LITERAL8
+	*/
+	curtoken.tokentype=IT_LITERAL_STRING_START;
+
+	if (c == '~')
+	{
+		c=READ();
+		if (c != '{')
+		{
+			curtoken.tokentype=IT_ERROR;
+			return (&curtoken);
+		}
+		curtoken.tokentype=IT_LITERAL8_STRING_START;
+	}
+
 	if (c == '{')
 	{
 		curtoken.tokennum=0;
@@ -354,7 +414,6 @@ unsigned l;
 			curtoken.tokentype=IT_ERROR;
 			return (&curtoken);
 		}
-		curtoken.tokentype=IT_LITERAL_STRING_START;
 		return (&curtoken);
 	}
 
@@ -383,6 +442,7 @@ unsigned l;
 	       && !isspace((int)(unsigned char)c)
 	       && (((unsigned char)c) & 0x80) == 0
 	       && c != '\\' && c != '"' && c != LPAREN_CHAR && c != RPAREN_CHAR
+	       && c != '~'
 	       && c != '{' && c != '}' && c != LBRACKET_CHAR && c != RBRACKET_CHAR)
 	{
 		curtoken.tokentype=IT_ATOM;
@@ -413,65 +473,52 @@ unsigned l;
 
 static struct imaptoken *readtoken(int touc)
 {
-struct imaptoken *tok=do_readtoken(touc);
+	struct imaptoken *tok=do_readtoken(touc);
 
-	if (tok->tokentype == IT_LITERAL_STRING_START)
-	{
-	unsigned long nbytes=curtoken.tokennum;
+	convert_literal_tokens(tok);
+	return (tok);
+}
 
-		if (nbytes > 8192)
-		{
-			writes("* NO [ALERT] IMAP command too long.\r\n");
-			tok->tokentype=IT_ERROR;
-		}
-		else
-		{
-		unsigned long i;
+/*
+** If a token is a LITERAL, read it and replace it with a QUOTED_STRING.
+*/
 
-			writes("+ OK\r\n");
-			writeflush();
-			alloc_tokenbuf(nbytes+1);
-			for (i=0; i<nbytes; i++)
-				tok->tokenbuf[i]= READ();
-			tok->tokenbuf[i]=0;
-			tok->tokentype=IT_QUOTED_STRING;
-		}
-	}
+void convert_literal_tokens(struct imaptoken *tok)
+{
+	unsigned long nbytes;
 
-	if (debugfile)
+	if (tok->tokentype != IT_LITERAL_STRING_START &&
+	    tok->tokentype != IT_LITERAL8_STRING_START)
+		return;
+
+	nbytes=curtoken.tokennum;
+
+	if (nbytes > BUFSIZ)
 	{
-	char	*p=0;
+		writes("* NO [ALERT] IMAP command too long.\r\n");
+		tok->tokentype=IT_ERROR;
+	}
+	else
+	{
+		unsigned long i;
 
-		fprintf(debugfile, "READ: ");
-		switch (tok->tokentype) {
-		case IT_ATOM:
-			p=curtoken.tokenbuf; fprintf(debugfile, "ATOM"); break;
-		case IT_NUMBER:
-			p=curtoken.tokenbuf; fprintf(debugfile, "NUMBER"); break;
-		case IT_QUOTED_STRING:
-			p=curtoken.tokenbuf; fprintf(debugfile, "QUOTED_STRING"); break;
-		case IT_LPAREN:
-			fprintf(debugfile, "LPAREN"); break;
-		case IT_RPAREN:
-			fprintf(debugfile, "RPAREN"); break;
-		case IT_NIL:
-			fprintf(debugfile, "NIL"); break;
-		case IT_ERROR:
-			fprintf(debugfile, "ERROR"); break;
-		case IT_EOL:
-			fprintf(debugfile, "EOL"); break;
-		case IT_LBRACKET:
-			fprintf(debugfile, "LBRACKET"); break;
-		case IT_RBRACKET:
-			fprintf(debugfile, "RBRACKET"); break;
-		}
+		/*
 
-		if (p)
-			fprintf(debugfile, ": %s", p);
-		fprintf(debugfile, "\n");
-		fflush(debugfile);
+		  RFC4466: In addition, the non-terminal "literal8" defined
+		  in [BINARY] got extended to allow for non-synchronizing
+		  literals if both [BINARY] and [LITERAL+] extensions are
+		  supported by the server.
+
+		*/
+
+		writes("+ OK\r\n");
+		writeflush();
+		alloc_tokenbuf(nbytes+1);
+		for (i=0; i<nbytes; i++)
+			tok->tokenbuf[i]= READ();
+		tok->tokenbuf[i]=0;
+		tok->tokentype=IT_QUOTED_STRING;
 	}
-	return (tok);
 }
 
 struct imaptoken *nexttoken(void)
diff --git a/imap/imaptoken.h b/imap/imaptoken.h
index 65f8119..0fdafc9 100644
--- a/imap/imaptoken.h
+++ b/imap/imaptoken.h
@@ -29,6 +29,7 @@ struct imaptoken {
 #define	IT_EOL			8
 #define	IT_LBRACKET		9
 #define	IT_RBRACKET		10
+#define	IT_LITERAL8_STRING_START 11
 
 struct imaptoken *nexttoken(void);
 struct imaptoken *currenttoken(void);
@@ -36,6 +37,7 @@ struct imaptoken *nexttoken_nouc(void);
 struct imaptoken *nexttoken_noparseliteral(void);
 struct imaptoken *nexttoken_okbracket(void);
 struct imaptoken *nexttoken_nouc_okbracket(void);
+void convert_literal_tokens(struct imaptoken *tok);
 
 int ismsgset(struct imaptoken *);
 	/* See if this token is a syntactically valid message set */
diff --git a/rfc2045/rfc2045.h b/rfc2045/rfc2045.h
index 90c2b56..87773d9 100644
--- a/rfc2045/rfc2045.h
+++ b/rfc2045/rfc2045.h
@@ -60,6 +60,8 @@ struct rfc2045 {
 	unsigned rfcviolation;	/* Boo-boos */
 
 #define	RFC2045_ERR8BITHEADER	1	/* 8 bit characters in headers */
+	/* But this is now OK, in UTF8 mode */
+
 #define	RFC2045_ERR8BITCONTENT	2	/* 8 bit contents, but no 8bit
 					content-transfer-encoding */
 #define	RFC2045_ERR2COMPLEX	4	/* Too many nested contents */