Update sqwebmail to talk to UTF-8-able maildrop.

Writes out mail filtering scripts in UTF-8.

maildir_filter_appendrule() and maildir_filter_ruleupdate() take an additional
parameter, a character set, and convert the header name, value, and rule name
parameters from the given character set to UTF-8.

sqwebmail passes the current character set, when adding or updating maildrop
rules.

8 files changed, 165 insertions, 56 deletions

diff --git a/maildir/Makefile.am b/maildir/Makefile.am
index a93a17a..cabd14b 100644
--- a/maildir/Makefile.am
+++ b/maildir/Makefile.am
@@ -66,8 +66,8 @@ maildirmake_LDADD=libmaildir.la ../unicode/libunicode.la \
 maildirmake_LDFLAGS=-static
 
 testmaildirfilter_SOURCES=maildirfiltertypelist.h testmaildirfilter.c
-testmaildirfilter_DEPENDENCIES=libmaildir.la ../numlib/libnumlib.la
-testmaildirfilter_LDADD=libmaildir.la ../numlib/libnumlib.la @LIBPCRE@
+testmaildirfilter_DEPENDENCIES=libmaildir.la ../numlib/libnumlib.la ../unicode/libunicode.la
+testmaildirfilter_LDADD=libmaildir.la ../numlib/libnumlib.la ../unicode/libunicode.la @LIBPCRE@
 testmaildirfilter_LDFLAGS=-static
 
 maildirkwtest_SOURCES=maildirkwtest.c
diff --git a/maildir/maildirfilter.c b/maildir/maildirfilter.c
index af8b353..d994998 100644
--- a/maildir/maildirfilter.c
+++ b/maildir/maildirfilter.c
@@ -11,6 +11,7 @@
 
 #include	"autoresponse.h"
 #include	"numlib/numlib.h"
+#include	"unicode/unicode.h"
 #include	<stdlib.h>
 #include	<string.h>
 #include	<stdio.h>
@@ -50,6 +51,7 @@ struct maildirfilterrule *maildir_filter_appendrule(struct maildirfilter *r,
 					const char *value,
 					const char *folder,
 					const char *fromhdr,
+					const char *charset,
 					int *errcode)
 {
 struct maildirfilterrule *p=malloc(sizeof(struct maildirfilterrule));
@@ -66,7 +68,8 @@ struct maildirfilterrule *p=malloc(sizeof(struct maildirfilterrule));
 	r->last=p;
 
 	if (maildir_filter_ruleupdate(r, p, name, type, flags,
-			  header, value, folder, fromhdr, errcode))
+				      header, value, folder, fromhdr, charset,
+				      errcode))
 	{
 		maildir_filter_ruledel(r, p);
 		return (0);
@@ -74,16 +77,82 @@ struct maildirfilterrule *p=malloc(sizeof(struct maildirfilterrule));
 	return (p);
 }
 
+static int maildir_filter_ruleupdate_utf8(struct maildirfilter *r,
+					  struct maildirfilterrule *p,
+					  const char *name,
+					  enum maildirfiltertype type,
+					  int flags,
+					  const char *header,
+					  const char *value,
+					  const char *folder,
+					  const char *fromhdr,
+					  int *errcode);
+
 int maildir_filter_ruleupdate(struct maildirfilter *r,
-		  struct maildirfilterrule *p,
-		  const char *name,
-		  enum maildirfiltertype type,
-		  int flags,
-		  const char *header,
-		  const char *value,
-		  const char *folder,
-		  const char *fromhdr,
-		  int *errcode)
+			      struct maildirfilterrule *p,
+			      const char *name,
+			      enum maildirfiltertype type,
+			      int flags,
+			      const char *header,
+			      const char *value,
+			      const char *folder,
+			      const char *fromhdr,
+			      const char *charset,
+			      int *errcode)
+{
+	char *name_utf8;
+	char *header_utf8;
+	char *value_utf8;
+	int rc;
+
+	name_utf8=libmail_u_convert_toutf8(name ? name:"", charset, NULL);
+
+	if (!name_utf8)
+	{
+		*errcode=MF_ERR_BADRULENAME;
+		return -1;
+	}
+
+	header_utf8=libmail_u_convert_toutf8(header ? header:"", charset, NULL);
+
+	if (!header_utf8)
+	{
+		free(name_utf8);
+		*errcode=MF_ERR_BADRULEHEADER;
+		return -1;
+	}
+
+	value_utf8=libmail_u_convert_toutf8(value ? value:"", charset, NULL);
+
+	if (!value_utf8)
+	{
+		free(name_utf8);
+		free(header_utf8);
+		*errcode=MF_ERR_BADRULEVALUE;
+		return -1;
+	}
+	rc=maildir_filter_ruleupdate_utf8(r, p, name_utf8, type, flags,
+					  header_utf8,
+					  value_utf8,
+					  folder,
+					  fromhdr,
+					  errcode);
+	free(name_utf8);
+	free(value_utf8);
+	free(header_utf8);
+	return rc;
+}
+
+static int maildir_filter_ruleupdate_utf8(struct maildirfilter *r,
+					  struct maildirfilterrule *p,
+					  const char *name,
+					  enum maildirfiltertype type,
+					  int flags,
+					  const char *header,
+					  const char *value,
+					  const char *folder,
+					  const char *fromhdr,
+					  int *errcode)
 {
 	const char *c;
 	struct maildirfilterrule *pom;
@@ -96,7 +165,7 @@ int maildir_filter_ruleupdate(struct maildirfilter *r,
 
 	/* rule name: may not contain quotes or control characters. */
 	*errcode=MF_ERR_BADRULENAME;
-	if (!name || !*name || strlen(name) > 200)
+	if (!*name || strlen(name) > 200)
 		return (-1);
 
 	for (c=name; *c; c++)
@@ -108,7 +177,7 @@ int maildir_filter_ruleupdate(struct maildirfilter *r,
 	*errcode=MF_ERR_EXISTS;
 	
 	for (pom=r->first; pom->next; pom=pom->next) {
-	    if (p!=pom && !strcmp(name, pom->rulename))
+	    if (p!=pom && !strcmp(name, pom->rulename_utf8))
 		return (-1);
 	}
 
@@ -134,8 +203,8 @@ int maildir_filter_ruleupdate(struct maildirfilter *r,
 	*errcode=MF_ERR_BADRULEHEADER;
 
 	c=header;
-	if (c && strlen(c) > 200)	return (-1);
-	if (c == 0 || *c == 0)
+	if (strlen(c) > 200)	return (-1);
+	if (*c == 0)
 	{
 		switch (type)	{
 		case hasrecipient:
@@ -159,8 +228,9 @@ int maildir_filter_ruleupdate(struct maildirfilter *r,
 	else for ( ; *c; c++)
 	{
 		/* no control characters */
-		if (*c <= ' ' || *c == MDIRSEP[0] || *c >= 127 || *c == '\'' ||
-			*c == '\\' || *c == '"' || *c == '`' || *c == '/')
+		if ((unsigned char)*c <= ' ' || *c == MDIRSEP[0] ||
+		    *c == '\'' ||
+		    *c == '\\' || *c == '"' || *c == '`' || *c == '/')
 			return (-1);
 	}
 
@@ -169,8 +239,8 @@ int maildir_filter_ruleupdate(struct maildirfilter *r,
 	*errcode=MF_ERR_BADRULEVALUE;
 
 	c=value;
-	if (c && strlen(c) > 200)	return (-1);
-	if (c == 0 || *c == 0)
+	if (strlen(c) > 200)	return (-1);
+	if (*c == 0)
 	{
 		switch (type)	{
 		case mimemultipart:
@@ -270,7 +340,7 @@ int maildir_filter_ruleupdate(struct maildirfilter *r,
 				const char *errptr;
 				int errindex;
 
-				pcre *p=pcre_compile(value, 0,
+				pcre *p=pcre_compile(value, PCRE_UTF8,
 						     &errptr,
 						     &errindex,
 						     0);
@@ -349,13 +419,13 @@ int maildir_filter_ruleupdate(struct maildirfilter *r,
 
 	*errcode=MF_ERR_INTERNAL;
 
-	if (p->rulename)	free(p->rulename);
-	if ((p->rulename=strdup(name)) == 0)	return (-1);
+	if (p->rulename_utf8)	free(p->rulename_utf8);
+	if ((p->rulename_utf8=strdup(name)) == 0)	return (-1);
 	p->type=type;
-	if (p->fieldname)	free(p->fieldname);
-	if ((p->fieldname=strdup(header ? header:"")) == 0)	return (-1);
-	if (p->fieldvalue)	free(p->fieldvalue);
-	if ((p->fieldvalue=strdup(value ? value:"")) == 0)	return (-1);
+	if (p->fieldname_utf8)	free(p->fieldname_utf8);
+	if ((p->fieldname_utf8=strdup(header ? header:"")) == 0)	return (-1);
+	if (p->fieldvalue_utf8)	free(p->fieldvalue_utf8);
+	if ((p->fieldvalue_utf8=strdup(value ? value:"")) == 0)	return (-1);
 	if (p->tofolder)	free(p->tofolder);
 	if ((p->tofolder=malloc(strlen(folder)+1)) == 0)	return (-1);
 	strcpy(p->tofolder, folder);
@@ -376,9 +446,9 @@ void maildir_filter_ruledel(struct maildirfilter *r, struct maildirfilterrule *p
 	if (p->next)	p->next->prev=p->prev;
 	else		r->last=p->prev;
 
-	if (p->rulename)	free(p->rulename);
-	if (p->fieldname)	free(p->fieldname);
-	if (p->fieldvalue)	free(p->fieldvalue);
+	if (p->rulename_utf8)	free(p->rulename_utf8);
+	if (p->fieldname_utf8)	free(p->fieldname_utf8);
+	if (p->fieldvalue_utf8)	free(p->fieldvalue_utf8);
 	if (p->tofolder)	free(p->tofolder);
 	if (p->fromhdr)		free(p->fromhdr);
 	free(p);
@@ -429,7 +499,8 @@ static void print_pattern(FILE *f, int flags, const char *v)
 
 	while (*v)
 	{
-		if (!isalnum((int)(unsigned char)*v))
+		if (((int)(unsigned char)*v) <= 0x80 &&
+		    !isalnum((int)(unsigned char)*v))
 			putc('\\', f);
 		putc((int)(unsigned char)*v, f);
 		++v;
@@ -461,8 +532,8 @@ struct maildirfilterrule *p;
 
 	for (p=r->first; p; p=p->next)
 	{
-	const char *fieldname=p->fieldname ? p->fieldname:"";
-	const char *fieldvalue=p->fieldvalue ? p->fieldvalue:"";
+	const char *fieldname=p->fieldname_utf8 ? p->fieldname_utf8:"";
+	const char *fieldvalue=p->fieldvalue_utf8 ? p->fieldvalue_utf8:"";
 	const char *tofolder=p->tofolder ? p->tofolder:"";
 
 		fprintf(f, "##Op:%s\n",
@@ -484,7 +555,7 @@ struct maildirfilterrule *p;
 		if (p->flags & MFR_CONTINUE)
 			fprintf(f, "##Continue\n");
 
-		fprintf(f, "##Name:%s\n\n", p->rulename ? p->rulename:"");
+		fprintf(f, "##Name:%s\n\n", p->rulename_utf8 ? p->rulename_utf8:"");
 
 		fprintf(f, "\nif (");
 
@@ -786,7 +857,8 @@ int	flags;
 			maildir_filter_appendrule(r, p, new_type, flags,
 						  new_header,
 						  new_value, new_folder,
-						  new_autoreplyfrom, &dummy);
+						  new_autoreplyfrom,
+						  "utf-8", &dummy);
 			new_type=contains;
 			new_header[0]=0;
 			new_value[0]=0;
diff --git a/maildir/maildirfilter.h b/maildir/maildirfilter.h
index 59cdee7..3d4bfdf 100644
--- a/maildir/maildirfilter.h
+++ b/maildir/maildirfilter.h
@@ -26,7 +26,7 @@ enum maildirfiltertype {
 
 struct maildirfilterrule {
 	struct maildirfilterrule *next, *prev;
-	char *rulename;
+	char *rulename_utf8;
 	enum maildirfiltertype type;
 	int flags;
 
@@ -37,8 +37,8 @@ struct maildirfilterrule {
 #define	MFR_CONTINUE	4	/* Continue filtering (cc instead of to) */
 #define MFR_PLAINSTRING	8	/* Pattern is a plain string, not a regex */
 
-	char *fieldname;	/* Match this header */
-	char *fieldvalue;	/* Match/search value */
+	char *fieldname_utf8;	/* Match this header */
+	char *fieldvalue_utf8;	/* Match/search value */
 	char *tofolder;		/* Destination folder, fwd address, err msg */
 	char *fromhdr;		/* From: header on autoreplies. */
 	} ;
@@ -62,6 +62,7 @@ struct maildirfilterrule *maildir_filter_appendrule(struct maildirfilter *r,
 					const char *value,
 					const char *folder,
 					const char *fromhdr,
+					const char *rulecharset,
 					int *errcode);	/* Append a new rule */
 
 int maildir_filter_setautoreplyfrom(struct maildirfilter *, const char *);
@@ -85,6 +86,7 @@ int maildir_filter_ruleupdate(struct maildirfilter *, struct maildirfilterrule *
 		  const char *,
 		  const char *,
 		  const char *,
+		  const char *,
 		  int *);
 
 #define	maildir_filter_freerules(r)	do { \
diff --git a/maildir/testmaildirfilter.c b/maildir/testmaildirfilter.c
index 6826208..c50d358 100644
--- a/maildir/testmaildirfilter.c
+++ b/maildir/testmaildirfilter.c
@@ -1,5 +1,5 @@
 /*
-** Copyright 2000 Double Precision, Inc.
+** Copyright 2000-2013 Double Precision, Inc.
 ** See COPYING for distribution information.
 */
 
@@ -8,7 +8,7 @@
 #include	<stdio.h>
 #include	<string.h>
 #include	<unistd.h>
-
+#include	<stdlib.h>
 
 int main(int argc, char **argv)
 {
@@ -21,6 +21,7 @@ int errcode, i;
 struct maildirfilter mf;
 struct maildirfilterrule *r;
 int flags=0;
+const char *charset=getenv("CHARSET");
 
 	if (argc < 6)
 	{
@@ -64,8 +65,12 @@ int flags=0;
 		return (1);
 	}
 
+	if (!charset)
+		charset="utf-8";
+
 	r=maildir_filter_appendrule(&mf, name, type, flags,
-		header, value, folder, "", &errcode);
+				    header, value, folder, "", charset,
+				    &errcode);
 
 	if (!r)
 	{
@@ -81,6 +86,6 @@ int flags=0;
 		return (1);
 	}
 	rename("testrules2", "testrules");
-	printf("Added %s\n", name);
+	printf("Added %s\n", r->rulename_utf8);
 	return (0);
 }
diff --git a/maildir/testsuite2 b/maildir/testsuite2
index 398c36e..bcc3dbd 100644
--- a/maildir/testsuite2
+++ b/maildir/testsuite2
@@ -31,5 +31,6 @@ rm -f testrules
 ./testmaildirfilter "rule12" "-islargerthan" "" "100000" "*bounce"
 ./testmaildirfilter "ruleauto1" "islargerthan" "" 0 "+dummy dsn=1 days=7"
 ./testmaildirfilter "ruleauto2" "islargerthan" "" 0 "+dummy2"
+CHARSET=iso-8859-1 ./testmaildirfilter "`echo -n utf8rule | tr 'e' '\351'`" "contains" "`echo -n subject | tr 'e' '\351'`" "`echo -n test | tr 'e' '\351'`" "INBOX"
 cat testrules | sed 's:/.*mailbot:mailbot:'
 rm -f testrules
diff --git a/maildir/testsuite2.txt b/maildir/testsuite2.txt
index d380ac3..c941924 100644
--- a/maildir/testsuite2.txt
+++ b/maildir/testsuite2.txt
@@ -27,6 +27,7 @@ Error appending err14: 4
 Added rule12
 Added ruleauto1
 Added ruleauto2
+Added utf8rulé
 #MFMAILDROP=2
 #
 # DO NOT EDIT THIS FILE.  This is an automatically generated filter.
@@ -239,4 +240,17 @@ if (($SIZE > 0))
    `mailbot -A "X-Sender: $FROM" -A "From: $AUTOREPLYFROM" -m "Maildir/autoresponses/dummy2" $SENDMAIL -t -f ""`
 }
 
+##Op:contains
+##Header:subjéct
+##Value:tést
+##Folder:.
+##From:
+##Name:utf8rulé
+
+
+if ((/^subjéct:.*tést/))
+{
+   to "Maildir/."
+}
+
 to "Maildir/."
diff --git a/sqwebmail/ChangeLog b/sqwebmail/ChangeLog
index 7c6fa94..0a211aa 100644
--- a/sqwebmail/ChangeLog
+++ b/sqwebmail/ChangeLog
@@ -1,3 +1,8 @@
+2013-08-28  Sam Varshavchik  <mrsam@courier-mta.com>
+
+	* sqwebmail: update the maildrop rule generator to support UTF-8-able
+	maildrop.
+
 5.6.1
 
 2013-02-20  Sam Varshavchik  <mrsam@courier-mta.com>
diff --git a/sqwebmail/mailfilter.c b/sqwebmail/mailfilter.c
index c228061..c58c593 100644
--- a/sqwebmail/mailfilter.c
+++ b/sqwebmail/mailfilter.c
@@ -22,6 +22,7 @@
 
 extern void list_folder(const char *);
 extern void output_attrencoded(const char *);
+extern const char *sqwebmail_content_charset;
 
 static const char *internal_err=0;
 
@@ -67,9 +68,15 @@ unsigned cnt;
 
 	for (cnt=0, r=mf.first; r; r=r->next, ++cnt)
 	{
+		char *p=libmail_u_convert_fromutf8(r->rulename_utf8,
+						   sqwebmail_content_charset,
+						   NULL);
+
 		printf("<option value=\"%u\">", cnt);
-		output_attrencoded(r->rulename);
+		output_attrencoded(p ? p:r->rulename_utf8);
 		printf("</option>");
+		if (p)
+			free(p);
 	}
 	maildir_filter_freerules(&mf);
 }
@@ -200,25 +207,28 @@ struct maildirfilterrule *r;
 				r->flags & MFR_DOESNOT ? "notcontains":"contains":"");
 
 		if (namebuf)	free(namebuf);
-		p=r->rulename ? r->rulename:"";
-		namebuf=malloc(strlen(p)+1);
+		p=r->rulename_utf8 ? r->rulename_utf8:"";
+
+		namebuf=libmail_u_convert_fromutf8(p, sqwebmail_content_charset,
+						   NULL);
+
 		if (!namebuf)	enomem();
-		strcpy(namebuf, p);
 		cgi_put("rulename", namebuf);
 
-		p=r->fieldname ? r->fieldname:"";
+		p=r->fieldname_utf8 ? r->fieldname_utf8:"";
 		if (r->type != startswith &&
 			r->type != endswith &&
 			r->type != contains)	p="";
 		if (r->flags & MFR_BODY)	p="";
 
 		if (headernamebuf)	free(headernamebuf);
-		headernamebuf=malloc(strlen(p)+1);
+		headernamebuf=libmail_u_convert_fromutf8(p,
+							 sqwebmail_content_charset,
+							 NULL);
 		if (!headernamebuf)	enomem();
-		strcpy(headernamebuf, p);
 		cgi_put("headername", headernamebuf);
 
-		p=r->fieldvalue ? r->fieldvalue:"";
+		p=r->fieldvalue_utf8 ? r->fieldvalue_utf8:"";
 		if (r->type != startswith &&
 			r->type != endswith &&
 			r->type != contains &&
@@ -231,11 +241,11 @@ struct maildirfilterrule *r;
 
 		if (headervaluebuf)	free(headervaluebuf);
 
-
-
-		headervaluebuf=malloc(strlen(p)+1);
+		headervaluebuf=
+			libmail_u_convert_fromutf8(p,
+						   sqwebmail_content_charset,
+						   NULL);
 		if (!headervaluebuf)	enomem();
-		strcpy(headervaluebuf, p);
 
 		cgi_put("hasrecipientaddr", "");
 		cgi_put("headervalue", "");
@@ -583,9 +593,9 @@ const char *autoreply_from="";
 
 	if (!r)
 		r=maildir_filter_appendrule(&mf, rulename, type, flags, fieldname_cpy,
-			fieldvalue, tofolder, autoreply_from, &err_num);
+					    fieldvalue, tofolder, autoreply_from, sqwebmail_content_charset, &err_num);
 	else if (maildir_filter_ruleupdate(&mf, r, rulename, type, flags, fieldname_cpy,
-			fieldvalue, tofolder, autoreply_from, &err_num))
+					   fieldvalue, tofolder, autoreply_from, sqwebmail_content_charset, &err_num))
 		r=0;
 	free(tofolder);
 	if (fieldname_cpy)