1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
|
;;; utf7.el --- UTF-7 encoding/decoding for Emacs -*- lexical-binding:t -*-
;; Copyright (C) 1999-2019 Free Software Foundation, Inc.
;; Author: Jon K Hellan <hellan@acm.org>
;; Keywords: mail
;; This file is part of GNU Emacs.
;; GNU Emacs is free software: you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.
;; GNU Emacs is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;; You should have received a copy of the GNU General Public License
;; along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>.
;;; Commentary:
;; UTF-7 - A Mail-Safe Transformation Format of Unicode - RFC 2152
;; This is a transformation format of Unicode that contains only 7-bit
;; ASCII octets and is intended to be readable by humans in the limiting
;; case that the document consists of characters from the US-ASCII
;; repertoire.
;; In short, runs of characters outside US-ASCII are encoded as base64
;; inside delimiters.
;; A variation of UTF-7 is specified in IMAP 4rev1 (RFC 2060) as the way
;; to represent characters outside US-ASCII in mailbox names in IMAP.
;; This library supports both variants, but the IMAP variation was the
;; reason I wrote it.
;; The routines convert UTF-7 -> UTF-16 (16 bit encoding of Unicode)
;; -> current character set, and vice versa.
;; However, until Emacs supports Unicode, the only Emacs character set
;; supported here is ISO-8859.1, which can trivially be converted to/from
;; Unicode.
;; When decoding results in a character outside the Emacs character set,
;; an error is thrown. It is up to the application to recover.
;; UTF-7 should be done by providing a coding system. Mule-UCS does
;; already, but I don't know if it does the IMAP version and it's not
;; clear whether that should really be a coding system. The UTF-16
;; part of the conversion can be done with coding systems available
;; with Mule-UCS or some versions of Emacs. Unfortunately these were
;; done wrongly (regarding handling of byte-order marks and how the
;; variants were named), so we don't have a consistent name for the
;; necessary coding system. The code below doesn't seem to DTRT
;; generally. E.g.:
;;
;; (utf7-encode "a+£")
;; => "a+ACsAow-"
;;
;; $ echo "a+£"|iconv -f utf-8 -t utf-7
;; a+-+AKM
;;
;; -- fx
;;; Code:
(require 'base64)
(require 'mm-util)
(defconst utf7-direct-encoding-chars " -%'-*,-[]-}"
"Character ranges which do not need escaping in UTF-7.")
(defconst utf7-imap-direct-encoding-chars
(concat utf7-direct-encoding-chars "+\\~")
"Character ranges which do not need escaping in the IMAP variant of UTF-7.")
(defconst utf7-utf-16-coding-system
(cond ((mm-coding-system-p 'utf-16-be-no-signature) ; Mule-UCS
'utf-16-be-no-signature)
((and (mm-coding-system-p 'utf-16-be) ; Emacs
;; Avoid versions with BOM.
(= 2 (length (encode-coding-string "a" 'utf-16-be))))
'utf-16-be)
((mm-coding-system-p 'utf-16-be-nosig) ; ?
'utf-16-be-nosig))
"Coding system which encodes big endian UTF-16 without a BOM signature.")
(defsubst utf7-imap-get-pad-length (len modulus)
"Return required length of padding for IMAP modified base64 fragment."
(mod (- len) modulus))
(defun utf7-encode-internal (&optional for-imap)
"Encode text in (temporary) buffer as UTF-7.
Use IMAP modification if FOR-IMAP is non-nil."
(let ((start (point-min))
(end (point-max)))
(narrow-to-region start end)
(goto-char start)
(let* ((esc-char (if for-imap ?& ?+))
(direct-encoding-chars
(if for-imap utf7-imap-direct-encoding-chars
utf7-direct-encoding-chars))
(not-direct-encoding-chars (concat "^" direct-encoding-chars)))
(while (not (eobp))
(skip-chars-forward direct-encoding-chars)
(unless (eobp)
(insert esc-char)
(let ((p (point))
(fc (following-char))
(run-length
(skip-chars-forward not-direct-encoding-chars)))
(if (and (= fc esc-char)
(= run-length 1)) ; Lone esc-char?
(delete-char -1) ; Now there's one too many
(utf7-fragment-encode p (point) for-imap))
(insert "-")))))))
(defun utf7-fragment-encode (start end &optional for-imap)
"Encode text from START to END in buffer as UTF-7 escape fragment.
Use IMAP modification if FOR-IMAP is non-nil."
(save-restriction
(let* ((buf (current-buffer))
(base (with-temp-buffer
(set-buffer-multibyte nil)
(insert-buffer-substring buf start end)
(funcall (utf7-get-u16char-converter 'to-utf-16))
(base64-encode-region (point-min) (point-max))
(buffer-string))))
(narrow-to-region start end)
(delete-region (point-min) (point-max))
(insert base))
(goto-char (point-min))
(let ((pm (point-max)))
(when for-imap
(while (search-forward "/" nil t)
(replace-match ",")))
(skip-chars-forward "^= \t\n" pm)
(delete-region (point) pm))))
(defun utf7-decode-internal (&optional for-imap)
"Decode UTF-7 text in (temporary) buffer.
Use IMAP modification if FOR-IMAP is non-nil."
(let ((start (point-min)))
(goto-char start)
(let* ((esc-pattern (concat "^" (char-to-string (if for-imap ?& ?+))))
(base64-chars (concat "A-Za-z0-9+"
(char-to-string (if for-imap ?, ?/)))))
(while (not (eobp))
(skip-chars-forward esc-pattern)
(unless (eobp)
(forward-char)
(let ((p (point))
(run-length (skip-chars-forward base64-chars)))
(when (and (not (eobp)) (= (following-char) ?-))
(delete-char 1))
(unless (= run-length 0) ; Encoded lone esc-char?
(save-excursion
(utf7-fragment-decode p (point) for-imap)
(goto-char p)
(delete-char -1)))))))))
(defun utf7-fragment-decode (start end &optional for-imap)
"Decode base64 encoded fragment from START to END of UTF-7 text in buffer.
Use IMAP modification if FOR-IMAP is non-nil."
(save-restriction
(narrow-to-region start end)
(when for-imap
(goto-char start)
(while (search-forward "," nil 'move-to-end) (replace-match "/")))
(let ((pl (utf7-imap-get-pad-length (- end start) 4)))
(insert (make-string pl ?=))
(base64-decode-region start (+ end pl)))
(funcall (utf7-get-u16char-converter 'from-utf-16))))
(defun utf7-get-u16char-converter (which-way)
"Return a function to convert between UTF-16 and current character set."
(if utf7-utf-16-coding-system
(if (eq which-way 'to-utf-16)
(lambda ()
(encode-coding-region (point-min) (point-max)
utf7-utf-16-coding-system))
(lambda ()
(decode-coding-region (point-min) (point-max)
utf7-utf-16-coding-system)))
;; Add test to check if we are really Latin-1.
(if (eq which-way 'to-utf-16)
'utf7-latin1-u16-char-converter
'utf7-u16-latin1-char-converter)))
(defun utf7-latin1-u16-char-converter ()
"Convert latin 1 (ISO-8859.1) characters to 16 bit Unicode.
Characters are converted to raw byte pairs in narrowed buffer."
(encode-coding-region (point-min) (point-max) 'iso-8859-1)
(goto-char (point-min))
(while (not (eobp))
(insert 0)
(forward-char)))
(defun utf7-u16-latin1-char-converter ()
"Convert 16 bit Unicode characters to latin 1 (ISO-8859.1).
Characters are in raw byte pairs in narrowed buffer."
(goto-char (point-min))
(while (not (eobp))
(if (= 0 (following-char))
(delete-char 1)
(error "Unable to convert from Unicode"))
(forward-char))
(decode-coding-region (point-min) (point-max) 'iso-8859-1)
(mm-enable-multibyte))
;;;###autoload
(defun utf7-encode (string &optional for-imap)
"Encode UTF-7 STRING. Use IMAP modification if FOR-IMAP is non-nil."
(if (and (coding-system-p 'utf-7) (coding-system-p 'utf-7-imap))
;; Emacs 23 with proper support for IMAP
(encode-coding-string string (if for-imap 'utf-7-imap 'utf-7))
(mm-with-multibyte-buffer
(insert string)
(utf7-encode-internal for-imap)
(buffer-string))))
(defun utf7-decode (string &optional for-imap)
"Decode UTF-7 STRING. Use IMAP modification if FOR-IMAP is non-nil."
(if (and (coding-system-p 'utf-7) (coding-system-p 'utf-7-imap))
;; Emacs 23 with proper support for IMAP
(decode-coding-string string (if for-imap 'utf-7-imap 'utf-7))
(mm-with-unibyte-buffer
(insert string)
(utf7-decode-internal for-imap)
(mm-enable-multibyte)
(buffer-string))))
(provide 'utf7)
;;; utf7.el ends here
|
