1f41bbe4cSAndreas Gohr<?php 2f41bbe4cSAndreas Gohr 3f41bbe4cSAndreas Gohrnamespace dokuwiki\Utf8; 4f41bbe4cSAndreas Gohr 5f41bbe4cSAndreas Gohr/** 6f41bbe4cSAndreas Gohr * Convert between UTF-8 and a list of Unicode Code Points 7f41bbe4cSAndreas Gohr */ 8f41bbe4cSAndreas Gohrclass Unicode 9f41bbe4cSAndreas Gohr{ 10f41bbe4cSAndreas Gohr 11f41bbe4cSAndreas Gohr /** 12f41bbe4cSAndreas Gohr * Takes an UTF-8 string and returns an array of ints representing the 13f41bbe4cSAndreas Gohr * Unicode characters. Astral planes are supported ie. the ints in the 14f41bbe4cSAndreas Gohr * output can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates 15f41bbe4cSAndreas Gohr * are not allowed. 16f41bbe4cSAndreas Gohr * 17f41bbe4cSAndreas Gohr * If $strict is set to true the function returns false if the input 18f41bbe4cSAndreas Gohr * string isn't a valid UTF-8 octet sequence and raises a PHP error at 19f41bbe4cSAndreas Gohr * level E_USER_WARNING 20f41bbe4cSAndreas Gohr * 21f41bbe4cSAndreas Gohr * Note: this function has been modified slightly in this library to 22f41bbe4cSAndreas Gohr * trigger errors on encountering bad bytes 23f41bbe4cSAndreas Gohr * 24f41bbe4cSAndreas Gohr * @author <hsivonen@iki.fi> 25f41bbe4cSAndreas Gohr * @author Harry Fuecks <hfuecks@gmail.com> 26f41bbe4cSAndreas Gohr * @see unicode_to_utf8 27f41bbe4cSAndreas Gohr * @link http://hsivonen.iki.fi/php-utf8/ 28f41bbe4cSAndreas Gohr * @link http://sourceforge.net/projects/phputf8/ 29f41bbe4cSAndreas Gohr * @todo break into less complex chunks 30f41bbe4cSAndreas Gohr * @todo use exceptions instead of user errors 31f41bbe4cSAndreas Gohr * 32f41bbe4cSAndreas Gohr * @param string $str UTF-8 encoded string 33f41bbe4cSAndreas Gohr * @param boolean $strict Check for invalid sequences? 34f41bbe4cSAndreas Gohr * @return mixed array of unicode code points or false if UTF-8 invalid 35f41bbe4cSAndreas Gohr */ 36f41bbe4cSAndreas Gohr public static function fromUtf8($str, $strict = false) 37f41bbe4cSAndreas Gohr { 38f41bbe4cSAndreas Gohr $mState = 0; // cached expected number of octets after the current octet 39f41bbe4cSAndreas Gohr // until the beginning of the next UTF8 character sequence 40f41bbe4cSAndreas Gohr $mUcs4 = 0; // cached Unicode character 41f41bbe4cSAndreas Gohr $mBytes = 1; // cached expected number of octets in the current sequence 42f41bbe4cSAndreas Gohr 43f41bbe4cSAndreas Gohr $out = array(); 44f41bbe4cSAndreas Gohr 45f41bbe4cSAndreas Gohr $len = strlen($str); 46f41bbe4cSAndreas Gohr 47f41bbe4cSAndreas Gohr for ($i = 0; $i < $len; $i++) { 48f41bbe4cSAndreas Gohr 49*2401f18dSSyntaxseed $in = ord($str[$i]); 50f41bbe4cSAndreas Gohr 51f41bbe4cSAndreas Gohr if ($mState === 0) { 52f41bbe4cSAndreas Gohr 53f41bbe4cSAndreas Gohr // When mState is zero we expect either a US-ASCII character or a 54f41bbe4cSAndreas Gohr // multi-octet sequence. 55f41bbe4cSAndreas Gohr if (0 === (0x80 & $in)) { 56f41bbe4cSAndreas Gohr // US-ASCII, pass straight through. 57f41bbe4cSAndreas Gohr $out[] = $in; 58f41bbe4cSAndreas Gohr $mBytes = 1; 59f41bbe4cSAndreas Gohr 60f41bbe4cSAndreas Gohr } else if (0xC0 === (0xE0 & $in)) { 61f41bbe4cSAndreas Gohr // First octet of 2 octet sequence 62f41bbe4cSAndreas Gohr $mUcs4 = $in; 63f41bbe4cSAndreas Gohr $mUcs4 = ($mUcs4 & 0x1F) << 6; 64f41bbe4cSAndreas Gohr $mState = 1; 65f41bbe4cSAndreas Gohr $mBytes = 2; 66f41bbe4cSAndreas Gohr 67f41bbe4cSAndreas Gohr } else if (0xE0 === (0xF0 & $in)) { 68f41bbe4cSAndreas Gohr // First octet of 3 octet sequence 69f41bbe4cSAndreas Gohr $mUcs4 = $in; 70f41bbe4cSAndreas Gohr $mUcs4 = ($mUcs4 & 0x0F) << 12; 71f41bbe4cSAndreas Gohr $mState = 2; 72f41bbe4cSAndreas Gohr $mBytes = 3; 73f41bbe4cSAndreas Gohr 74f41bbe4cSAndreas Gohr } else if (0xF0 === (0xF8 & $in)) { 75f41bbe4cSAndreas Gohr // First octet of 4 octet sequence 76f41bbe4cSAndreas Gohr $mUcs4 = $in; 77f41bbe4cSAndreas Gohr $mUcs4 = ($mUcs4 & 0x07) << 18; 78f41bbe4cSAndreas Gohr $mState = 3; 79f41bbe4cSAndreas Gohr $mBytes = 4; 80f41bbe4cSAndreas Gohr 81f41bbe4cSAndreas Gohr } else if (0xF8 === (0xFC & $in)) { 82f41bbe4cSAndreas Gohr /* First octet of 5 octet sequence. 83f41bbe4cSAndreas Gohr * 84f41bbe4cSAndreas Gohr * This is illegal because the encoded codepoint must be either 85f41bbe4cSAndreas Gohr * (a) not the shortest form or 86f41bbe4cSAndreas Gohr * (b) outside the Unicode range of 0-0x10FFFF. 87f41bbe4cSAndreas Gohr * Rather than trying to resynchronize, we will carry on until the end 88f41bbe4cSAndreas Gohr * of the sequence and let the later error handling code catch it. 89f41bbe4cSAndreas Gohr */ 90f41bbe4cSAndreas Gohr $mUcs4 = $in; 91f41bbe4cSAndreas Gohr $mUcs4 = ($mUcs4 & 0x03) << 24; 92f41bbe4cSAndreas Gohr $mState = 4; 93f41bbe4cSAndreas Gohr $mBytes = 5; 94f41bbe4cSAndreas Gohr 95f41bbe4cSAndreas Gohr } else if (0xFC === (0xFE & $in)) { 96f41bbe4cSAndreas Gohr // First octet of 6 octet sequence, see comments for 5 octet sequence. 97f41bbe4cSAndreas Gohr $mUcs4 = $in; 98f41bbe4cSAndreas Gohr $mUcs4 = ($mUcs4 & 1) << 30; 99f41bbe4cSAndreas Gohr $mState = 5; 100f41bbe4cSAndreas Gohr $mBytes = 6; 101f41bbe4cSAndreas Gohr 102f41bbe4cSAndreas Gohr } elseif ($strict) { 103f41bbe4cSAndreas Gohr /* Current octet is neither in the US-ASCII range nor a legal first 104f41bbe4cSAndreas Gohr * octet of a multi-octet sequence. 105f41bbe4cSAndreas Gohr */ 106f41bbe4cSAndreas Gohr trigger_error( 107f41bbe4cSAndreas Gohr 'utf8_to_unicode: Illegal sequence identifier ' . 108f41bbe4cSAndreas Gohr 'in UTF-8 at byte ' . $i, 109f41bbe4cSAndreas Gohr E_USER_WARNING 110f41bbe4cSAndreas Gohr ); 111f41bbe4cSAndreas Gohr return false; 112f41bbe4cSAndreas Gohr 113f41bbe4cSAndreas Gohr } 114f41bbe4cSAndreas Gohr 115f41bbe4cSAndreas Gohr } else { 116f41bbe4cSAndreas Gohr 117f41bbe4cSAndreas Gohr // When mState is non-zero, we expect a continuation of the multi-octet 118f41bbe4cSAndreas Gohr // sequence 119f41bbe4cSAndreas Gohr if (0x80 === (0xC0 & $in)) { 120f41bbe4cSAndreas Gohr 121f41bbe4cSAndreas Gohr // Legal continuation. 122f41bbe4cSAndreas Gohr $shift = ($mState - 1) * 6; 123f41bbe4cSAndreas Gohr $tmp = $in; 124f41bbe4cSAndreas Gohr $tmp = ($tmp & 0x0000003F) << $shift; 125f41bbe4cSAndreas Gohr $mUcs4 |= $tmp; 126f41bbe4cSAndreas Gohr 127f41bbe4cSAndreas Gohr /** 128f41bbe4cSAndreas Gohr * End of the multi-octet sequence. mUcs4 now contains the final 129f41bbe4cSAndreas Gohr * Unicode codepoint to be output 130f41bbe4cSAndreas Gohr */ 131f41bbe4cSAndreas Gohr if (0 === --$mState) { 132f41bbe4cSAndreas Gohr 133f41bbe4cSAndreas Gohr /* 134f41bbe4cSAndreas Gohr * Check for illegal sequences and codepoints. 135f41bbe4cSAndreas Gohr */ 136f41bbe4cSAndreas Gohr // From Unicode 3.1, non-shortest form is illegal 137f41bbe4cSAndreas Gohr if (((2 === $mBytes) && ($mUcs4 < 0x0080)) || 138f41bbe4cSAndreas Gohr ((3 === $mBytes) && ($mUcs4 < 0x0800)) || 139f41bbe4cSAndreas Gohr ((4 === $mBytes) && ($mUcs4 < 0x10000)) || 140f41bbe4cSAndreas Gohr (4 < $mBytes) || 141f41bbe4cSAndreas Gohr // From Unicode 3.2, surrogate characters are illegal 142f41bbe4cSAndreas Gohr (($mUcs4 & 0xFFFFF800) === 0xD800) || 143f41bbe4cSAndreas Gohr // Codepoints outside the Unicode range are illegal 144f41bbe4cSAndreas Gohr ($mUcs4 > 0x10FFFF)) { 145f41bbe4cSAndreas Gohr 146f41bbe4cSAndreas Gohr if ($strict) { 147f41bbe4cSAndreas Gohr trigger_error( 148f41bbe4cSAndreas Gohr 'utf8_to_unicode: Illegal sequence or codepoint ' . 149f41bbe4cSAndreas Gohr 'in UTF-8 at byte ' . $i, 150f41bbe4cSAndreas Gohr E_USER_WARNING 151f41bbe4cSAndreas Gohr ); 152f41bbe4cSAndreas Gohr 153f41bbe4cSAndreas Gohr return false; 154f41bbe4cSAndreas Gohr } 155f41bbe4cSAndreas Gohr 156f41bbe4cSAndreas Gohr } 157f41bbe4cSAndreas Gohr 158f41bbe4cSAndreas Gohr if (0xFEFF !== $mUcs4) { 159f41bbe4cSAndreas Gohr // BOM is legal but we don't want to output it 160f41bbe4cSAndreas Gohr $out[] = $mUcs4; 161f41bbe4cSAndreas Gohr } 162f41bbe4cSAndreas Gohr 163f41bbe4cSAndreas Gohr //initialize UTF8 cache 164f41bbe4cSAndreas Gohr $mState = 0; 165f41bbe4cSAndreas Gohr $mUcs4 = 0; 166f41bbe4cSAndreas Gohr $mBytes = 1; 167f41bbe4cSAndreas Gohr } 168f41bbe4cSAndreas Gohr 169f41bbe4cSAndreas Gohr } elseif ($strict) { 170f41bbe4cSAndreas Gohr /** 171f41bbe4cSAndreas Gohr *((0xC0 & (*in) != 0x80) && (mState != 0)) 172f41bbe4cSAndreas Gohr * Incomplete multi-octet sequence. 173f41bbe4cSAndreas Gohr */ 174f41bbe4cSAndreas Gohr trigger_error( 175f41bbe4cSAndreas Gohr 'utf8_to_unicode: Incomplete multi-octet ' . 176f41bbe4cSAndreas Gohr ' sequence in UTF-8 at byte ' . $i, 177f41bbe4cSAndreas Gohr E_USER_WARNING 178f41bbe4cSAndreas Gohr ); 179f41bbe4cSAndreas Gohr 180f41bbe4cSAndreas Gohr return false; 181f41bbe4cSAndreas Gohr } 182f41bbe4cSAndreas Gohr } 183f41bbe4cSAndreas Gohr } 184f41bbe4cSAndreas Gohr return $out; 185f41bbe4cSAndreas Gohr } 186f41bbe4cSAndreas Gohr 187f41bbe4cSAndreas Gohr /** 188f41bbe4cSAndreas Gohr * Takes an array of ints representing the Unicode characters and returns 189f41bbe4cSAndreas Gohr * a UTF-8 string. Astral planes are supported ie. the ints in the 190f41bbe4cSAndreas Gohr * input can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates 191f41bbe4cSAndreas Gohr * are not allowed. 192f41bbe4cSAndreas Gohr * 193f41bbe4cSAndreas Gohr * If $strict is set to true the function returns false if the input 194f41bbe4cSAndreas Gohr * array contains ints that represent surrogates or are outside the 195f41bbe4cSAndreas Gohr * Unicode range and raises a PHP error at level E_USER_WARNING 196f41bbe4cSAndreas Gohr * 197f41bbe4cSAndreas Gohr * Note: this function has been modified slightly in this library to use 198f41bbe4cSAndreas Gohr * output buffering to concatenate the UTF-8 string (faster) as well as 199f41bbe4cSAndreas Gohr * reference the array by it's keys 200f41bbe4cSAndreas Gohr * 201f41bbe4cSAndreas Gohr * @param array $arr of unicode code points representing a string 202f41bbe4cSAndreas Gohr * @param boolean $strict Check for invalid sequences? 203f41bbe4cSAndreas Gohr * @return string|false UTF-8 string or false if array contains invalid code points 204f41bbe4cSAndreas Gohr * 205f41bbe4cSAndreas Gohr * @author <hsivonen@iki.fi> 206f41bbe4cSAndreas Gohr * @author Harry Fuecks <hfuecks@gmail.com> 207f41bbe4cSAndreas Gohr * @see utf8_to_unicode 208f41bbe4cSAndreas Gohr * @link http://hsivonen.iki.fi/php-utf8/ 209f41bbe4cSAndreas Gohr * @link http://sourceforge.net/projects/phputf8/ 210f41bbe4cSAndreas Gohr * @todo use exceptions instead of user errors 211f41bbe4cSAndreas Gohr */ 212f41bbe4cSAndreas Gohr public static function toUtf8($arr, $strict = false) 213f41bbe4cSAndreas Gohr { 214f41bbe4cSAndreas Gohr if (!is_array($arr)) return ''; 215f41bbe4cSAndreas Gohr ob_start(); 216f41bbe4cSAndreas Gohr 217f41bbe4cSAndreas Gohr foreach (array_keys($arr) as $k) { 218f41bbe4cSAndreas Gohr 219f41bbe4cSAndreas Gohr if (($arr[$k] >= 0) && ($arr[$k] <= 0x007f)) { 220f41bbe4cSAndreas Gohr # ASCII range (including control chars) 221f41bbe4cSAndreas Gohr 222f41bbe4cSAndreas Gohr echo chr($arr[$k]); 223f41bbe4cSAndreas Gohr 224f41bbe4cSAndreas Gohr } else if ($arr[$k] <= 0x07ff) { 225f41bbe4cSAndreas Gohr # 2 byte sequence 226f41bbe4cSAndreas Gohr 227f41bbe4cSAndreas Gohr echo chr(0xc0 | ($arr[$k] >> 6)); 228f41bbe4cSAndreas Gohr echo chr(0x80 | ($arr[$k] & 0x003f)); 229f41bbe4cSAndreas Gohr 230f41bbe4cSAndreas Gohr } else if ($arr[$k] == 0xFEFF) { 231f41bbe4cSAndreas Gohr # Byte order mark (skip) 232f41bbe4cSAndreas Gohr // nop -- zap the BOM 233f41bbe4cSAndreas Gohr 234f41bbe4cSAndreas Gohr } else if ($arr[$k] >= 0xD800 && $arr[$k] <= 0xDFFF) { 235f41bbe4cSAndreas Gohr # Test for illegal surrogates 236f41bbe4cSAndreas Gohr 237f41bbe4cSAndreas Gohr // found a surrogate 238f41bbe4cSAndreas Gohr if ($strict) { 239f41bbe4cSAndreas Gohr trigger_error( 240f41bbe4cSAndreas Gohr 'unicode_to_utf8: Illegal surrogate ' . 241f41bbe4cSAndreas Gohr 'at index: ' . $k . ', value: ' . $arr[$k], 242f41bbe4cSAndreas Gohr E_USER_WARNING 243f41bbe4cSAndreas Gohr ); 244f41bbe4cSAndreas Gohr return false; 245f41bbe4cSAndreas Gohr } 246f41bbe4cSAndreas Gohr 247f41bbe4cSAndreas Gohr } else if ($arr[$k] <= 0xffff) { 248f41bbe4cSAndreas Gohr # 3 byte sequence 249f41bbe4cSAndreas Gohr 250f41bbe4cSAndreas Gohr echo chr(0xe0 | ($arr[$k] >> 12)); 251f41bbe4cSAndreas Gohr echo chr(0x80 | (($arr[$k] >> 6) & 0x003f)); 252f41bbe4cSAndreas Gohr echo chr(0x80 | ($arr[$k] & 0x003f)); 253f41bbe4cSAndreas Gohr 254f41bbe4cSAndreas Gohr } else if ($arr[$k] <= 0x10ffff) { 255f41bbe4cSAndreas Gohr # 4 byte sequence 256f41bbe4cSAndreas Gohr 257f41bbe4cSAndreas Gohr echo chr(0xf0 | ($arr[$k] >> 18)); 258f41bbe4cSAndreas Gohr echo chr(0x80 | (($arr[$k] >> 12) & 0x3f)); 259f41bbe4cSAndreas Gohr echo chr(0x80 | (($arr[$k] >> 6) & 0x3f)); 260f41bbe4cSAndreas Gohr echo chr(0x80 | ($arr[$k] & 0x3f)); 261f41bbe4cSAndreas Gohr 262f41bbe4cSAndreas Gohr } elseif ($strict) { 263f41bbe4cSAndreas Gohr 264f41bbe4cSAndreas Gohr trigger_error( 265f41bbe4cSAndreas Gohr 'unicode_to_utf8: Codepoint out of Unicode range ' . 266f41bbe4cSAndreas Gohr 'at index: ' . $k . ', value: ' . $arr[$k], 267f41bbe4cSAndreas Gohr E_USER_WARNING 268f41bbe4cSAndreas Gohr ); 269f41bbe4cSAndreas Gohr 270f41bbe4cSAndreas Gohr // out of range 271f41bbe4cSAndreas Gohr return false; 272f41bbe4cSAndreas Gohr } 273f41bbe4cSAndreas Gohr } 274f41bbe4cSAndreas Gohr 275f41bbe4cSAndreas Gohr return ob_get_clean(); 276f41bbe4cSAndreas Gohr } 277f41bbe4cSAndreas Gohr} 278