1f41bbe4cSAndreas Gohr<?php 2f41bbe4cSAndreas Gohr 3f41bbe4cSAndreas Gohrnamespace dokuwiki\Utf8; 4f41bbe4cSAndreas Gohr 5f41bbe4cSAndreas Gohr/** 6f41bbe4cSAndreas Gohr * Convert between UTF-8 and a list of Unicode Code Points 7f41bbe4cSAndreas Gohr */ 8f41bbe4cSAndreas Gohrclass Unicode 9f41bbe4cSAndreas Gohr{ 10f41bbe4cSAndreas Gohr 11f41bbe4cSAndreas Gohr /** 12f41bbe4cSAndreas Gohr * Takes an UTF-8 string and returns an array of ints representing the 13f41bbe4cSAndreas Gohr * Unicode characters. Astral planes are supported ie. the ints in the 14f41bbe4cSAndreas Gohr * output can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates 15f41bbe4cSAndreas Gohr * are not allowed. 16f41bbe4cSAndreas Gohr * 17f41bbe4cSAndreas Gohr * If $strict is set to true the function returns false if the input 18f41bbe4cSAndreas Gohr * string isn't a valid UTF-8 octet sequence and raises a PHP error at 19f41bbe4cSAndreas Gohr * level E_USER_WARNING 20f41bbe4cSAndreas Gohr * 21f41bbe4cSAndreas Gohr * Note: this function has been modified slightly in this library to 22f41bbe4cSAndreas Gohr * trigger errors on encountering bad bytes 23f41bbe4cSAndreas Gohr * 24f41bbe4cSAndreas Gohr * @author <hsivonen@iki.fi> 25f41bbe4cSAndreas Gohr * @author Harry Fuecks <hfuecks@gmail.com> 26f41bbe4cSAndreas Gohr * @see unicode_to_utf8 27f41bbe4cSAndreas Gohr * @link http://hsivonen.iki.fi/php-utf8/ 28f41bbe4cSAndreas Gohr * @link http://sourceforge.net/projects/phputf8/ 29f41bbe4cSAndreas Gohr * @todo break into less complex chunks 30f41bbe4cSAndreas Gohr * @todo use exceptions instead of user errors 31f41bbe4cSAndreas Gohr * 32f41bbe4cSAndreas Gohr * @param string $str UTF-8 encoded string 33f41bbe4cSAndreas Gohr * @param boolean $strict Check for invalid sequences? 34f41bbe4cSAndreas Gohr * @return mixed array of unicode code points or false if UTF-8 invalid 35f41bbe4cSAndreas Gohr */ 36f41bbe4cSAndreas Gohr public static function fromUtf8($str, $strict = false) 37f41bbe4cSAndreas Gohr { 38f41bbe4cSAndreas Gohr $mState = 0; // cached expected number of octets after the current octet 39f41bbe4cSAndreas Gohr // until the beginning of the next UTF8 character sequence 40f41bbe4cSAndreas Gohr $mUcs4 = 0; // cached Unicode character 41f41bbe4cSAndreas Gohr $mBytes = 1; // cached expected number of octets in the current sequence 42f41bbe4cSAndreas Gohr 43*e025be72SAndreas Gohr $out = []; 44f41bbe4cSAndreas Gohr 45f41bbe4cSAndreas Gohr $len = strlen($str); 46f41bbe4cSAndreas Gohr 47f41bbe4cSAndreas Gohr for ($i = 0; $i < $len; $i++) { 48f41bbe4cSAndreas Gohr 492401f18dSSyntaxseed $in = ord($str[$i]); 50f41bbe4cSAndreas Gohr 51f41bbe4cSAndreas Gohr if ($mState === 0) { 52f41bbe4cSAndreas Gohr // When mState is zero we expect either a US-ASCII character or a 53f41bbe4cSAndreas Gohr // multi-octet sequence. 54f41bbe4cSAndreas Gohr if (0 === (0x80 & $in)) { 55f41bbe4cSAndreas Gohr // US-ASCII, pass straight through. 56f41bbe4cSAndreas Gohr $out[] = $in; 57f41bbe4cSAndreas Gohr $mBytes = 1; 58f41bbe4cSAndreas Gohr } elseif (0xC0 === (0xE0 & $in)) { 59f41bbe4cSAndreas Gohr // First octet of 2 octet sequence 60f41bbe4cSAndreas Gohr $mUcs4 = $in; 61f41bbe4cSAndreas Gohr $mUcs4 = ($mUcs4 & 0x1F) << 6; 62f41bbe4cSAndreas Gohr $mState = 1; 63f41bbe4cSAndreas Gohr $mBytes = 2; 64f41bbe4cSAndreas Gohr } elseif (0xE0 === (0xF0 & $in)) { 65f41bbe4cSAndreas Gohr // First octet of 3 octet sequence 66f41bbe4cSAndreas Gohr $mUcs4 = $in; 67f41bbe4cSAndreas Gohr $mUcs4 = ($mUcs4 & 0x0F) << 12; 68f41bbe4cSAndreas Gohr $mState = 2; 69f41bbe4cSAndreas Gohr $mBytes = 3; 70f41bbe4cSAndreas Gohr } elseif (0xF0 === (0xF8 & $in)) { 71f41bbe4cSAndreas Gohr // First octet of 4 octet sequence 72f41bbe4cSAndreas Gohr $mUcs4 = $in; 73f41bbe4cSAndreas Gohr $mUcs4 = ($mUcs4 & 0x07) << 18; 74f41bbe4cSAndreas Gohr $mState = 3; 75f41bbe4cSAndreas Gohr $mBytes = 4; 76f41bbe4cSAndreas Gohr } elseif (0xF8 === (0xFC & $in)) { 77f41bbe4cSAndreas Gohr /* First octet of 5 octet sequence. 78f41bbe4cSAndreas Gohr * 79f41bbe4cSAndreas Gohr * This is illegal because the encoded codepoint must be either 80f41bbe4cSAndreas Gohr * (a) not the shortest form or 81f41bbe4cSAndreas Gohr * (b) outside the Unicode range of 0-0x10FFFF. 82f41bbe4cSAndreas Gohr * Rather than trying to resynchronize, we will carry on until the end 83f41bbe4cSAndreas Gohr * of the sequence and let the later error handling code catch it. 84f41bbe4cSAndreas Gohr */ 85f41bbe4cSAndreas Gohr $mUcs4 = $in; 86f41bbe4cSAndreas Gohr $mUcs4 = ($mUcs4 & 0x03) << 24; 87f41bbe4cSAndreas Gohr $mState = 4; 88f41bbe4cSAndreas Gohr $mBytes = 5; 89f41bbe4cSAndreas Gohr } elseif (0xFC === (0xFE & $in)) { 90f41bbe4cSAndreas Gohr // First octet of 6 octet sequence, see comments for 5 octet sequence. 91f41bbe4cSAndreas Gohr $mUcs4 = $in; 92f41bbe4cSAndreas Gohr $mUcs4 = ($mUcs4 & 1) << 30; 93f41bbe4cSAndreas Gohr $mState = 5; 94f41bbe4cSAndreas Gohr $mBytes = 6; 95f41bbe4cSAndreas Gohr } elseif ($strict) { 96f41bbe4cSAndreas Gohr /* Current octet is neither in the US-ASCII range nor a legal first 97f41bbe4cSAndreas Gohr * octet of a multi-octet sequence. 98f41bbe4cSAndreas Gohr */ 99f41bbe4cSAndreas Gohr trigger_error( 100f41bbe4cSAndreas Gohr 'utf8_to_unicode: Illegal sequence identifier ' . 101f41bbe4cSAndreas Gohr 'in UTF-8 at byte ' . $i, 102f41bbe4cSAndreas Gohr E_USER_WARNING 103f41bbe4cSAndreas Gohr ); 104f41bbe4cSAndreas Gohr return false; 105f41bbe4cSAndreas Gohr 106f41bbe4cSAndreas Gohr } 107*e025be72SAndreas Gohr } elseif (0x80 === (0xC0 & $in)) { 108f41bbe4cSAndreas Gohr // When mState is non-zero, we expect a continuation of the multi-octet 109f41bbe4cSAndreas Gohr // sequence 110f41bbe4cSAndreas Gohr // Legal continuation. 111f41bbe4cSAndreas Gohr $shift = ($mState - 1) * 6; 112f41bbe4cSAndreas Gohr $tmp = $in; 113f41bbe4cSAndreas Gohr $tmp = ($tmp & 0x0000003F) << $shift; 114f41bbe4cSAndreas Gohr $mUcs4 |= $tmp; 115f41bbe4cSAndreas Gohr /** 116f41bbe4cSAndreas Gohr * End of the multi-octet sequence. mUcs4 now contains the final 117f41bbe4cSAndreas Gohr * Unicode codepoint to be output 118f41bbe4cSAndreas Gohr */ 119f41bbe4cSAndreas Gohr if (0 === --$mState) { 120f41bbe4cSAndreas Gohr 121f41bbe4cSAndreas Gohr /* 122f41bbe4cSAndreas Gohr * Check for illegal sequences and codepoints. 123f41bbe4cSAndreas Gohr */ 124f41bbe4cSAndreas Gohr // From Unicode 3.1, non-shortest form is illegal 125f41bbe4cSAndreas Gohr if (((2 === $mBytes) && ($mUcs4 < 0x0080)) || 126f41bbe4cSAndreas Gohr ((3 === $mBytes) && ($mUcs4 < 0x0800)) || 127f41bbe4cSAndreas Gohr ((4 === $mBytes) && ($mUcs4 < 0x10000)) || 128f41bbe4cSAndreas Gohr (4 < $mBytes) || 129f41bbe4cSAndreas Gohr // From Unicode 3.2, surrogate characters are illegal 130f41bbe4cSAndreas Gohr (($mUcs4 & 0xFFFFF800) === 0xD800) || 131f41bbe4cSAndreas Gohr // Codepoints outside the Unicode range are illegal 132f41bbe4cSAndreas Gohr ($mUcs4 > 0x10FFFF)) { 133f41bbe4cSAndreas Gohr 134f41bbe4cSAndreas Gohr if ($strict) { 135f41bbe4cSAndreas Gohr trigger_error( 136f41bbe4cSAndreas Gohr 'utf8_to_unicode: Illegal sequence or codepoint ' . 137f41bbe4cSAndreas Gohr 'in UTF-8 at byte ' . $i, 138f41bbe4cSAndreas Gohr E_USER_WARNING 139f41bbe4cSAndreas Gohr ); 140f41bbe4cSAndreas Gohr 141f41bbe4cSAndreas Gohr return false; 142f41bbe4cSAndreas Gohr } 143f41bbe4cSAndreas Gohr 144f41bbe4cSAndreas Gohr } 145f41bbe4cSAndreas Gohr 146f41bbe4cSAndreas Gohr if (0xFEFF !== $mUcs4) { 147f41bbe4cSAndreas Gohr // BOM is legal but we don't want to output it 148f41bbe4cSAndreas Gohr $out[] = $mUcs4; 149f41bbe4cSAndreas Gohr } 150f41bbe4cSAndreas Gohr 151f41bbe4cSAndreas Gohr //initialize UTF8 cache 152f41bbe4cSAndreas Gohr $mState = 0; 153f41bbe4cSAndreas Gohr $mUcs4 = 0; 154f41bbe4cSAndreas Gohr $mBytes = 1; 155f41bbe4cSAndreas Gohr } 156f41bbe4cSAndreas Gohr } elseif ($strict) { 157f41bbe4cSAndreas Gohr /** 158f41bbe4cSAndreas Gohr *((0xC0 & (*in) != 0x80) && (mState != 0)) 159f41bbe4cSAndreas Gohr * Incomplete multi-octet sequence. 160f41bbe4cSAndreas Gohr */ 161f41bbe4cSAndreas Gohr trigger_error( 162f41bbe4cSAndreas Gohr 'utf8_to_unicode: Incomplete multi-octet ' . 163f41bbe4cSAndreas Gohr ' sequence in UTF-8 at byte ' . $i, 164f41bbe4cSAndreas Gohr E_USER_WARNING 165f41bbe4cSAndreas Gohr ); 166f41bbe4cSAndreas Gohr 167f41bbe4cSAndreas Gohr return false; 168f41bbe4cSAndreas Gohr } 169f41bbe4cSAndreas Gohr } 170f41bbe4cSAndreas Gohr return $out; 171f41bbe4cSAndreas Gohr } 172f41bbe4cSAndreas Gohr 173f41bbe4cSAndreas Gohr /** 174f41bbe4cSAndreas Gohr * Takes an array of ints representing the Unicode characters and returns 175f41bbe4cSAndreas Gohr * a UTF-8 string. Astral planes are supported ie. the ints in the 176f41bbe4cSAndreas Gohr * input can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates 177f41bbe4cSAndreas Gohr * are not allowed. 178f41bbe4cSAndreas Gohr * 179f41bbe4cSAndreas Gohr * If $strict is set to true the function returns false if the input 180f41bbe4cSAndreas Gohr * array contains ints that represent surrogates or are outside the 181f41bbe4cSAndreas Gohr * Unicode range and raises a PHP error at level E_USER_WARNING 182f41bbe4cSAndreas Gohr * 183f41bbe4cSAndreas Gohr * Note: this function has been modified slightly in this library to use 184f41bbe4cSAndreas Gohr * output buffering to concatenate the UTF-8 string (faster) as well as 185f41bbe4cSAndreas Gohr * reference the array by it's keys 186f41bbe4cSAndreas Gohr * 187f41bbe4cSAndreas Gohr * @param array $arr of unicode code points representing a string 188f41bbe4cSAndreas Gohr * @param boolean $strict Check for invalid sequences? 189f41bbe4cSAndreas Gohr * @return string|false UTF-8 string or false if array contains invalid code points 190f41bbe4cSAndreas Gohr * 191f41bbe4cSAndreas Gohr * @author <hsivonen@iki.fi> 192f41bbe4cSAndreas Gohr * @author Harry Fuecks <hfuecks@gmail.com> 193f41bbe4cSAndreas Gohr * @see utf8_to_unicode 194f41bbe4cSAndreas Gohr * @link http://hsivonen.iki.fi/php-utf8/ 195f41bbe4cSAndreas Gohr * @link http://sourceforge.net/projects/phputf8/ 196f41bbe4cSAndreas Gohr * @todo use exceptions instead of user errors 197f41bbe4cSAndreas Gohr */ 198f41bbe4cSAndreas Gohr public static function toUtf8($arr, $strict = false) 199f41bbe4cSAndreas Gohr { 200f41bbe4cSAndreas Gohr if (!is_array($arr)) return ''; 201f41bbe4cSAndreas Gohr ob_start(); 202f41bbe4cSAndreas Gohr 203f41bbe4cSAndreas Gohr foreach (array_keys($arr) as $k) { 204f41bbe4cSAndreas Gohr 205f41bbe4cSAndreas Gohr if (($arr[$k] >= 0) && ($arr[$k] <= 0x007f)) { 206f41bbe4cSAndreas Gohr # ASCII range (including control chars) 207f41bbe4cSAndreas Gohr echo chr($arr[$k]); 208f41bbe4cSAndreas Gohr } elseif ($arr[$k] <= 0x07ff) { 209f41bbe4cSAndreas Gohr # 2 byte sequence 210f41bbe4cSAndreas Gohr echo chr(0xc0 | ($arr[$k] >> 6)); 211f41bbe4cSAndreas Gohr echo chr(0x80 | ($arr[$k] & 0x003f)); 212f41bbe4cSAndreas Gohr } elseif ($arr[$k] == 0xFEFF) { 213f41bbe4cSAndreas Gohr # Byte order mark (skip) 214f41bbe4cSAndreas Gohr // nop -- zap the BOM 215f41bbe4cSAndreas Gohr } elseif ($arr[$k] >= 0xD800 && $arr[$k] <= 0xDFFF) { 216f41bbe4cSAndreas Gohr # Test for illegal surrogates 217f41bbe4cSAndreas Gohr // found a surrogate 218f41bbe4cSAndreas Gohr if ($strict) { 219f41bbe4cSAndreas Gohr trigger_error( 220f41bbe4cSAndreas Gohr 'unicode_to_utf8: Illegal surrogate ' . 221f41bbe4cSAndreas Gohr 'at index: ' . $k . ', value: ' . $arr[$k], 222f41bbe4cSAndreas Gohr E_USER_WARNING 223f41bbe4cSAndreas Gohr ); 224f41bbe4cSAndreas Gohr return false; 225f41bbe4cSAndreas Gohr } 226f41bbe4cSAndreas Gohr } elseif ($arr[$k] <= 0xffff) { 227f41bbe4cSAndreas Gohr # 3 byte sequence 228f41bbe4cSAndreas Gohr echo chr(0xe0 | ($arr[$k] >> 12)); 229f41bbe4cSAndreas Gohr echo chr(0x80 | (($arr[$k] >> 6) & 0x003f)); 230f41bbe4cSAndreas Gohr echo chr(0x80 | ($arr[$k] & 0x003f)); 231f41bbe4cSAndreas Gohr } elseif ($arr[$k] <= 0x10ffff) { 232f41bbe4cSAndreas Gohr # 4 byte sequence 233f41bbe4cSAndreas Gohr echo chr(0xf0 | ($arr[$k] >> 18)); 234f41bbe4cSAndreas Gohr echo chr(0x80 | (($arr[$k] >> 12) & 0x3f)); 235f41bbe4cSAndreas Gohr echo chr(0x80 | (($arr[$k] >> 6) & 0x3f)); 236f41bbe4cSAndreas Gohr echo chr(0x80 | ($arr[$k] & 0x3f)); 237f41bbe4cSAndreas Gohr } elseif ($strict) { 238f41bbe4cSAndreas Gohr 239f41bbe4cSAndreas Gohr trigger_error( 240f41bbe4cSAndreas Gohr 'unicode_to_utf8: Codepoint out of Unicode range ' . 241f41bbe4cSAndreas Gohr 'at index: ' . $k . ', value: ' . $arr[$k], 242f41bbe4cSAndreas Gohr E_USER_WARNING 243f41bbe4cSAndreas Gohr ); 244f41bbe4cSAndreas Gohr 245f41bbe4cSAndreas Gohr // out of range 246f41bbe4cSAndreas Gohr return false; 247f41bbe4cSAndreas Gohr } 248f41bbe4cSAndreas Gohr } 249f41bbe4cSAndreas Gohr 250f41bbe4cSAndreas Gohr return ob_get_clean(); 251f41bbe4cSAndreas Gohr } 252f41bbe4cSAndreas Gohr} 253