Drupal
/
drupal
mirror of https://git.drupalcode.org/project/drupal.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

- #26688: Add mbstring support to Drupal and clear up string handling fuzzies.

4.7.x
Steven Wittens 2005-07-25 20:38:30 +00:00
parent 1eb8b2c13a
commit dd2c576599
2 changed files with 2 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

230
includes/common.inc
View File

@ -1737,235 +1737,6 @@ function drupal_map_assoc($array, $function = NULL) {
}
}
/**
* Prepare a new XML parser.
*
* This is a wrapper around xml_parser_create() which extracts the encoding from
* the XML data first and sets the output encoding to UTF-8. This function should
* be used instead of xml_parser_create(), because PHP's XML parser doesn't check
* the input encoding itself.
*
* This is also where unsupported encodings will be converted.
* Callers should take this into account: $data might have been changed after
* the call.
*
* @param &$data
* The XML data which will be parsed later.
* @return
* An XML parser object.
*/
function drupal_xml_parser_create(&$data) {
// Default XML encoding is UTF-8
$encoding = 'utf-8';
$bom = false;
// Check for UTF-8 byte order mark (PHP5's XML parser doesn't handle it).
if (!strncmp($data, "\xEF\xBB\xBF", 3)) {
$bom = true;
$data = substr($data, 3);
}
// Check for an encoding declaration in the XML prolog if no BOM was found.
if (!$bom && ereg('^<\?xml[^>]+encoding="([^"]+)"', $data, $match)) {
$encoding = $match[1];
}
// Unsupported encodings are converted here into UTF-8.
$php_supported = array('utf-8', 'iso-8859-1', 'us-ascii');
if (!in_array(strtolower($encoding), $php_supported)) {
$out = drupal_convert_to_utf8($data, $encoding);
if ($out !== false) {
$data = $out;
$encoding = 'utf-8';
}
else {
watchdog('php', t("Could not convert XML encoding '%s' to UTF-8.", array('%s' => $encoding)), WATCHDOG_WARNING);
return 0;
}
}
$xml_parser = xml_parser_create($encoding);
xml_parser_set_option($xml_parser, XML_OPTION_TARGET_ENCODING, 'utf-8');
return $xml_parser;
}
/**
* Convert data to UTF-8
*
* Requires the iconv, GNU recode or mbstring PHP extension.
*
* @param $data
* The data to be converted.
* @param $encoding
* The encoding that the data is in
* @return
* Converted data or FALSE.
*/
function drupal_convert_to_utf8($data, $encoding) {
if (function_exists('iconv')) {
$out = @iconv($encoding, 'utf-8', $data);
}
else if (function_exists('mb_convert_encoding')) {
$out = @mb_convert_encoding($data, 'utf-8', $encoding);
}
else if (function_exists('recode_string')) {
$out = @recode_string($encoding .'..utf-8', $data);
}
else {
watchdog('php', t("Unsupported encoding '%s'. Please install iconv, GNU recode or mbstring for PHP.", array('%s' => $encoding)), WATCHDOG_ERROR);
return FALSE;
}
return $out;
}
/**
* Truncate a UTF-8-encoded string safely.
*
* If the end position is in the middle of a UTF-8 sequence, it scans backwards
* until the beginning of the byte sequence.
*
* Use this function whenever you want to chop off a string at an unsure
* location. On the other hand, if you're sure that you're splitting on a
* character boundary (e.g. after using strpos() or similar), you can safely use
* substr() instead.
*
* @param $string
* The string to truncate.
* @param $len
* An upper limit on the returned string length.
* @param $wordsafe
* Flag to truncate at nearest space. Defaults to FALSE.
* @return
* The truncated string.
*/
function truncate_utf8($string, $len, $wordsafe = FALSE, $dots = FALSE) {
$slen = strlen($string);
if ($slen <= $len) {
return $string;
}
if ($wordsafe) {
while (($string[--$len] != ' ') && ($len > 0)) {};
}
if ((ord($string[$len]) < 0x80) || (ord($string[$len]) >= 0xC0)) {
return substr($string, 0, $len) . ($dots ? ' ...' : '');
}
while (ord($string[--$len]) < 0xC0) {};
return substr($string, 0, $len) . ($dots ? ' ...' : '');
}
/**
* Encodes MIME/HTTP header values that contain non-ASCII, UTF-8 encoded
* characters.
*
* For example, mime_header_encode('tést.txt') returns "=?UTF-8?B?dMOpc3QudHh0?=".
*
* See http://www.rfc-editor.org/rfc/rfc2047.txt for more information.
*
* Notes:
* - Only encode strings that contain non-ASCII characters.
* - We progressively cut-off a chunk with truncate_utf8(). This is to ensure
* each chunk starts and ends on a character boundary.
* - Using \n as the chunk separator may cause problems on some systems and may
* have to be changed to \r\n or \r.
*/
function mime_header_encode($string) {
if (!preg_match('/^[\x20-\x7E]*$/', $string)) {
$chunk_size = 47; // floor((75 - strlen("=?UTF-8?B??=")) * 0.75);
$len = strlen($string);
$output = '';
while ($len > 0) {
$chunk = truncate_utf8($string, $chunk_size);
$output .= ' =?UTF-8?B?'. base64_encode($chunk) ."?=\n";
$c = strlen($chunk);
$string = substr($string, $c);
$len -= $c;
}
return trim($output);
}
return $string;
}
/**
* Decode all HTML entities (including numerical ones) to regular UTF-8 bytes.
* Double-escaped entities will only be decoded once ("&amp;lt;" becomes "&lt;", not "<").
*
* @param $text
* The text to decode entities in.
* @param $exclude
* An array of characters which should not be decoded. For example,
* array('<', '&', '"'). This affects both named and numerical entities.
*/
function decode_entities($text, $exclude = array()) {
static $table;
// We store named entities in a table for quick processing.
if (!isset($table)) {
// Get all named HTML entities.
$table = array_flip(get_html_translation_table(HTML_ENTITIES));
// PHP gives us ISO-8859-1 data, we need UTF-8.
$table = array_map('utf8_encode', $table);
// Add apostrophe (XML)
$table['&apos;'] = "'";
}
$newtable = array_diff($table, $exclude);
// Use a regexp to select all entities in one pass, to avoid decoding double-escaped entities twice.
return preg_replace('/&(#x?)?([A-Za-z0-9]+);/e', '_decode_entities("$1", "$2", "$0", $newtable, $exclude)', $text);
}
/**
* Helper function for decode_entities
*/
function _decode_entities($prefix, $codepoint, $original, &$table, &$exclude) {
// Named entity
if (!$prefix) {
if (isset($table[$original])) {
return $table[$original];
}
else {
return $original;
}
}
// Hexadecimal numerical entity
if ($prefix == '#x') {
$codepoint = base_convert($codepoint, 16, 10);
}
// Encode codepoint as UTF-8 bytes
if ($codepoint < 0x80) {
$str = chr($codepoint);
}
else if ($codepoint < 0x800) {
$str = chr(0xC0 | ($codepoint >> 6))
. chr(0x80 | ($codepoint & 0x3F));
}
else if ($codepoint < 0x10000) {
$str = chr(0xE0 | ( $codepoint >> 12))
. chr(0x80 | (($codepoint >> 6) & 0x3F))
. chr(0x80 | ( $codepoint & 0x3F));
}
else if ($codepoint < 0x200000) {
$str = chr(0xF0 | ( $codepoint >> 18))
. chr(0x80 | (($codepoint >> 12) & 0x3F))
. chr(0x80 | (($codepoint >> 6) & 0x3F))
. chr(0x80 | ( $codepoint & 0x3F));
}
// Check for excluded characters
if (in_array($str, $exclude)) {
return $original;
}
else {
return $str;
}
}
/**
* Count the amount of characters in a UTF-8 string. This is less than or
* equal to the byte count.
*/
function string_length(&$text) {
return strlen(preg_replace("/[\x80-\xBF]/", '', $text));
}
/**
* Evaluate a string of PHP code.
*
@ -2071,6 +1842,7 @@ function _drupal_bootstrap_full() {
require_once './includes/menu.inc';
require_once './includes/tablesort.inc';
require_once './includes/file.inc';
require_once './includes/unicode.inc';
require_once './includes/xmlrpc.inc';
require_once './includes/image.inc';
// Set the Drupal custom error handler.

2
includes/tablesort.inc
View File

@ -52,7 +52,7 @@ function tablesort_sql($header, $before = '') {
$ts = tablesort_init($header);
if ($ts['sql']) {
$sql = db_escape_string($ts['sql']);
$sort = strtoupper(db_escape_string($ts['sort']));
$sort = drupal_strtoupper(db_escape_string($ts['sort']));
return " ORDER BY $before $sql $sort";
}
}