<?php /*

 Composr
 Copyright (c) ocProducts, 2004-2016

 See text/EN/licence.txt for full licencing information.


 NOTE TO PROGRAMMERS:
   Do not edit this file. If you need to make changes, save your changed file to the appropriate *_custom folder
   **** If you ignore this advice, then your website upgrades (e.g. for bug fixes) will likely kill your changes ****

*/

/*EXTRA FUNCTIONS: shell_exec|fsockopen|ctype_xdigit*/

/**
 * @license    http://opensource.org/licenses/cpal_1.0 Common Public Attribution License
 * @copyright  ocProducts Ltd
 * @package    core
 */

/**
 * Standard code module initialisation function.
 *
 * @ignore
 */
function init__files2()
{
    require_code('files');

    global $HTTP_DOWNLOAD_MIME_TYPE, $HTTP_DOWNLOAD_SIZE, $HTTP_DOWNLOAD_URL, $HTTP_MESSAGE, $HTTP_MESSAGE_B, $HTTP_NEW_COOKIES, $HTTP_FILENAME, $HTTP_CHARSET, $HTTP_DOWNLOAD_MTIME;
    /** The mime type returned from the last file lookup.
     *
     * @global ?ID_TEXT $HTTP_DOWNLOAD_MIME_TYPE
     */
    $HTTP_DOWNLOAD_MIME_TYPE = null;
    /** The download size returned from the last file lookup.
     *
     * @global ?ID_TEXT $HTTP_DOWNLOAD_SIZE
     */
    $HTTP_DOWNLOAD_SIZE = null;
    /** The redirected URL for the last file lookup.
     *
     * @global ?ID_TEXT $HTTP_DOWNLOAD_URL
     */
    $HTTP_DOWNLOAD_URL = null;
    /** The file modification time returned from the last file lookup.
     *
     * @global ?ID_TEXT $HTTP_DOWNLOAD_MTIME
     */
    $HTTP_DOWNLOAD_MTIME = null;
    /** The status code returned from the last file lookup (e.g. "200" or "404").
     *
     * @global string $HTTP_MESSAGE
     */
    $HTTP_MESSAGE = null;
    $HTTP_MESSAGE_B = null;
    /** The cookies returned from the last file lookup.
     *
     * @global array $HTTP_NEW_COOKIES
     */
    $HTTP_NEW_COOKIES = array();
    /** The filename returned from the last file lookup.
     *
     * @global ?ID_TEXT $HTTP_FILENAME
     */
    $HTTP_FILENAME = null;
    /** The character set returned from the last file lookup.
     *
     * @global ?ID_TEXT $HTTP_CHARSET
     */
    $HTTP_CHARSET = null;
}

/**
 * Call a function, with inbuilt on-disk caching support.
 *
 * @param string $func Function to call
 * @param array $args Arguments to call with
 * @param ?integer $timeout Timeout in minutes (null: no timeout)
 * @return mixed The function result OR for http_download_file calls a tuple of result details
 */
function cache_and_carry($func, $args, $timeout = null)
{
    global $HTTP_DOWNLOAD_MIME_TYPE, $HTTP_DOWNLOAD_SIZE, $HTTP_DOWNLOAD_URL, $HTTP_MESSAGE, $HTTP_MESSAGE_B, $HTTP_NEW_COOKIES, $HTTP_FILENAME, $HTTP_CHARSET, $HTTP_DOWNLOAD_MTIME;

    $ret = mixed();

    $path = get_custom_file_base() . '/safe_mode_temp/' . md5(serialize($args)) . '.dat';
    if (is_file($path) && (($timeout === null) || (filemtime($path) > time() - $timeout * 60))) {
        $_ret = cms_file_get_contents_safe($path);
        if ($func == 'http_download_file') {
            $ret = @unserialize($_ret);
        } else {
            $ret = $_ret;
        }
    } else {
        $_ret = call_user_func_array($func, $args);
        require_code('files');
        if ($func == 'http_download_file') {
            $ret = array($_ret, $HTTP_DOWNLOAD_MIME_TYPE, $HTTP_DOWNLOAD_SIZE, $HTTP_DOWNLOAD_URL, $HTTP_MESSAGE, $HTTP_MESSAGE_B, $HTTP_NEW_COOKIES, $HTTP_FILENAME, $HTTP_CHARSET, $HTTP_DOWNLOAD_MTIME);
            cms_file_put_contents_safe($path, serialize($ret), FILE_WRITE_FAILURE_SILENT | FILE_WRITE_FIX_PERMISSIONS);
        } else {
            $ret = is_string($_ret) ? $_ret : serialize($_ret);
            cms_file_put_contents_safe($path, $ret, FILE_WRITE_FAILURE_SILENT | FILE_WRITE_FIX_PERMISSIONS);
        }
    }
    return $ret;
}

/**
 * Make a missing required directory, or exit with an error if we cannot (unless error suppression is on).
 *
 * @param PATH $dir Path to create
 * @return boolean Success status
 */
function make_missing_directory($dir)
{
    if (@mkdir($dir, 0777, true) === false) {
        if (error_reporting() == 0) {
            return false;
        }
        if (function_exists('do_lang_tempcode')) {
            warn_exit(do_lang_tempcode('WRITE_ERROR_DIRECTORY_REPAIR', escape_html($dir)));
        } else {
            warn_exit('Could not auto-create missing directory ' . htmlentities($dir));
        }
    }
    fix_permissions($dir);
    sync_file($dir);
    return true;
}

/**
 * Discern the cause of a file-write error, and show an appropriate error message.
 *
 * @param PATH $path File path that could not be written (full path, not relative)
 * @ignore
 */
function _intelligent_write_error($path)
{
    if (error_reporting() == 0) {
        return;
    }

    if (!function_exists('do_lang_tempcode')) {
        warn_exit('Could not write to ' . htmlentities($path));
    }

    if (file_exists($path)) {
        if (filesize($path) == 0) {
            return; // Probably was OR'd where 0 casted to false
        }

        warn_exit(do_lang_tempcode('WRITE_ERROR', escape_html($path)));
    } elseif (file_exists(dirname($path))) {
        if (strpos($path, '/templates_cached/') !== false) {
            critical_error('PASSON', do_lang('WRITE_ERROR_CREATE', escape_html($path), escape_html(dirname($path))));
        }
        warn_exit(do_lang_tempcode('WRITE_ERROR_CREATE', escape_html($path), escape_html(dirname($path))));
    } else {
        warn_exit(do_lang_tempcode('WRITE_ERROR_MISSING_DIRECTORY', escape_html(dirname($path)), escape_html(dirname(dirname($path)))));
    }
}

/**
 * Discern the cause of a file-write error, and return an appropriate error message.
 *
 * @param  PATH $path File path that could not be written
 * @return Tempcode Message
 *
 * @ignore
 */
function _intelligent_write_error_inline($path)
{
    if (file_exists($path)) {
        return do_lang_tempcode('WRITE_ERROR', escape_html($path));
    } elseif (file_exists(dirname($path))) {
        return do_lang_tempcode('WRITE_ERROR_CREATE', escape_html($path), escape_html(dirname($path)));
    }
    return do_lang_tempcode('WRITE_ERROR_MISSING_DIRECTORY', escape_html(dirname($path)), escape_html(dirname(dirname($path))));
}

/**
 * Find details of where we can save temporary files, taking into account PHP's platform-dependent difficulties.
 *
 * @return array A tuple: preferred temporary path to save to, whether there's a problem saving in the system path, the system path to save to, the local path to save to.
 */
function cms_get_temp_dir()
{
    $local_path = get_custom_file_base() . '/safe_mode_temp';
    if (!file_exists($local_path)) {
        make_missing_directory($local_path);
    }
    if (function_exists('sys_get_temp_dir')) {
        $server_path = sys_get_temp_dir();
    } else {
        $server_path = '/tmp';
    }
    $problem_saving = ((str_replace(array('on', 'true', 'yes'), array('1', '1', '1'), strtolower(ini_get('safe_mode'))) == '1') || (get_option('force_local_temp_dir') == '1') || ((@strval(ini_get('open_basedir')) != '') && (preg_match('#(^|:|;)' . preg_quote($server_path, '#') . '($|:|;|/)#', ini_get('open_basedir')) == 0)));
    $path = ($problem_saving ? $local_path : $server_path) . '/';
    return array($path, $problem_saving, $server_path, $local_path);
}

/**
 * Create file with unique file name, but works around compatibility issues between servers. Note that the file is NOT automatically deleted. You should also delete it using "@unlink", as some servers have problems with permissions.
 *
 * @param  string $prefix The prefix of the temporary file name.
 * @return ~string The name of the temporary file (false: error).
 *
 * @ignore
 */
function _cms_tempnam($prefix = '')
{
    list($tmp_path, $problem_saving, $server_path, $local_path) = cms_get_temp_dir();
    if (php_function_allowed('tempnam')) {
        // Create a real temporary file
        $tempnam = tempnam($tmp_path, 'tmpfile__' . $prefix);
        if ((($tempnam === false) || ($tempnam == ''/*Should not be blank, but seen in the wild*/)) && (!$problem_saving)) {
            $tempnam = tempnam($local_path, 'tmpfile__' . $prefix); // Try saving in local path even if we didn't think there'd be a problem saving into the system path
        }
    } else {
        // A fake temporary file, as true ones have been disabled on PHP
        require_code('crypt');
        $tempnam = 'tmpfile__' . $prefix . produce_salt();
        $myfile = fopen($local_path . '/' . $tempnam, 'wb');
        fclose($myfile);
        fix_permissions($local_path . '/' . $tempnam);
    }
    return $tempnam;
}

/**
 * Provides a hook for file synchronisation between mirrored servers. Called after any file creation, deletion or edit.
 *
 * @param  PATH $filename File/directory name to sync on (full path)
 * @ignore
 */
function _sync_file($filename)
{
    global $FILE_BASE, $_MODIFIED_FILES, $_CREATED_FILES;
    if (substr($filename, 0, strlen($FILE_BASE) + 1) == $FILE_BASE . '/') {
        $filename = substr($filename, strlen($FILE_BASE) + 1);
    }
    static $has_sync_script = null;
    if ($has_sync_script === null) {
        $has_sync_script = is_file($FILE_BASE . '/data_custom/sync_script.php');
    }
    if ($has_sync_script) {
        require_once($FILE_BASE . '/data_custom/sync_script.php');
        if (function_exists('master__sync_file')) {
            master__sync_file($filename);
        }
    }
    if (isset($_MODIFIED_FILES)) {
        foreach ($_MODIFIED_FILES as $i => $x) {
            if (($x == $FILE_BASE . '/' . $filename) || ($x == $filename)) {
                unset($_MODIFIED_FILES[$i]);
            }
        }
    }
    if (isset($_CREATED_FILES)) {
        foreach ($_CREATED_FILES as $i => $x) {
            if (($x == $FILE_BASE . '/' . $filename) || ($x == $filename)) {
                unset($_CREATED_FILES[$i]);
            }
        }
    }
}

/**
 * Provides a hook for file-move synchronisation between mirrored servers. Called after any rename or move action.
 *
 * @param  PATH $old File/directory name to move from (may be full or relative path)
 * @param  PATH $new File/directory name to move to (may be full or relative path)
 * @ignore
 */
function _sync_file_move($old, $new)
{
    global $FILE_BASE;
    if (is_file($FILE_BASE . '/data_custom/sync_script.php')) {
        require_once($FILE_BASE . '/data_custom/sync_script.php');
        if (substr($old, 0, strlen($FILE_BASE)) == $FILE_BASE) {
            $old = substr($old, strlen($FILE_BASE));
        }
        if (substr($new, 0, strlen($FILE_BASE)) == $FILE_BASE) {
            $new = substr($new, strlen($FILE_BASE));
        }
        if (function_exists('master__sync_file_move')) {
            master__sync_file_move($old, $new);
        }
    }
}

/**
 * Delete all the contents of a directory, and any subdirectories of that specified directory (recursively).
 *
 * @param  PATH $dir The pathname to the directory to delete
 * @param  boolean $default_preserve Whether to preserve files there by default
 * @param  boolean $just_files Whether to just delete files
 *
 * @ignore
 */
function _deldir_contents($dir, $default_preserve = false, $just_files = false)
{
    $current_dir = @opendir($dir);
    if ($current_dir !== false) {
        while (false !== ($entryname = readdir($current_dir))) {
            if ($default_preserve) {
                if ($entryname == 'index.html') {
                    continue;
                }
                if ($entryname[0] == '.') {
                    continue;
                }
                if (in_array(str_replace(get_file_base() . '/', '', $dir) . '/' . $entryname, array('uploads/banners/advertise_here.png', 'uploads/banners/donate.png', 'themes/map.ini', 'themes/default'))) {
                    continue;
                }
            }
            if ((is_dir($dir . '/' . $entryname)) && ($entryname != '.') && ($entryname != '..')) {
                deldir_contents($dir . '/' . $entryname, $default_preserve, $just_files);
                if (!$just_files) {
                    $test = @rmdir($dir . '/' . $entryname);
                    if (($test === false) && (!$just_files/*tolerate weird locked dirs if we only need to delete files anyways*/)) {
                        attach_message(do_lang_tempcode('WRITE_ERROR', escape_html($dir . '/' . $entryname)), 'warn');
                    }
                }
            } elseif (($entryname != '.') && ($entryname != '..')) {
                $test = @unlink($dir . '/' . $entryname);
                if ($test === false) {
                    attach_message(do_lang_tempcode('WRITE_ERROR', escape_html($dir . '/' . $entryname)), 'warn');
                }
            }
            sync_file($dir . '/' . $entryname);
        }
        closedir($current_dir);
    }
}

/**
 * Output data to a CSV file.
 *
 * @param  array $data List of maps, each map representing a row
 * @param  ID_TEXT $filename Filename to output
 * @param  boolean $headers Whether to output CSV headers
 * @param  boolean $output_and_exit Whether to output/exit when we're done instead of return
 * @param  ?PATH $outfile_path File to spool into (null: none)
 * @param  ?mixed $callback Callback for dynamic row insertion (null: none). Only implemented for the excel_support addon. Is passed: row just done, next row (or null), returns rows to insert
 * @param  ?array $metadata List of maps, each map representing metadata of a row; supports 'url' (null: none)
 * @return string CSV data (we might not return though, depending on $exit; if $outfile_path is not null, this will be blank)
 */
function make_csv($data, $filename = 'data.csv', $headers = true, $output_and_exit = true, $outfile_path = null, $callback = null, $metadata = null)
{
    if ($headers) {
        header('Content-Type: text/csv; charset=' . get_charset());
        header('Content-Disposition: attachment; filename="' . escape_header($filename, true) . '"');

        if (cms_srv('REQUEST_METHOD') == 'HEAD') {
            return '';
        }
    }

    $outfile = mixed();
    if (!is_null($outfile_path)) {
        $outfile = fopen($outfile_path, 'w+b');
        flock($outfile, LOCK_EX);
    }

    $out = '';

    if (get_charset() == 'utf-8') {
        $bom = chr(0xEF) . chr(0xBB) . chr(0xBF);
        //$out .= $bom; Shows as gibberish on Mac unless you explicitly import it with the correct settings
    }

    foreach ($data as $i => $line) {
        if ($i == 0) { // Header
            foreach (array_keys($line) as $j => $val) {
                if ($j != 0) {
                    $out .= ',';
                }
                $out .= '"' . str_replace('"', '""', $val) . '"';
            }
            $out .= "\n";
        }

        // Main data
        $j = 0;
        foreach ($line as $val) {
            if (is_null($val)) {
                $val = '';
            } elseif (!is_string($val)) {
                $val = strval($val);
            }
            if ($j != 0) {
                $out .= ',';
            }
            $out .= '"' . str_replace('"', '""', $val) . '"';
            $j++;
        }
        $out .= "\n";

        if (!is_null($outfile)) {
            fwrite($outfile, $out);
            $out = '';
        }
    }

    if ($output_and_exit) {
        $GLOBALS['SCREEN_TEMPLATE_CALLED'] = '';

        safe_ini_set('ocproducts.xss_detect', '0');

        if (!is_null($outfile)) {
            rewind($outfile);
            fpassthru($outfile);
            flock($outfile, LOCK_UN);
            fclose($outfile);
            @unlink($outfile_path);
        }
        exit($out);
    }

    if ($outfile !== null) {
        flock($outfile, LOCK_UN);
        fclose($outfile);
    }

    return $out;
}

/**
 * Delete a column from a CSV file.
 *
 * @param  PATH $in_path Path to the CSV file
 * @param  string $column_name Column name
 */
function delete_csv_column($in_path, $column_name)
{
    if (!is_writable_wrap($in_path)) {
        fatal_exit(do_lang_tempcode('WRITE_ERROR', $in_path));
    }

    // Find which field index this named column is
    $in_file = fopen($in_path, 'rb');
    flock($in_file, LOCK_SH);
    $header_row = fgetcsv($in_file);
    $column_i = null;
    foreach ($header_row as $i => $h) {
        if ($h == $column_name) {
            $column_i = $i;
            break;
        }
    }
    if (is_null($column_i)) {
        return;
    }

    // Rewrite out to a temp file
    $tmp_path = cms_tempnam();
    $tmp_file = fopen($tmp_path, 'wb');

    // Write out header
    unset($header_row[$i]);
    foreach ($header_row as $i => $h) {
        if ($i != 0) {
            fwrite($tmp_file, ',');
        }
        fwrite($tmp_file, str_replace('"', '""', $h));
    }
    fwrite($tmp_file, "\n");

    // Write out each row
    while (($row = fgetcsv($in_file)) !== false) {
        unset($row[$column_i]);

        foreach ($row as $i => $c) {
            if ($i != 0) {
                fwrite($tmp_file, ',');
            }
            fwrite($tmp_file, '"' . str_replace('"', '""', $c) . '"');
        }
        fwrite($tmp_file, "\n");
    }

    // Clean up; put temp file back over main file
    flock($in_file, LOCK_UN);
    fclose($in_file);
    fclose($tmp_file);
    @unlink($in_path);
    rename($tmp_path, $in_path);
    sync_file($in_path);
    fix_permissions($in_path);
}

/**
 * Find path to the PHP executable.
 *
 * @param  boolean $cgi Whether we need a CGI interpreter
 * @return PATH Path to PHP
 */
function find_php_path($cgi = false)
{
    $search_dirs = array(
        '/bin',
        '/usr/bin',
        '/usr/local/bin',
        '/usr/php/bin',
        '/usr/php/sbin',
        '/usr/php5/bin',
        '/usr/php5/sbin',
        '/usr/php6/bin',
        '/usr/php6/sbin',
        'c:\\php',
        'c:\\php5',
        'c:\\php6',
        'c:\\progra~1\\php',
        'c:\\progra~1\\php5',
        'c:\\progra~1\\php6',
    );
    $filenames = array(
        'php.dSYM',
        'php',
        'php5',
        'php6',
        'php-cli.dSYM',
        'php-cli',
        'php5-cli',
        'php6-cli',
        'php-cgi.dSYM',
        'php-cgi',
        'php5-cgi',
        'php6-cgi',
    );
    foreach ($search_dirs as $dir) {
        foreach ($filenames as $file) {
            if ((!$cgi) || (strpos($file, 'cgi') !== false)) {
                if (@file_exists($dir . '/' . $file)) {
                    break 2;
                }
            }
        }
    }
    if (!@file_exists($dir . '/' . $file)) {
        $php_path = $cgi ? 'php-cgi' : 'php';
    } else {
        $php_path = $dir . '/' . $file;
    }
    return $php_path;
}

/**
 * Get the contents of a directory, recursively. It is assumed that the directory exists.
 *
 * @param  PATH $path The path to search
 * @param  PATH $rel_path The path we prepend to everything we find (intended to be used inside the recursion)
 * @param  boolean $special_too Whether to also get special files
 * @param  boolean $recurse Whether to recurse (if not, will return directories as files)
 * @param  boolean $files_wanted Whether to get files (if not, will return directories as files)
 * @return array The contents of the directory
 */
function get_directory_contents($path, $rel_path = '', $special_too = false, $recurse = true, $files_wanted = true)
{
    $out = array();

    require_code('files');

    $d = @opendir($path);
    if ($d === false) {
        return array();
    }
    while (($file = readdir($d)) !== false) {
        if (!$special_too) {
            if (should_ignore_file($rel_path . (($rel_path == '') ? '' : '/') . $file, IGNORE_ACCESS_CONTROLLERS)) {
                continue;
            }
        } elseif (($file == '.') || ($file == '..')) {
            continue;
        }

        $is_file = is_file($path . '/' . $file);
        if (($is_file) || (!$recurse)) {
            if (($files_wanted) || (!$is_file)) {
                $out[] = $rel_path . (($rel_path == '') ? '' : '/') . $file;
            }
        } elseif (is_dir($path . '/' . $file)) {
            if (!$files_wanted) {
                $out[] = $rel_path . (($rel_path == '') ? '' : '/') . $file;
            }
            $out = array_merge($out, get_directory_contents($path . '/' . $file, $rel_path . (($rel_path == '') ? '' : '/') . $file, $special_too, $recurse, $files_wanted));
        }
    }
    closedir($d);

    return $out;
}

/**
 * Get the size in bytes of a directory. It is assumed that the directory exists.
 *
 * @param  PATH $path The path to search
 * @param  boolean $recurse Whether to recurse (if not, will return directories as files)
 * @return integer The extra space requested
 */
function get_directory_size($path, $recurse = true)
{
    $size = 0;

    $d = @opendir($path);
    if ($d === false) {
        return 0;
    }
    while (($e = readdir($d)) !== false) {
        if (($e == '.') || ($e == '..')) {
            continue;
        }

        if (is_file($path . '/' . $e)) {
            $size += filesize($path . '/' . $e);
        } elseif (is_dir($path . '/' . $e)) {
            if ($recurse) {
                $size += get_directory_size($path . '/' . $e, $recurse);
            }
        }
    }

    return $size;
}

/**
 * Get the URL to the config option group for editing limits
 *
 * @return ?URLPATH The URL to the config option group for editing limits (null: no access)
 */
function get_upload_limit_config_url()
{
    $config_url = null;
    if (has_actual_page_access(get_member(), 'admin_config')) {
        $_config_url = build_url(array('page' => 'admin_config', 'type' => 'category', 'id' => 'SITE'), get_module_zone('admin_config'));
        $config_url = $_config_url->evaluate();
        $config_url .= '#group_UPLOAD';
    }
    return $config_url;
}

/**
 * Get the maximum allowed upload filesize, as specified in the configuration
 *
 * @param  ?MEMBER $source_member Member we consider quota for (null: do not consider quota)
 * @param  ?object $connection Database connection to get quota from (null: site DB)
 * @param  boolean $consider_php_limits Whether to consider limitations in PHP's configuration
 * @return integer The maximum allowed upload filesize, in bytes
 */
function get_max_file_size($source_member = null, $connection = null, $consider_php_limits = true)
{
    $possibilities = array();

    require_code('files');
    $a = php_return_bytes(ini_get('upload_max_filesize'));
    $b = GOOGLE_APPENGINE ? 0 : php_return_bytes(ini_get('post_max_size'));
    $c = intval(get_option('max_download_size')) * 1024;
    if (has_privilege(get_member(), 'exceed_filesize_limit')) {
        $c = 0;
    }

    $d = mixed();
    if ((!is_null($source_member)) && (!has_privilege(get_member(), 'exceed_filesize_limit'))) { // We'll be considering quota also
        if (get_forum_type() == 'cns') {
            require_code('cns_groups');
            $daily_quota = cns_get_member_best_group_property($source_member, 'max_daily_upload_mb');
        } else {
            $daily_quota = 5; // 5 is a hard-coded default for non-Conversr forums
        }
        if (is_null($connection)) {
            $connection = $GLOBALS['SITE_DB'];
        }
        $_size_uploaded_today = $connection->query('SELECT SUM(a_file_size) AS the_answer FROM ' . $connection->get_table_prefix() . 'attachments WHERE a_member_id=' . strval($source_member) . ' AND a_add_time>' . strval(time() - 60 * 60 * 24) . ' AND a_add_time<=' . strval(time()));
        $size_uploaded_today = intval($_size_uploaded_today[0]['the_answer']);
        $d = max(0, $daily_quota * 1024 * 1024 - $size_uploaded_today);
    }

    if ($consider_php_limits) {
        if ($a != 0) {
            $possibilities[] = $a;
        }
        if ($b != 0) {
            $possibilities[] = $b;
        }
    }
    if ($c != 0) {
        $possibilities[] = $c;
    }
    if ($d !== null) {
        $possibilities[] = $d;
    }

    return (count($possibilities) == 0) ? (1024 * 1024 * 1024 * 1024) : min($possibilities);
}

/**
 * Check uploaded file extensions for possible malicious intent, and if some is found, an error is put out, and the hackattack logged.
 *
 * @param  string $name The filename
 * @param  boolean $skip_server_side_security_check Whether to skip the server side security check
 * @param  ?string $file_to_delete Delete this file if we have to exit (null: no file to delete)
 * @param  boolean $accept_errors Whether to allow errors without dying
 * @return boolean Success status
 */
function check_extension($name, $skip_server_side_security_check = false, $file_to_delete = null, $accept_errors = false)
{
    $ext = get_file_extension($name);
    $_types = get_option('valid_types');
    $types = array_flip(explode(',', $_types));
    $_types = '';
    ksort($types);
    if (!$skip_server_side_security_check) {
        if (!has_privilege(get_member(), 'use_very_dangerous_comcode')) {
            unset($types['js']);
            unset($types['swf']);
            unset($types['html']);
            unset($types['htm']);
            unset($types['shtml']);
            unset($types['svg']);
            unset($types['xml']);
        }
    }
    foreach (array_flip($types) as $val) {
        $_types .= $val . ',';
    }
    $_types = substr($_types, 0, strlen($_types) - 1);
    if (!$skip_server_side_security_check) {
        if (($ext == 'py') || ($ext == 'fcgi') || ($ext == 'yaws') || ($ext == 'dll') || ($ext == 'cgi') || ($ext == 'cfm') || ($ext == 'vbs') || ($ext == 'rhtml') || ($ext == 'rb') || ($ext == 'pl') || ($ext == 'phtml') || ($ext == 'php') || ($ext == 'php3') || ($ext == 'php4') || ($ext == 'php5') || ($ext == 'php6') || ($ext == 'phtml') || ($ext == 'aspx') || ($ext == 'ashx') || ($ext == 'asmx') || ($ext == 'asx') || ($ext == 'axd') || ($ext == 'asp') || ($ext == 'aspx') || ($ext == 'jsp') || ($ext == 'sh') || ($ext == 'cgi') || (strtolower($name) == '.htaccess')) {
            if (!is_null($file_to_delete)) {
                unlink($file_to_delete);
            }
            if ($accept_errors) {
                return false;
            }
            log_hack_attack_and_exit('SCRIPT_UPLOAD_HACK');
        }
    }
    if ($_types != '') {
        $types = explode(',', $_types);
        foreach ($types as $val) {
            if (strtolower(trim($val)) == $ext) {
                return true;
            }
        }
        if (!is_null($file_to_delete)) {
            unlink($file_to_delete);
        }
        $message = do_lang_tempcode('INVALID_FILE_TYPE', escape_html($ext), escape_html(str_replace(',', ', ', $_types)));
        if (has_actual_page_access(get_member(), 'admin_config')) {
            $_link = build_url(array('page' => 'admin_config', 'type' => 'category', 'id' => 'SECURITY'), get_module_zone('admin_config'));
            $link = $_link->evaluate();
            $link .= '#group_UPLOAD';
            $message = do_lang_tempcode('INVALID_FILE_TYPE_ADMIN', escape_html($ext), escape_html(str_replace(',', ', ', $_types)), escape_html($link));
        }
        if ($accept_errors) {
            require_code('site');
            attach_message($message, 'warn');
            return false;
        } else {
            warn_exit($message);
        }
    }

    return true;
}

/**
 * Delete an uploaded file from disk, if it's URL has changed (i.e. it's been replaced, leaving a redundant disk file).
 * This MUST be run before the edit/delete operation, as it scans for the existing value to know what is changing.
 *
 * @param  string $upload_path The path to the upload directory
 * @param  ID_TEXT $table The table name
 * @param  ID_TEXT $field The table field name
 * @param  mixed $id_field The table ID field name, or a map array
 * @param  mixed $id The table ID
 * @param  ?string $new_url The new URL to use (null: deleting without replacing: no change check)
 */
function delete_upload($upload_path, $table, $field, $id_field, $id, $new_url = null)
{
    // Try and delete the file
    if (($GLOBALS['FORUM_DRIVER']->is_staff(get_member())) || (get_option('cleanup_files') == '1')) { // This isn't really a permission - more a failsafe in case there is a security hole. Staff can cleanup leftover files from the Cleanup module anyway. NB: Also repeated in cms_galleries.php.
        $where = is_array($id_field) ? $id_field : array($id_field => $id);
        $url = $GLOBALS['SITE_DB']->query_select_value_if_there($table, $field, $where);
        if (empty($url)) {
            return;
        }

        if ((is_null($new_url)) || (($url != $new_url) && ($new_url != STRING_MAGIC_NULL))) {
            if ((url_is_local($url)) && (substr($url, 0, strlen($upload_path) + 1) == $upload_path . '/')) {
                $count = $GLOBALS['SITE_DB']->query_select_value($table, 'COUNT(*)', array($field => $url));

                if ($count <= 1) {
                    @unlink(get_custom_file_base() . '/' . rawurldecode($url));
                    sync_file(rawurldecode($url));
                }
            }
            if ((url_is_local($url)) && (substr($url, 0, strlen('themes/default/images_custom') + 1) == 'themes/default/images_custom/')) {
                require_code('themes2');
                tidy_theme_img_code($new_url, $url, $table, $field, $GLOBALS['SITE_DB']);
            }
        }
    }
}

/**
 * Check bandwidth usage against page view ratio for shared hosting.
 *
 * @param  integer $extra The extra bandwidth requested
 */
function check_shared_bandwidth_usage($extra)
{
    global $SITE_INFO;
    if (!empty($SITE_INFO['throttle_bandwidth_registered'])) {
        $views_till_now = intval(get_value('page_views'));
        $bandwidth_allowed = $SITE_INFO['throttle_bandwidth_registered'];
        $total_bandwidth = intval(get_value('download_bandwidth'));
        if ($bandwidth_allowed * 1024 * 1024 >= $total_bandwidth + $extra) {
            return;
        }
    }
    if (!empty($SITE_INFO['throttle_bandwidth_complementary'])) {
        // $timestamp_start = $SITE_INFO['custom_user_'] . current_share_user(); Actually we'll do by views
        // $days_till_now = (time() - $timestamp_start) / (24 * 60 * 60);
        $views_till_now = intval(get_value('page_views'));
        $bandwidth_allowed = $SITE_INFO['throttle_bandwidth_complementary'] + $SITE_INFO['throttle_bandwidth_views_per_meg'] * $views_till_now;
        $total_bandwidth = intval(get_value('download_bandwidth'));
        if ($bandwidth_allowed * 1024 * 1024 < $total_bandwidth + $extra) {
            critical_error('RELAY', 'The hosted user has exceeded their shared-hosting "bandwidth-limit to page-view" ratio. More pages must be viewed before this may be downloaded.');
        }
    }
}

/**
 * Check disk space usage against page view ratio for shared hosting.
 *
 * @param  integer $extra The extra space in bytes requested
 */
function check_shared_space_usage($extra)
{
    global $SITE_INFO;
    if (!empty($SITE_INFO['throttle_space_registered'])) {
        $views_till_now = intval(get_value('page_views'));
        $bandwidth_allowed = $SITE_INFO['throttle_space_registered'];
        $total_space = get_directory_size(get_custom_file_base() . '/uploads');
        if ($bandwidth_allowed * 1024 * 1024 >= $total_space + $extra) {
            return;
        }
    }
    if (!empty($SITE_INFO['throttle_space_complementary'])) {
        // $timestamp_start = $SITE_INFO['custom_user_'] . current_share_user(); Actually we'll do by views
        // $days_till_now = (time() - $timestamp_start) / (24 * 60 * 60);
        $views_till_now = intval(get_value('page_views'));
        $space_allowed = $SITE_INFO['throttle_space_complementary'] + $SITE_INFO['throttle_space_views_per_meg'] * $views_till_now;
        $total_space = get_directory_size(get_custom_file_base() . '/uploads');
        if ($space_allowed * 1024 * 1024 < $total_space + $extra) {
            critical_error('RELAY', 'The hosted user has exceeded their shared-hosting "disk-space to page-view" ratio. More pages must be viewed before this may be uploaded.');
        }
    }
}

/**
 * Return the file in the URL by downloading it over HTTP. If a byte limit is given, it will only download that many bytes. It outputs warnings, returning null, on error.
 *
 * @param  URLPATH $url The URL to download
 * @param  ?integer $byte_limit The number of bytes to download. This is not a guarantee, it is a minimum (null: all bytes)
 * @range  1 max
 * @param  boolean $trigger_error Whether to throw a Composr error, on error
 * @param  boolean $no_redirect Whether to block redirects (returns null when found)
 * @param  string $ua The user-agent to identify as
 * @param  ?array $post_params An optional array of POST parameters to send; if this is null, a GET request is used (null: none). If $raw_post is set, it should be array($data)
 * @param  ?array $cookies An optional array of cookies to send (null: none)
 * @param  ?string $accept 'accept' header value (null: don't pass one)
 * @param  ?string $accept_charset 'accept-charset' header value (null: don't pass one)
 * @param  ?string $accept_language 'accept-language' header value (null: don't pass one)
 * @param  ?resource $write_to_file File handle to write to (null: do not do that)
 * @param  ?string $referer The HTTP referer (null: none)
 * @param  ?array $auth A pair: authentication username and password (null: none)
 * @param  float $timeout The timeout
 * @param  boolean $raw_post Whether to treat the POST parameters as a raw POST (rather than using MIME)
 * @param  ?array $files Files to send. Map between field to file path (null: none)
 * @param  ?array $extra_headers Extra headers to send (null: none)
 * @param  ?string $http_verb HTTP verb (null: auto-decide based on other parameters)
 * @param  string $raw_content_type The content type to use if a raw HTTP post
 * @return ?string The data downloaded (null: error)
 * @ignore
 */
function _http_download_file($url, $byte_limit = null, $trigger_error = true, $no_redirect = false, $ua = 'Composr', $post_params = null, $cookies = null, $accept = null, $accept_charset = null, $accept_language = null, $write_to_file = null, $referer = null, $auth = null, $timeout = 6.0, $raw_post = false, $files = null, $extra_headers = null, $http_verb = null, $raw_content_type = 'application/xml')
{
    // Normalise the URL
    require_code('urls');
    $url = str_replace(' ', '%20', $url);
    if (url_is_local($url)) {
        $url = get_custom_base_url() . '/' . $url;
    }
    if ((strpos($url, '/') !== false) && (strrpos($url, '/') < 7)) {
        $url .= '/';
    }

    // Initialisation
    global $DOWNLOAD_LEVEL, $HTTP_DOWNLOAD_MIME_TYPE, $HTTP_CHARSET, $HTTP_DOWNLOAD_SIZE, $HTTP_DOWNLOAD_URL, $HTTP_DOWNLOAD_MTIME, $HTTP_MESSAGE, $HTTP_MESSAGE_B, $HTTP_NEW_COOKIES, $HTTP_FILENAME;
    $DOWNLOAD_LEVEL++;
    $HTTP_DOWNLOAD_MIME_TYPE = null;
    $HTTP_CHARSET = null;
    $HTTP_DOWNLOAD_SIZE = 0;
    $HTTP_DOWNLOAD_URL = $url;
    $HTTP_DOWNLOAD_MTIME = null;
    $HTTP_MESSAGE = null;
    $HTTP_MESSAGE_B = null;
    if ($DOWNLOAD_LEVEL == 0) {
        $HTTP_NEW_COOKIES = array();
    }
    $HTTP_FILENAME = null;

    static $has_ctype_xdigit = null;
    if ($has_ctype_xdigit === null) {
        $has_ctype_xdigit = function_exists('ctype_xdigit');
    }

    // Prevent DOS loop attack
    if (cms_srv('HTTP_USER_AGENT') == $ua) {
        $ua = 'Composr-recurse';
    }
    if (cms_srv('HTTP_USER_AGENT') == 'Composr-recurse') {
        return null;
    }
    if ($DOWNLOAD_LEVEL == 8) {
        return '';
    }//critical_error('FILE_DOS', $url);

    // Work out what we'll be doing
    $url_parts = @parse_url($url);
    if ($url_parts === false || !isset($url_parts['host'])) {
        if ($trigger_error) {
            warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_BAD_URL', escape_html($url)));
        } else {
            $HTTP_MESSAGE_B = do_lang_tempcode('HTTP_DOWNLOAD_BAD_URL', escape_html($url));
        }
        $DOWNLOAD_LEVEL--;
        $HTTP_MESSAGE = 'malconstructed-URL';
        return null;
    }
    if (!array_key_exists('host', $url_parts)) {
        $url_parts['host'] = '127.0.0.1';
    }
    $connect_to = $url_parts['host'];
    $base_url_parsed = parse_url(get_base_url());
    if (!array_key_exists('host', $base_url_parsed)) {
        $base_url_parsed['host'] = '127.0.0.1';
    }
    $config_ip_forwarding = function_exists('get_option') ? get_option('ip_forwarding') : '';
    $do_ip_forwarding = ($base_url_parsed['host'] == $connect_to) && ($config_ip_forwarding != '') && ($config_ip_forwarding != '0');
    if ($do_ip_forwarding) { // For cases where we have IP-forwarding, and a strong firewall (i.e. blocked to our own domain's IP by default)
        if ($config_ip_forwarding == '1') {
            $connect_to = cms_srv('LOCAL_ADDR');
            if ($connect_to == '') {
                $connect_to = cms_srv('SERVER_ADDR');
            }
            if ($connect_to == '') {
                $connect_to = '127.0.0.1'; // "localhost" can fail due to IP6
            }
        } else {
            $protocol_end_pos = strpos($config_ip_forwarding, '://');
            if ($protocol_end_pos !== false) {
                $url = preg_replace('#^(https?://)#', substr($config_ip_forwarding, 0, $protocol_end_pos + 3), $url);
                $config_ip_forwarding = substr($config_ip_forwarding, $protocol_end_pos + 3);
            }
            $connect_to = $config_ip_forwarding;
        }
    } elseif ((php_function_allowed('gethostbyname')) && ($url_parts['scheme'] == 'http')) {
        $connect_to = @gethostbyname($connect_to); // for DNS caching
    }
    if (!array_key_exists('scheme', $url_parts)) {
        $url_parts['scheme'] = 'http';
    }

    $_url = preg_replace('#^(https?://)' . preg_quote($url_parts['host'], '#') . '([/:]|$)#', '${1}' . $connect_to . '${2}', $url);

    // File-system/shell_exec method, for local calls
    $faux = function_exists('get_value') ? get_value('http_faux_loopback') : null;
    if ((!is_null($faux)) && ($faux != '') && (is_null($post_params)) && (is_null($files))) { // NB: Does not support cookies, accept headers, referers
        if (substr($faux, 0, 1) != '#') {
            $faux = '#' . $faux . '#i';
        }
        if (preg_match($faux, $url) != 0) {
            $parsed = parse_url($url);
            $parsed_base_url = parse_url(get_custom_base_url());
            $file_base = get_custom_file_base();
            $file_base = preg_replace('#' . preg_quote(urldecode($parsed_base_url['path'])) . '$#', '', $file_base);
            $file_path = $file_base . urldecode($parsed['path']);

            if ((php_function_allowed('escapeshellcmd')) && (php_function_allowed('shell_exec')) && (substr($file_path, -4) == '.php')) {
                $cmd = 'DOCUMENT_ROOT=' . escapeshellarg_wrap(dirname(get_file_base())) . ' PATH_TRANSLATED=' . escapeshellarg_wrap($file_path) . ' SCRIPT_NAME=' . escapeshellarg_wrap($file_path) . ' HTTP_USER_AGENT=' . escapeshellarg_wrap($ua) . ' QUERY_STRING=' . escapeshellarg_wrap($parsed['query']) . ' HTTP_HOST=' . escapeshellarg_wrap($parsed['host']) . ' ' . escapeshellcmd(find_php_path(true)) . ' ' . escapeshellarg_wrap($file_path);
                $contents = shell_exec($cmd);
                $split_pos = strpos($contents, "\r\n\r\n");
                if ($split_pos !== false) {
                    $_headers = explode("\r\n", substr($contents, 0, $split_pos));
                    foreach ($_headers as $line) {
                        _read_in_headers($line);
                    }
                    $contents = substr($contents, $split_pos + 4);
                }
            } else {
                if ($trigger_error) {
                    $contents = file_get_contents($file_path);
                } else {
                    $contents = @file_get_contents($file_path);
                }

                require_code('mime_types');
                $HTTP_DOWNLOAD_MIME_TYPE = get_mime_type(get_file_extension($file_path), true);
                $HTTP_DOWNLOAD_SIZE = filesize($file_path);
                $HTTP_DOWNLOAD_MTIME = filemtime($file_path);
                $HTTP_MESSAGE = '200';
                $HTTP_FILENAME = basename($file_path);
            }

            if ($byte_limit !== null) {
                $contents = substr($contents, 0, $byte_limit);
            }

            if (!is_null($write_to_file)) {
                fwrite($write_to_file, $contents);
                return '';
            }
            $DOWNLOAD_LEVEL--;
            return _detect_character_encoding($contents);
        }
    }

    $use_curl = (($url_parts['scheme'] != 'http') || ((function_exists('get_value')) && (get_value('prefer_curl') === '1'))) && (function_exists('curl_version'));
    if ((function_exists('get_value')) && (get_value('prefer_curl') === '0')) {
        $use_curl = false;
    }

    $raw_payload = ''; // Note that this will contain HTTP headers (it is appended directly after headers with no \r\n between -- so it contains \r\n\r\n itself when the content body is going to start)
    $raw_payload_curl = '';
    $sent_http_post_content = false;
    $put = mixed();
    $put_path = mixed();
    $put_no_delete = false;
    if ((!is_null($post_params)) || ($raw_post) || (count($files) != 0)) {
        if (is_null($post_params)) {
            $post_params = array(); // POST is implied
        }

        if ($raw_post) {
            $_postdetails_params = $post_params[0];
        } else {
            $_postdetails_params = '';//$url_parts['scheme'].'://'.$url_parts['host'].$url2.'?';
            if (array_keys($post_params) == array('_')) {
                $_postdetails_params = $post_params['_'];
            } else {
                if (count($post_params) > 0) {
                    $_postdetails_params .= http_build_query($post_params);
                }
            }
        }

        if (is_null($files)) { // If no files, use simple application/x-www-form-urlencoded
            if ($raw_post) {
                if (!isset($extra_headers['Content-Type'])) {
                    $raw_payload .= 'Content-Type: ' . $raw_content_type . "\r\n";
                }
            } else {
                $raw_payload .= 'Content-Type: application/x-www-form-urlencoded; charset=' . get_charset() . "\r\n";
            }
            $raw_payload .= 'Content-Length: ' . strval(strlen($_postdetails_params)) . "\r\n";
            $raw_payload .= "\r\n";

            $raw_payload .= $_postdetails_params;
            $raw_payload_curl = $_postdetails_params; // Other settings will be passed via cURL itself
            $sent_http_post_content = true;
        } else { // If files, use more complex multipart/form-data
            if (strtolower($http_verb) == 'put') {
                $put_no_delete = (count($post_params) == 0) && (count($files) == 1); // Can we just use the one referenced file as a direct PUT
                if ($put_no_delete) { // Yes
                    reset($files);
                    $put_path = current($files);
                    $put = fopen($put_path, 'rb');
                } else { // No, we need to spool out HTTP blah to make a new file to PUT
                    $put_path = cms_tempnam();
                    $put = fopen($put_path, 'wb');
                }
            }

            $divider = uniqid('', true);
            $raw_payload2 = '';
            if (($put === null) || (count($post_params) != 0) || (count($files) != 1)) {
                $raw_payload .= 'Content-Type: multipart/form-data; boundary="--cms' . $divider . '"; charset=' . get_charset() . "\r\n";
            }
            foreach ($post_params as $key => $val) {
                $raw_payload2 .= '----cms' . $divider . "\r\n";
                if ($raw_post) {
                    if (!isset($extra_headers['Content-Type'])) {
                        $raw_payload2 .= 'Content-Type: ' . $raw_content_type . "\r\n\r\n";
                    }
                } else {
                    $raw_payload2 .= 'Content-Disposition: form-data; name="' . urlencode($key) . '"' . "\r\n\r\n";
                }
                $raw_payload2 .= $val . "\r\n";
            }
            if ($put !== null && !$put_no_delete) {
                fwrite($put, $raw_payload2);
                $raw_payload2 = '';
            }
            foreach ($files as $upload_field => $file_path) {
                if (($put === null) || (count($post_params) != 0) || (count($files) != 1)) {
                    $raw_payload2 .= '----cms' . $divider . "\r\n";
                    if (strpos($upload_field, '/') === false) {
                        $raw_payload2 .= 'Content-Disposition: form-data; name="' . str_replace('"', '\"', $upload_field) . '"; filename="' . urlencode(basename($file_path)) . '"' . "\r\n";
                        $raw_payload2 .= 'Content-Type: application/octet-stream' . "\r\n\r\n";
                    } else {
                        $raw_payload2 .= 'Content-Type: ' . $upload_field . "\r\n\r\n";
                    }
                } else {
                    if ((strpos($upload_field, '/') === false) && ($raw_content_type == '')) {
                        $raw_content_type = $upload_field;
                    }
                }
                if ($put !== null && !$put_no_delete) {
                    fwrite($put, $raw_payload2);
                    $raw_payload2 = '';
                }
                if ($put !== null && !$put_no_delete) {
                    $myfile = fopen($file_path, 'rb');
                    while (!feof($myfile)) {
                        $data = @fread($myfile, 1024 * 100);
                        if (($data !== false) && ($data !== null)) {
                            fwrite($put, $data);
                        } else {
                            break;
                        }
                    }
                    @fclose($myfile);
                } else {
                    $raw_payload2 .= file_get_contents($file_path);
                }
                if (($put === null) || (count($post_params) != 0) || (count($files) != 1)) {
                    $raw_payload2 .= "\r\n";
                }
                if ($put !== null && !$put_no_delete) {
                    fwrite($put, $raw_payload2);
                    $raw_payload2 = '';
                }
            }
            if (($put === null) || (count($post_params) != 0) || (count($files) != 1)) {
                $raw_payload2 .= '----cms' . $divider . "--\r\n";
            }
            if ($put !== null && !$put_no_delete) {
                fwrite($put, $raw_payload2);
                $raw_payload2 = '';
            }
            if ($put !== null) {
                $raw_payload .= 'Content-Length: ' . strval(filesize($put_path)) . "\r\n";
            } else {
                $raw_payload .= 'Content-Length: ' . strval(strlen($raw_payload2)) . "\r\n";
            }
            $raw_payload .= "\r\n" . $raw_payload2;

            $raw_payload_curl = $raw_payload2; // Other settings will be passed via cURL itself
        }
    }

    // Prep cookies
    if ((!is_null($cookies)) && (count($cookies) != 0)) {
        $_cookies = '';
        $done_one_cookie = false;
        foreach ($cookies as $key => $val) {
            if ($done_one_cookie) {
                $_cookies .= '; ';
            }
            if (is_array($val)) {
                foreach ($val as $key2 => $val2) {
                    if (!is_string($key2)) {
                        $key2 = strval($key2);
                    }
                    if ($done_one_cookie) {
                        $_cookies .= '; ';
                    }
                    $_cookies .= $key . '[' . $key2 . ']=' . rawurlencode($val2);
                    $done_one_cookie = true;
                }
            } else {
                $_cookies .= $key . '=' . rawurlencode($val);
            }
            $done_one_cookie = true;
        }
    }

    if (is_null($http_verb)) {
        $http_verb = (((is_null($post_params)) && (is_null($files))) ? (($byte_limit === 0) ? 'HEAD' : 'GET') : 'POST');
    }

    // CURL method
    if ($use_curl) {
        if (function_exists('curl_version')) {
            if (function_exists('curl_init')) {
                if (function_exists('curl_setopt')) {
                    if (function_exists('curl_exec')) {
                        if (function_exists('curl_error')) {
                            if (function_exists('curl_close')) {
                                if (function_exists('curl_getinfo')) {
                                    if (($url_parts['scheme'] == 'https') || ($url_parts['scheme'] == 'http')) {
                                        $curl_version = curl_version();
                                        if (((is_string($curl_version)) && (strpos($curl_version, 'OpenSSL') !== false)) || ((is_array($curl_version)) && (array_key_exists('ssl_version', $curl_version)))) {
                                            $ch = curl_init($do_ip_forwarding ? $_url : $url);
                                            $curl_headers = array();
                                            if ((!is_null($cookies)) && (count($cookies) != 0)) {
                                                curl_setopt($ch, CURLOPT_COOKIE, $_cookies);
                                            }
                                            $crt_path = get_file_base() . '/data/curl-ca-bundle.crt';
                                            curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, !((function_exists('get_value')) && (get_value('disable_ssl_for__' . $url_parts['host']) === '1')));
                                            if (ini_get('curl.cainfo') == '') {
                                                curl_setopt($ch, CURLOPT_CAINFO, $crt_path);
                                                curl_setopt($ch, CURLOPT_CAPATH, $crt_path);
                                            }
                                            if (defined('CURL_SSLVERSION_TLSv1')) {
                                                curl_setopt($ch, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
                                            } else {
                                                if ((!is_array($curl_version)) || (!isset($curl_version['ssl_version'])) || (strpos($curl_version['ssl_version'], 'NSS') === false) || (version_compare($curl_version['version'], '7.36.0') >= 0)) {
                                                    curl_setopt($ch, CURLOPT_SSL_CIPHER_LIST, 'TLSv1');
                                                } else {
                                                    curl_setopt($ch, CURLOPT_SSLVERSION, 1); // the above fails on old NSS, so we use numeric equivalent to the CURL_SSLVERSION_TLSv1 constant here
                                                }
                                            }
                                            if ($do_ip_forwarding) {
                                                curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
                                                curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
                                            }
                                            //if (!$no_redirect) @curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); // May fail with safe mode, meaning we can't follow Location headers. But we can do better ourselves anyway and protect against file:// exploits.
                                            curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, intval($timeout));
                                            curl_setopt($ch, CURLOPT_TIMEOUT, intval($timeout));
                                            curl_setopt($ch, CURLOPT_USERAGENT, $ua);
                                            if ($http_verb == 'HEAD') {
                                                curl_setopt($ch, CURLOPT_NOBODY, true); // Branch needed as doing a HEAD via CURLOPT_CUSTOMREQUEST can cause a timeout bug in cURL
                                            } else {
                                                curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $http_verb);
                                            }
                                            if (!is_null($accept)) {
                                                $curl_headers[] = 'Accept: ' . $accept;
                                            }
                                            if (!is_null($accept_charset)) {
                                                $curl_headers[] = 'Accept-Charset: ' . $accept_charset;
                                            }
                                            if (!is_null($accept_language)) {
                                                $curl_headers[] = 'Accept-Language: ' . $accept_language;
                                            }
                                            if (!is_null($extra_headers)) {
                                                foreach ($extra_headers as $key => $val) {
                                                    $curl_headers[] = $key . ': ' . $val;
                                                }
                                            }
                                            if (($raw_post) && ((is_null($files)) || ($put !== null))) {
                                                if (!isset($extra_headers['Content-Type'])) {
                                                    $curl_headers[] = 'Content-Type: ' . $raw_content_type;
                                                }
                                            }
                                            if (!is_null($post_params)) {
                                                $curl_headers[] = 'Expect:';
                                                if ($put !== null) {
                                                    fclose($put);
                                                    $put = fopen($put_path, 'rb');
                                                    curl_setopt($ch, CURLOPT_PUT, true);
                                                    curl_setopt($ch, CURLOPT_INFILE, $put);
                                                    curl_setopt($ch, CURLOPT_INFILESIZE, filesize($put_path));
                                                } else {
                                                    curl_setopt($ch, CURLOPT_POST, true);
                                                    curl_setopt($ch, CURLOPT_POSTFIELDS, $raw_payload_curl);
                                                    if (!is_null($files)) {
                                                        $curl_headers[] = 'Content-Type: multipart/form-data; boundary="--cms' . $divider . '"; charset=' . get_charset();
                                                    }
                                                }
                                            }
                                            if ($do_ip_forwarding) {
                                                $curl_headers[] = 'Host: ' . $url_parts['host'] . "\r\n";
                                            }
                                            if ((count($curl_headers) != 0) && ((is_null($files)/*Breaks file uploads for some reason*/) || (!is_null($extra_headers)))) {
                                                if (defined('CURLINFO_HEADER_OUT')) {
                                                    curl_setopt($ch, CURLINFO_HEADER_OUT, true);
                                                }
                                                curl_setopt($ch, CURLOPT_HTTPHEADER, $curl_headers);
                                            }
                                            curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
                                            curl_setopt($ch, CURLOPT_HEADER, true); // include header in output
                                            if (!is_null($auth)) {
                                                curl_setopt($ch, CURLOPT_USERPWD, implode(':', $auth));
                                            }
                                            if (!is_null($referer)) {
                                                curl_setopt($ch, CURLOPT_REFERER, $referer);
                                            }
                                            $proxy = function_exists('get_option') ? get_option('proxy') : '';
                                            if (($proxy != '') && ($url_parts['host'] != 'localhost') && ($url_parts['host'] != '127.0.0.1')) {
                                                $port = get_option('proxy_port');
                                                curl_setopt($ch, CURLOPT_PROXY, $proxy . ':' . $port);
                                                $proxy_user = get_option('proxy_user');
                                                if ($proxy_user != '') {
                                                    $proxy_password = get_option('proxy_password');
                                                    curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxy_user . ':' . $proxy_password);
                                                }
                                            }
                                            if ($byte_limit !== null) {
                                                curl_setopt($ch, CURLOPT_RANGE, '0-' . strval(($byte_limit == 0) ? 0 : ($byte_limit - 1)));
                                            }
                                            $line = curl_exec($ch);
                                            /*if ((count($curl_headers)!=0) && ((!is_null($files)))) { // Useful for debugging
                                                var_dump(curl_getinfo($ch,CURLINFO_HEADER_OUT));exit();
                                            }*/
                                            if ($line === false) {
                                                $error = curl_error($ch);
                                                curl_close($ch);
                                            } else {
                                                if (substr($line, 0, 25) == "HTTP/1.1 100 Continue\r\n\r\n") {
                                                    $line = substr($line, 25);
                                                }
                                                if (substr($line, 0, 25) == "HTTP/1.0 100 Continue\r\n\r\n") {
                                                    $line = substr($line, 25);
                                                }
                                                $HTTP_DOWNLOAD_MIME_TYPE = curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
                                                $HTTP_DOWNLOAD_SIZE = curl_getinfo($ch, CURLINFO_CONTENT_LENGTH_DOWNLOAD);
                                                $HTTP_DOWNLOAD_URL = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
                                                $HTTP_MESSAGE = strval(curl_getinfo($ch, CURLINFO_HTTP_CODE));
                                                if ($HTTP_MESSAGE == '206') {
                                                    $HTTP_MESSAGE = '200'; // We don't care about partial-content return code, as Composr implementation gets ranges differently and we check '200' as a return result
                                                }
                                                if (strpos($HTTP_DOWNLOAD_MIME_TYPE, ';') !== false) {
                                                    $HTTP_CHARSET = substr($HTTP_DOWNLOAD_MIME_TYPE, 8 + strpos($HTTP_DOWNLOAD_MIME_TYPE, 'charset='));
                                                    $HTTP_DOWNLOAD_MIME_TYPE = substr($HTTP_DOWNLOAD_MIME_TYPE, 0, strpos($HTTP_DOWNLOAD_MIME_TYPE, ';'));
                                                }
                                                curl_close($ch);
                                                if (substr($line, 0, strlen('HTTP/1.0 200 Connection Established')) == 'HTTP/1.0 200 Connection Established') {
                                                    $line = substr($line, strpos($line, "\r\n\r\n") + 4);
                                                }
                                                $pos = strpos($line, "\r\n\r\n");

                                                if (substr($line, 0, strlen('HTTP/1.1 100 ')) == 'HTTP/1.1 100 ' || substr($line, 0, strlen('HTTP/1.0 100 ')) == 'HTTP/1.0 100 ') {
                                                    $pos = strpos($line, "\r\n\r\n", $pos + 4);
                                                }
                                                if ($pos === false) {
                                                    $pos = strlen($line);
                                                } else {
                                                    $pos += 4;
                                                }
                                                $lines = explode("\r\n", substr($line, 0, $pos));

                                                foreach ($lines as $lno => $_line) {
                                                    $_line .= "\r\n";
                                                    $matches = array();

                                                    if (preg_match('#^Content-Disposition: [^;]*;\s*filename="([^"]*)"#i', $_line, $matches) != 0) {
                                                        _read_in_headers($_line);
                                                    }
                                                    if (preg_match("#^Set-Cookie: ([^\r\n=]*)=([^\r\n]*)\r\n#i", $_line, $matches) != 0) {
                                                        _read_in_headers($_line);
                                                    }
                                                    if (preg_match("#^Location: (.*)\r\n#i", $_line, $matches) != 0) {
                                                        if (is_null($HTTP_FILENAME)) {
                                                            $HTTP_FILENAME = urldecode(basename($matches[1]));
                                                        }

                                                        if (strpos($matches[1], '://') === false) {
                                                            $matches[1] = qualify_url($matches[1], $url);
                                                        }
                                                        $HTTP_DOWNLOAD_URL = $matches[1];
                                                        if (($matches[1] != $url) && (preg_match('#^3\d\d$#', $HTTP_MESSAGE) != 0)) {
                                                            $bak = $HTTP_FILENAME;
                                                            $combined_cookies = collapse_2d_complexity('key', 'value', $HTTP_NEW_COOKIES) + (($cookies === null) ? array() : $cookies);
                                                            $text = $no_redirect ? mixed() : _http_download_file($matches[1], $byte_limit, $trigger_error, false, $ua, null, $combined_cookies, $accept, $accept_charset, $accept_language, $write_to_file, $referer, $auth, $timeout, $raw_post, $files, $extra_headers, $http_verb, $raw_content_type);
                                                            if (is_null($HTTP_FILENAME)) {
                                                                $HTTP_FILENAME = $bak;
                                                            }
                                                            $DOWNLOAD_LEVEL--;
                                                            if ($put !== null) {
                                                                fclose($put);
                                                                if (!$put_no_delete) {
                                                                    @unlink($put_path);
                                                                }
                                                            }
                                                            if (!is_null($text)) {
                                                                if (!is_null($write_to_file)) {
                                                                    fwrite($write_to_file, $text);
                                                                    $text = '';
                                                                }
                                                            }
                                                            return _detect_character_encoding($text);
                                                        }
                                                    }
                                                }

                                                $DOWNLOAD_LEVEL--;
                                                if ($put !== null) {
                                                    fclose($put);
                                                    if (!$put_no_delete) {
                                                        @unlink($put_path);
                                                    }
                                                }
                                                $text = substr($line, $pos);
                                                if (!is_null($write_to_file)) {
                                                    fwrite($write_to_file, $text);
                                                    $text = '';
                                                }
                                                return _detect_character_encoding($text);
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }

    // Direct sockets method
    $headers = '';
    if ((!is_null($cookies)) && (count($cookies) != 0)) {
        $headers .= 'Cookie: ' . $_cookies . "\r\n";
    }
    $headers .= 'User-Agent: ' . rawurlencode($ua) . "\r\n";
    if (!is_null($auth)) {
        $headers .= 'Authorization: Basic ' . base64_encode(implode(':', $auth)) . "==\r\n";
    }
    if (!is_null($extra_headers)) {
        foreach ($extra_headers as $key => $val) {
            $headers .= $key . ': ' . rawurlencode($val) . "\r\n";
        }
    }
    if (!is_null($accept)) {
        $headers .= 'Accept: ' . rawurlencode($accept) . "\r\n";
    } else {
        $headers .= "Accept: */*\r\n"; // There's a mod_security rule that checks for this
    }
    if (!is_null($accept_charset)) {
        $headers .= 'Accept-Charset: ' . rawurlencode($accept_charset) . "\r\n";
    }
    if (!is_null($accept_language)) {
        $headers .= 'Accept-Language: ' . rawurlencode($accept_language) . "\r\n";
    }
    if (!is_null($referer)) {
        $headers .= 'Referer: ' . rawurlencode($referer) . "\r\n";
    }
    $errno = 0;
    $errstr = '';
    if (($url_parts['scheme'] == 'http') && (!GOOGLE_APPENGINE) && (php_function_allowed('fsockopen'))) {
        $proxy = function_exists('get_option') ? get_option('proxy') : null;
        if ($proxy == '') {
            $proxy = null;
        }
        if ((!is_null($proxy)) && ($connect_to != 'localhost') && ($connect_to != '127.0.0.1')) {
            $proxy_port = get_option('proxy_port');
            $mysock = @fsockopen($proxy, intval($proxy_port), $errno, $errstr, $timeout);
        } else {
            $mysock = @fsockopen($connect_to, array_key_exists('port', $url_parts) ? $url_parts['port'] : 80, $errno, $errstr, $timeout);
        }
        if (is_null($mysock)) {
            $mysock = false; // For Quercus #4549
        }
    } else {
        $mysock = false; // Can't handle SSL here, so let it flow onto the PHP stream wrappers implementation
    }
    if ($mysock !== false) {
        if (function_exists('stream_set_timeout')) {
            if (@stream_set_timeout($mysock, intval($timeout)) === false) {
                $mysock = false;
            }
        } elseif (function_exists('socket_set_timeout')) {
            if (@socket_set_timeout($mysock, intval($timeout)) === false) {
                $mysock = false;
            }
        }
    }
    if ($mysock !== false) {
        $url2 = array_key_exists('path', $url_parts) ? $url_parts['path'] : '/';
        if (array_key_exists('query', $url_parts)) {
            $url2 .= '?' . $url_parts['query'];
        }

        if (($proxy != '') && ($connect_to != 'localhost') && ($connect_to != '127.0.0.1')) {
            $out = '';
            $out .= $http_verb . ' ' . escape_header($url) . " HTTP/1.1\r\n";
            $proxy_user = get_option('proxy_user');
            if ($proxy_user != '') {
                $proxy_password = get_option('proxy_password');
                $out .= 'Proxy-Authorization: Basic ' . base64_encode($proxy_user . ':' . $proxy_password) . "\r\n";
            }
        } else {
            $out = ((is_null($post_params)) ? (($byte_limit === 0) ? 'HEAD ' : 'GET ') : 'POST ') . escape_header($url) . " HTTP/1.1\r\n";
        }
        $out .= 'Host: ' . $url_parts['host'] . "\r\n";
        $out .= $headers;
        $out .= $raw_payload;
        if (!$sent_http_post_content) {
            $out .= 'Connection: Close' . "\r\n\r\n";
        }
        @fwrite($mysock, $out);
        if ($put !== null) {
            rewind($put);
            while (!feof($put)) {
                $data = @fread($put, 1024 * 100);
                if (($data !== false) && ($data !== null)) {
                    @fwrite($mysock, $data);
                } else {
                    break;
                }
            }
        }
        $data_started = false;
        $input = '';
        $input_len = 0;
        $first_fail_time = mixed();
        $chunked = false;
        $buffer_unprocessed = '';
        while (($chunked) || (!@feof($mysock))) { // @'d because socket might have died. If so fread will will return false and hence we'll break
            $line = @fread($mysock, 32000);
            if ($line === false) {
                if ((!$chunked) || ($buffer_unprocessed == '')) {
                    break;
                }
                $line = '';
            }
            if ($line == '') {
                if ($first_fail_time !== null) {
                    if ($first_fail_time < time() - 5) {
                        break;
                    }
                } else {
                    $first_fail_time = time();
                }
            } else {
                $first_fail_time = null;
            }
            if ($data_started) {
                $line = $buffer_unprocessed . $line;
                $buffer_unprocessed = '';

                if ($chunked) {
                    if (isset($line[1]) && $line[0] == "\r" && $line[1] == "\n") {
                        $line = substr($line, 2);
                    }

                    $hexdec_chunk_details = '';
                    $chunk_line_length = strlen($line);
                    for ($hexdec_read = 0; $hexdec_read < $chunk_line_length; $hexdec_read++) {
                        $chunk_char = $line[$hexdec_read];
                        if ($chunk_char == "\r") {
                            $chunk_char_is_hex = false;
                        } else {
                            if ($has_ctype_xdigit) {
                                $chunk_char_is_hex = ctype_xdigit($chunk_char);
                            } else {
                                $chunk_char_ord = ord($chunk_char);
                                $chunk_char_is_hex = ($chunk_char_ord >= 48 && $chunk_char_ord <= 57 || $chunk_char_ord >= 65 && $chunk_char_ord <= 90 || $chunk_char_ord >= 97 && $chunk_char_ord <= 122);
                            }
                        }
                        if ($chunk_char_is_hex) {
                            $hexdec_chunk_details .= $chunk_char;
                        } else {
                            break;
                        }
                    }
                    if ($hexdec_chunk_details == '') { // No data
                        continue;
                    }
                    $chunk_end_pos = strpos($line, "\r\n");
                    if ($chunk_end_pos === false) {
                        $buffer_unprocessed = $line;
                        continue;
                    }
                    $amount_wanted = hexdec($hexdec_chunk_details);
                    $amount_available = $chunk_line_length - ($chunk_end_pos + 2);
                    if ($amount_available < $amount_wanted) { // Chunk was more than what we grabbed, so we need to iterate more (more fread) to parse
                        $buffer_unprocessed = $line;
                        continue;
                    }
                    $buffer_unprocessed = substr($line, $chunk_end_pos + 2 + $amount_wanted); // May be some more extra read
                    $line = substr($line, $chunk_end_pos + 2, $amount_wanted);
                    if ($line == '') {
                        break; // Terminating chunk
                    }

                    $input_len += $amount_wanted;
                } else {
                    $input_len += strlen($line);
                }

                if ($write_to_file === null) {
                    $input .= $line;
                } else {
                    fwrite($write_to_file, $line);
                }

                if (($byte_limit !== null) && ($input_len >= $byte_limit)) {
                    $input = substr($input, 0, $byte_limit);
                    break;
                }
            } elseif ($line != '') {
                $old_line = $line;
                $lines = explode("\r\n", $line);

                $tally = 0;
                foreach ($lines as $lno => $line) {
                    $line .= "\r\n";

                    $tally += strlen($line);

                    $matches = array();
                    if (preg_match("#Transfer-Encoding: chunked\r\n#i", $line, $matches) != 0) {
                        $chunked = true;
                    }
                    _read_in_headers($line);
                    if (preg_match("#^Refresh: (\d*);(.*)\r\n#i", $line, $matches) != 0) {
                        if (is_null($HTTP_FILENAME)) {
                            $HTTP_FILENAME = urldecode(basename($matches[1]));
                        }

                        @fclose($mysock);
                        if (strpos($matches[1], '://') === false) {
                            $matches[1] = qualify_url($matches[1], $url);
                        }
                        $bak = $HTTP_FILENAME;
                        $combined_cookies = collapse_2d_complexity('key', 'value', $HTTP_NEW_COOKIES) + (($cookies === null) ? array() : $cookies);
                        $text = $no_redirect ? null : _http_download_file($matches[2], $byte_limit, $trigger_error, false, $ua, null, $combined_cookies, $accept, $accept_charset, $accept_language, $write_to_file, $referer, $auth, $timeout, $raw_post, $files, $extra_headers, $http_verb, $raw_content_type);
                        if ($no_redirect) {
                            $HTTP_DOWNLOAD_URL = $matches[2];
                        }
                        if (is_null($HTTP_FILENAME)) {
                            $HTTP_FILENAME = $bak;
                        }
                        $DOWNLOAD_LEVEL--;
                        if ($put !== null) {
                            fclose($put);
                            if (!$put_no_delete) {
                                @unlink($put_path);
                            }
                        }
                        return _detect_character_encoding($text);
                    }
                    if (preg_match("#^Location: (.*)\r\n#i", $line, $matches) != 0) {
                        if (is_null($HTTP_FILENAME)) {
                            $HTTP_FILENAME = urldecode(basename($matches[1]));
                        }

                        @fclose($mysock);
                        if (strpos($matches[1], '://') === false) {
                            $matches[1] = qualify_url($matches[1], $url);
                        }
                        if ($matches[1] != $url) {
                            $bak = $HTTP_FILENAME;
                            $HTTP_DOWNLOAD_URL = $matches[1];
                            if (($matches[1] != $url) && (preg_match('#^3\d\d$#', $HTTP_MESSAGE) != 0)) {
                                $combined_cookies = collapse_2d_complexity('key', 'value', $HTTP_NEW_COOKIES) + (($cookies === null) ? array() : $cookies);
                                $text = $no_redirect ? null : _http_download_file($matches[1], $byte_limit, $trigger_error, false, $ua, null, $combined_cookies, $accept, $accept_charset, $accept_language, $write_to_file, $referer, $auth, $timeout, $raw_post, $files, $extra_headers, $http_verb, $raw_content_type);
                                if (is_null($HTTP_FILENAME)) {
                                    $HTTP_FILENAME = $bak;
                                }
                                $DOWNLOAD_LEVEL--;
                                if ($put !== null) {
                                    fclose($put);
                                    if (!$put_no_delete) {
                                        @unlink($put_path);
                                    }
                                }
                                return _detect_character_encoding($text);
                            }
                        }
                    }
                    if (preg_match("#HTTP/(\d*\.\d*) (\d*) #", $line, $matches) != 0) {
                        // 200=Ok
                        // 301/302/307=Redirected: Not good, we should not be here
                        // 401/403=Unauthorized
                        // 404=Not found
                        // 400/500=Internal error
                        // 405=Method not allowed

                        $HTTP_MESSAGE = $matches[2];
                        switch ($matches[2]) {
                            case '301':
                            case '302':
                            case '307':
                                // We'll expect a location header
                                break;
                            case '200':
                                // Good
                                break;
                            case '401':
                            case '403':
                                if ($trigger_error) {
                                    warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNAUTHORIZED', escape_html($url)));
                                } else {
                                    $HTTP_MESSAGE_B = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNAUTHORIZED', escape_html($url));
                                }
                                @fclose($mysock);
                                $HTTP_DOWNLOAD_MIME_TYPE = 'security';
                                $DOWNLOAD_LEVEL--;
                                if ($put !== null) {
                                    fclose($put);
                                    if (!$put_no_delete) {
                                        @unlink($put_path);
                                    }
                                }
                                return null;
                            case '404':
                                if ($trigger_error) {
                                    warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_NOT_FOUND', escape_html($url)));
                                } else {
                                    $HTTP_MESSAGE_B = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_NOT_FOUND', escape_html($url));
                                }
                                @fclose($mysock);
                                $DOWNLOAD_LEVEL--;
                                if ($put !== null) {
                                    fclose($put);
                                    if (!$put_no_delete) {
                                        @unlink($put_path);
                                    }
                                }
                                return null;
                            case '400':
                            case '500':
                                if ($trigger_error) {
                                    warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_SERVER_ERROR', escape_html($url)));
                                } else {
                                    $HTTP_MESSAGE_B = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_SERVER_ERROR', escape_html($url));
                                }
                                @fclose($mysock);
                                $DOWNLOAD_LEVEL--;
                                if ($put !== null) {
                                    fclose($put);
                                    if (!$put_no_delete) {
                                        @unlink($put_path);
                                    }
                                }
                                return null;
                            case '405':
                                if ($byte_limit == 0 && !$no_redirect && empty($post_params)) { // Try again as non-HEAD request if we just did a HEAD request that got "Method not allowed"
                                    $combined_cookies = collapse_2d_complexity('key', 'value', $HTTP_NEW_COOKIES) + (($cookies === null) ? array() : $cookies);
                                    $text = _http_download_file($url, 1, $trigger_error, false, $ua, $post_params, $combined_cookies, $accept, $accept_charset, $accept_language, $write_to_file, $referer, $auth, $timeout, $raw_post, $files, $extra_headers, $http_verb, $raw_content_type);
                                    $DOWNLOAD_LEVEL--;
                                    if ($put !== null) {
                                        fclose($put);
                                        if (!$put_no_delete) {
                                            @unlink($put_path);
                                        }
                                    }
                                    return _detect_character_encoding($text);
                                }
                            default:
                                if ($trigger_error) {
                                    warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNKNOWN', escape_html($url), escape_html($matches[2])));
                                } else {
                                    $HTTP_MESSAGE_B = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNKNOWN', escape_html($url), escape_html($matches[2]));
                                }
                        }
                    }
                    if ($line == "\r\n") {
                        $data_started = true;
                        $buffer_unprocessed = substr($old_line, $tally);
                        if ($buffer_unprocessed === false) {
                            $buffer_unprocessed = '';
                        }
                        break;
                    }
                }
            }
        }

        // Process any non-chunked extra buffer (chunked would have been handled in main loop)
        if (!$chunked) {
            if ($buffer_unprocessed != '') {
                if ($write_to_file === null) {
                    $input .= $buffer_unprocessed;
                } else {
                    fwrite($write_to_file, $buffer_unprocessed);
                }
                $input_len += strlen($buffer_unprocessed);
                if (($byte_limit !== null) && ($input_len >= $byte_limit)) {
                    $input = substr($input, 0, $byte_limit);
                }
            }
        }

        @fclose($mysock);
        if (!$data_started) {
            if ($byte_limit === 0) {
                return '';
            }

            if ($trigger_error) {
                warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_NO_SERVER', escape_html($url)));
            } else {
                $HTTP_MESSAGE_B = do_lang_tempcode('HTTP_DOWNLOAD_NO_SERVER', escape_html($url));
            }
            $DOWNLOAD_LEVEL--;
            $HTTP_MESSAGE = 'no-data';
            if ($put !== null) {
                fclose($put);
                if (!$put_no_delete) {
                    @unlink($put_path);
                }
            }
            return null;
        }
        $size_expected = $HTTP_DOWNLOAD_SIZE;
        if ($byte_limit !== null) {
            if ($byte_limit < $HTTP_DOWNLOAD_SIZE) {
                $size_expected = $byte_limit;
            }
        }
        if ($input_len < $size_expected) {
            if ($trigger_error) {
                warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_CUT_SHORT', escape_html($url), escape_html(integer_format($size_expected)), escape_html(integer_format($input_len))));
            } else {
                $HTTP_MESSAGE_B = do_lang_tempcode('HTTP_DOWNLOAD_CUT_SHORT', escape_html($url), escape_html(integer_format($size_expected)), escape_html(integer_format($input_len)));
            }
            $DOWNLOAD_LEVEL--;
            $HTTP_MESSAGE = 'short-data';
            if ($put !== null) {
                fclose($put);
                if (!$put_no_delete) {
                    @unlink($put_path);
                }
            }
            return _detect_character_encoding($input);
        }

        $DOWNLOAD_LEVEL--;

        if ($put !== null) {
            fclose($put);
            if (!$put_no_delete) {
                @unlink($put_path);
            }
        }
        return _detect_character_encoding($input);
    } else {
        // PHP streams method
        if (($errno != 110) && (($errno != 10060) || (@ini_get('default_socket_timeout') == '1')) && ((@ini_get('allow_url_fopen')) || (php_function_allowed('ini_set')))) {
            // Perhaps fsockopen is restricted... try fread/file_get_contents
            safe_ini_set('allow_url_fopen', '1');
            $timeout_before = @ini_get('default_socket_timeout');
            safe_ini_set('default_socket_timeout', strval(intval($timeout)));
            if ($put !== null) {
                $raw_payload .= file_get_contents($put_path);
            }
            $crt_path = get_file_base() . '/data/curl-ca-bundle.crt';
            $last_headers = stripos($raw_payload, "\r\n\r\n");
            if ($last_headers !== false) {
                $headers .= substr($raw_payload, 0, $last_headers + 2);
                $raw_payload = substr($raw_payload, $last_headers + 4);
            }
            $opts = array(
                'http' => array(
                    'method' => $http_verb,
                    'header' => rtrim((($url_parts['host'] != $connect_to) ? ('Host: ' . $url_parts['host'] . "\r\n") : '') . $headers),
                    'user_agent' => $ua,
                    'content' => $raw_payload,
                    'follow_location' => $no_redirect ? 0 : 1,
                    //'ignore_errors' => true, // Useful when debugging
                ),
                'ssl' => array(
                    'verify_peer' => !$do_ip_forwarding,
                    'cafile' => $crt_path,
                    'SNI_enabled' => true,
                    'ciphers' => 'TLSv1',
                ),
            );
            $proxy = function_exists('get_option') ? get_option('proxy') : '';
            if ($proxy != '') {
                $port = get_option('proxy_port');
                $proxy_user = get_option('proxy_user');
                if ($proxy_user != '') {
                    $proxy_password = get_option('proxy_password');
                    $opts['http']['proxy'] = 'tcp://' . $proxy_user . ':' . $proxy_password . '@' . $proxy . ':' . $port;
                } else {
                    $opts['http']['proxy'] = 'tcp://' . $proxy . ':' . $port;
                }
            }
            $context = stream_context_create($opts);
            $php_errormsg = mixed();
            if (($byte_limit === null) && (is_null($write_to_file))) {
                if ($trigger_error) {
                    global $SUPPRESS_ERROR_DEATH;
                    $bak = $SUPPRESS_ERROR_DEATH;
                    $SUPPRESS_ERROR_DEATH = true; // Errors will be attached instead. We don't rely on only $php_errormsg because stream errors don't go into that fully.
                    $read_file = file_get_contents($_url, false, $context);
                    $SUPPRESS_ERROR_DEATH = $bak;
                } else {
                    $read_file = @file_get_contents($_url, false, $context);
                }
            } else {
                if ($trigger_error) {
                    global $SUPPRESS_ERROR_DEATH;
                    $bak = $SUPPRESS_ERROR_DEATH;
                    $SUPPRESS_ERROR_DEATH = true; // Errors will be attached instead. We don't rely on only $php_errormsg because stream errors don't go into that fully.
                    $_read_file = fopen($_url, 'rb', false, $context);
                    $SUPPRESS_ERROR_DEATH = $bak;
                } else {
                    $_read_file = @fopen($_url, 'rb', false, $context);
                }
                if ($_read_file !== false) {
                    $read_file = '';
                    while ((!feof($_read_file)) && (($byte_limit === null) || (strlen($read_file) < $byte_limit))) {
                        $read_file .= fread($_read_file, 1024);
                        if (!is_null($write_to_file)) {
                            fwrite($write_to_file, $read_file);
                            $read_file = '';
                        }
                    }
                    fclose($_read_file);
                } else {
                    $read_file = false;
                }
            }
            if (($byte_limit !== null) && ($read_file !== false) && ($read_file != ''/*substr would fail with false*/)) {
                $read_file = substr($read_file, 0, $byte_limit);
            }
            safe_ini_set('allow_url_fopen', '0');
            safe_ini_set('default_socket_timeout', $timeout_before);
            if (isset($http_response_header)) {
                foreach ($http_response_header as $header) {
                    _read_in_headers($header . "\r\n");
                }
            }
            if ($read_file !== false) {
                $DOWNLOAD_LEVEL--;
                if ($put !== null) {
                    fclose($put);
                    if (!$put_no_delete) {
                        @unlink($put_path);
                    }
                }
                return $read_file;
            }
        }

        $errstr = @strval($php_errormsg);
        if ($trigger_error) {
            if ($errstr == '') {
                $errstr = strval($errno);
            }
            $error = do_lang_tempcode('_HTTP_DOWNLOAD_NO_SERVER', escape_html($url), escape_html($errstr));
            warn_exit($error);
        } else {
            $HTTP_MESSAGE_B = do_lang_tempcode('HTTP_DOWNLOAD_NO_SERVER', escape_html($url));
        }
        $DOWNLOAD_LEVEL--;
        $HTTP_MESSAGE = 'could not connect to host'; // Could append  (' . $errstr . ') but only when debugging because we use this string like a constant in some places
        if ($put !== null) {
            fclose($put);
            if (!$put_no_delete) {
                @unlink($put_path);
            }
        }
        return null;
    }
}

/**
 * Read in any HTTP headers from an HTTP line, that we probe for.
 *
 * @param  string $line The line
 *
 * @ignore
 */
function _read_in_headers($line)
{
    global $HTTP_DOWNLOAD_MIME_TYPE, $HTTP_DOWNLOAD_SIZE, $HTTP_DOWNLOAD_URL, $HTTP_MESSAGE, $HTTP_MESSAGE_B, $HTTP_NEW_COOKIES, $HTTP_FILENAME, $HTTP_CHARSET, $HTTP_DOWNLOAD_MTIME;

    $matches = array();
    if (preg_match("#Content-Disposition: [^\r\n]*filename=\"([^;\r\n]*)\"\r\n#i", $line, $matches) != 0) {
        $HTTP_FILENAME = $matches[1];
    }
    if (preg_match("#^Set-Cookie: ([^\r\n=]*)=([^\r\n]*)\r\n#i", $line, $matches) != 0) {
        $cookie_key = trim(rawurldecode($matches[1]));

        $cookie_value = trim($matches[2]);
        $_cookie_parts = explode('; ', $cookie_value);

        $cookie_parts = array();

        $cookie_parts['key'] = $cookie_key;
        $cookie_parts['value'] = trim(rawurldecode(array_shift($_cookie_parts)));

        foreach ($_cookie_parts as $i => $part) {
            $temp = explode('=', $part, 2);
            if (array_key_exists(1, $temp)) {
                $cookie_parts[trim($temp[0])] = trim(rawurldecode($temp[1]));
            }
        }
        if (function_exists('get_cookie_domain')) {
            $cookie_parts['domain'] = get_cookie_domain();
        }

        $HTTP_NEW_COOKIES[$cookie_key] = $cookie_parts;
    }
    if (preg_match("#^Content-Length: ([^;\r\n]*)\r\n#i", $line, $matches) != 0) {
        $HTTP_DOWNLOAD_SIZE = intval($matches[1]);
    }
    if (preg_match("#^Last-Modified: ([^;\r\n]*)\r\n#i", $line, $matches) != 0) {
        $HTTP_DOWNLOAD_MTIME = strtotime($matches[1]);
    }
    if (preg_match("#^Content-Type: ([^;\r\n]*)(;[^\r\n]*)?\r\n#i", $line, $matches) != 0) {
        $HTTP_DOWNLOAD_MIME_TYPE = $matches[1];
        if (array_key_exists(2, $matches)) {
            $_ct_more = explode(';', str_replace(' ', '', trim($matches[2])));
            foreach ($_ct_more as $ct_more) {
                $ct_bits = explode('=', $ct_more, 2);
                if ((count($ct_bits) == 2) && (strtolower($ct_bits[0]) == 'charset')) {
                    $HTTP_CHARSET = trim($ct_bits[1]);
                }
            }
        }
    }
}

/**
 * Extract meta details from a URL.
 *
 * @param  URLPATH $url Webpage URL
 * @return array A map of meta details extracted from the webpage
 */
function get_webpage_meta_details($url)
{
    static $cache = array();

    if (array_key_exists($url, $cache)) {
        return $cache[$url];
    }
    if (get_param_integer('keep_oembed_cache', 1) == 1) {
        $_meta_details = $GLOBALS['SITE_DB']->query_select('url_title_cache', array('*'), array('t_url' => $url), '', 1);
        if (array_key_exists(0, $_meta_details)) {
            $meta_details = $_meta_details[0];
            $cache[$url] = $meta_details;
            return $meta_details;
        }
    }

    $meta_details = array(
        't_url' => substr($url, 0, 255),
        't_title' => '',
        't_meta_title' => '',
        't_keywords' => '',
        't_description' => '',
        't_image_url' => '',
        't_mime_type' => '',
        't_json_discovery' => '',
        't_xml_discovery' => '',
    );

    if (url_is_local($url)) {
        $url = get_custom_base_url() . '/' . $url;
    }

    if (!looks_like_url($url)) {
        return $meta_details;
    }

    $result = cache_and_carry('http_download_file', array($url, 1024 * 10, false, false, 'Composr', null, null, null, null, null, null, null, null, 2.0));
    if ((is_array($result)) && ($result[1] !== null) && (strpos($result[1], 'html') !== false) && $result[4] == '200') {
        $html = $result[0];

        // In ascending precedence
        $headers = array(
            't_title' => array(
                '<title[^>]*>\s*(.*)\s*</title>',
            ),
            't_meta_title' => array(
                '<meta\s+name="?DC\.Title"?\s+content="?([^"<>]*)"?\s*/?' . '>',
                '<meta\s+name="?twitter:title"?\s+content="?([^"<>]*)"?\s*/?' . '>',
                '<meta\s+property="?og:title"?\s+content="?([^"<>]*)"?\s*/?' . '>',
            ),
            't_keywords' => array(
                '<meta\s+name="?keywords"?\s+content="?([^"<>]*)"?\s*/?' . '>',
            ),
            't_description' => array(
                '<meta\s+name="?description"?\s+content="?([^"<>]*)"?\s*/?' . '>',
                '<meta\s+name="?DC\.Description"?\s+content="?([^"<>]*)"?\s*/?' . '>',
                '<meta\s+name="?twitter:description"?\s+content="?([^"<>]*)"?\s*/?' . '>',
                '<meta\s+property="?og:description"?\s+content="?([^"<>]*)"?\s*/?' . '>',
            ),
            't_image_url' => array(
                '<meta\s+name="?twitter:image"?\s+content="?([^"<>]*)"?\s*/?' . '>',
                '<meta\s+property="?og:image"?\s+content="?([^"]*)"?\s*/?' . '>',
            ),
        );

        require_code('character_sets');
        $html = convert_to_internal_encoding($html);

        foreach ($headers as $header => $regexps) {
            foreach ($regexps as $regexp) {
                $matches = array();
                if (preg_match('#' . $regexp . '#isU', $html, $matches) != 0) {
                    $value = str_replace('"', '&quot;', stripslashes($matches[1]));

                    if ($header == 't_title' || $header == 't_image_url') { // Non-HTML
                        $value = str_replace(array('&ndash;', '&mdash;'), array('-', '-'), $value);
                        $value = @html_entity_decode($value, ENT_QUOTES, get_charset());
                        $value = trim($value);
                        $value = substr($value, 0, 255);
                    }

                    if ($value != '') {
                        $meta_details[$header] = $value;
                    }
                }
            }
        }

        if ($meta_details['t_image_url'] != '') {
            $meta_details['t_image_url'] = qualify_url($meta_details['t_image_url'], $url);
        }

        if (($result[1] == 'application/octet-stream') || ($result[1] == '')) {
            // Lame, no real mime type - maybe the server is just not configured to know it - try and guess by using the file extension and our own Composr list
            require_code('mime_types');
            require_code('files');
            $meta_details['t_mime_type'] = get_mime_type(get_file_extension($url), true);
        } else {
            $meta_details['t_mime_type'] = $result[1];
        }

        $matches = array();
        $num_matches = preg_match_all('#<link\s+[^<>]*>#i', $html, $matches);
        for ($i = 0; $i < $num_matches; $i++) {
            $line = $matches[0][$i];
            $matches2 = array();
            if ((preg_match('#\srel=["\']?alternate["\']?#i', $line) != 0) && (preg_match('#\shref=["\']?([^"\']+)["\']?#i', $line, $matches2) != 0)) {
                if (preg_match('#\stype=["\']?application/json\+oembed["\']?#i', $line) != 0) {
                    $meta_details['t_json_discovery'] = @html_entity_decode($matches2[1], ENT_QUOTES, get_charset());
                }
                if (preg_match('#\stype=["\']?text/xml\+oembed["\']?#i', $line) != 0) {
                    $meta_details['t_xml_discovery'] = @html_entity_decode($matches2[1], ENT_QUOTES, get_charset());
                }
            }
        }

        $GLOBALS['SITE_DB']->query_insert('url_title_cache', $meta_details, false, true); // 'true' to stop race conditions
    } elseif (is_array($result)) {
        $meta_details['t_mime_type'] = $result[1];
    }

    $cache[$url] = $meta_details;
    return $meta_details;
}

/**
 * Final filter for downloader output: try a bit harder to detect the character encoding, in case it was not in an HTTP filter.
 * Manipulates the $HTTP_CHARSET global.
 *
 * @param  string $out The HTTP stream we will look through
 * @return string Same as $out
 *
 * @ignore
 */
function _detect_character_encoding($out)
{
    global $HTTP_CHARSET;
    if ($HTTP_CHARSET === null) {
        $matches = array();
        if (preg_match('#<' . '?xml[^<>]*\s+encoding="([^"]+)"#', $out, $matches) != 0) {
            $HTTP_CHARSET = trim($matches[1]);
        } elseif (preg_match('#<meta\s+http-equiv="Content-Type"\s+content="[^"]*;\s*charset=([^"]+)"#i', $out, $matches) != 0) {
            $HTTP_CHARSET = trim($matches[1]);
        }
    }

    return $out;
}