<?php /*

 Composr
 Copyright (c) Christopher Graham, 2004-2024

 See docs/LICENSE.md for full licensing information.


 NOTE TO PROGRAMMERS:
   Do not edit this file. If you need to make changes, save your changed file to the appropriate *_custom folder
   **** If you ignore this advice, then your website upgrades (e.g. for bug fixes) will likely kill your changes ****

*/

/**
 * @license    http://opensource.org/licenses/cpal_1.0 Common Public Attribution License
 * @copyright  Christopher Graham
 * @package    core
 */

/*EXTRA FUNCTIONS: shell_exec|fsockopen|ctype_xdigit|curl_.**/

/**
 * Standard code module initialisation function.
 *
 * @ignore
 */
function init__http()
{
    global $DOWNLOAD_LEVEL;
    $DOWNLOAD_LEVEL = 0;
}

/**
 * Call a function, with inbuilt on-disk caching support.
 *
 * @param  mixed $func Function to call
 * @param  array $args Arguments to call with
 * @param  ?integer $timeout Caching timeout in minutes (null: no timeout)
 * @param  boolean $cache_errors Whether to cache HTTP statuses that do not start '2'
 * @return mixed The function result OR for cms_http_request calls a tuple of result details
 */
function cache_and_carry($func, array $args, ?int $timeout = null, bool $cache_errors = false)
{
    $ret = mixed();

    $args_cache_signature = $args;
    if ((!$cache_errors) && (is_string($func)) && (in_array($func, ['cms_http_request', 'http_get_contents']))) {
        // Things likely to change that would break our cache signature
        unset($args_cache_signature[1]['extra_headers']['Authorization']);
    }

    $path = get_custom_file_base() . '/caches/http/' . md5(serialize($func)) . '__' . md5(serialize($args_cache_signature)) . '.bin';
    if (is_file($path) && ((!$GLOBALS['FORUM_DRIVER']->is_super_admin(get_member())) || (get_param_integer('keep_cache_and_carry', 1) == 1)) && (($timeout === null) || (filemtime($path) > time() - $timeout * 60))) {
        $_ret = cms_file_get_contents_safe($path, FILE_READ_LOCK);
        if ($func === 'cms_http_request') {
            $ret = @unserialize($_ret);
        } else {
            $ret = $_ret;
        }

        if ($ret !== null) {
            return $ret;
        }
    }

    $_ret = call_user_func_array($func, $args);
    require_code('files');
    if ($func === 'cms_http_request') {
        $ret = [$_ret->data, $_ret->download_mime_type, $_ret->download_size, $_ret->download_url, $_ret->message, $_ret->message_b, $_ret->new_cookies, $_ret->filename, $_ret->charset, $_ret->download_mtime];
        if (($cache_errors) || (($_ret->message !== null) && (substr($_ret->message, 0, 1) == '2'))) {
            cms_file_put_contents_safe($path, serialize($ret), FILE_WRITE_FAILURE_SOFT | FILE_WRITE_FIX_PERMISSIONS);
        }
    } else {
        $ret = is_string($_ret) ? $_ret : serialize($_ret);
        cms_file_put_contents_safe($path, $ret, FILE_WRITE_FAILURE_SOFT | FILE_WRITE_FIX_PERMISSIONS);
    }
    return $ret;
}

/**
 * Extract meta details from a URL.
 *
 * @param  URLPATH $url Webpage URL
 * @return array A map of meta details extracted from the webpage
 */
function get_webpage_meta_details(string $url) : array
{
    static $cache = [];

    if (array_key_exists($url, $cache)) {
        return $cache[$url];
    }
    if (get_param_integer('keep_oembed_cache', 1) == 1) {
        $_meta_details = $GLOBALS['SITE_DB']->query_select('url_title_cache', ['*'], ['t_url' => $url], '', 1);
        if (array_key_exists(0, $_meta_details)) {
            $meta_details = $_meta_details[0];
            $cache[$url] = $meta_details;
            return $meta_details;
        }
    }

    $meta_details = [
        't_url' => substr($url, 0, 255),
        't_title' => '',
        't_meta_title' => '',
        't_keywords' => '',
        't_description' => '',
        't_image_url' => '',
        't_mime_type' => '',
        't_json_discovery' => '',
        't_xml_discovery' => '',
    ];

    if (url_is_local($url)) {
        $matches = [];
        if (preg_match('#^uploads/.*\.(\w+)$#', $url, $matches) != 0) {
            if (!in_array($matches[1], ['bin', 'dat'/*LEGACY*/, 'htm', 'html'])) {
                require_code('mime_types');
                $meta_details['t_mime_type'] = get_mime_type($matches[1], true);
                $cache[$url] = $meta_details;
                return $meta_details;
            }
        }

        $url = get_custom_base_url() . '/' . $url;
    }

    if (!looks_like_url($url)) {
        return $meta_details;
    }

    $result = cache_and_carry('cms_http_request', [$url, ['byte_limit' => 1024 * 10, 'trigger_error' => false, 'timeout' => 2.0]]);
    if ((is_array($result)) && ($result[1] !== null) && (strpos($result[1], 'html') !== false) && $result[4] == '200') {
        $html = $result[0];

        // In ascending precedence
        $headers = [
            't_title' => [
                '<title[^>]*>\s*(.*)\s*</title>',
            ],
            't_meta_title' => [
                '<meta\s+name="DC\.Title"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+name="twitter:title"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+name="og:title"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+property="og:title"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+property="og:title"\s+name="og:title"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+name="og:title"\s+property="og:title"\s+content="([^"<>]*)"[^<>]*>',

                '<meta\s+name=\'DC\.Title\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+name=\'twitter:title\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+name=\'og:title\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+property=\'og:title\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+property=\'og:title\'\s+name=\'og:title\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+name=\'og:title\'\s+property=\'og:title\'\s+content=\'([^\'<>]*)\'[^<>]*>',
            ],
            't_keywords' => [
                '<meta\s+name="keywords"\s+content="([^"<>]*)"[^<>]*>',

                '<meta\s+name=\'keywords\'\s+content=\'([^\'<>]*)\'[^<>]*>',
            ],
            't_description' => [
                '<meta\s+name="description"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+name="DC\.Description"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+name="twitter:description"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+name="og:description"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+property="og:description"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+property="og:description"\s+name="og:description"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+name="og:description"\s+property="og:description"\s+content="([^"<>]*)"[^<>]*>',

                '<meta\s+name=\'description\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+name=\'DC\.Description\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+name=\'twitter:description\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+name=\'og:description\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+property=\'og:description\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+property=\'og:description\'\s+name=\'og:description\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+name=\'og:description\'\s+property=\'og:description\'\s+content=\'([^\'<>]*)\'[^<>]*>',
            ],
            't_image_url' => [
                '<meta\s+name="twitter:image"\s+content="([^"<>]*)"[^<>]*>',
                '<meta\s+property="og:image"\s+content="([^"]*)"[^<>]*>',
                '<meta\s+property="og:image"\s+name="og:image"\s+content="([^"]*)"[^<>]*>',
                '<meta\s+name="og:image"\s+property="og:image"\s+content="([^"]*)"[^<>]*>',

                '<meta\s+name=\'twitter:image\'\s+content=\'([^\'<>]*)\'[^<>]*>',
                '<meta\s+property=\'og:image\'\s+content=\'([^\']*)\'[^<>]*>',
                '<meta\s+property=\'og:image\'\s+name=\'og:image\'\s+content=\'([^\']*)\'[^<>]*>',
                '<meta\s+name=\'og:image\'\s+property=\'og:image\'\s+content=\'([^\']*)\'[^<>]*>',
            ],
        ];

        require_code('character_sets');
        $html = convert_to_internal_encoding($html, $result[8]);

        foreach ($headers as $header => $regexps) {
            foreach ($regexps as $regexp) {
                $matches = [];
                if (preg_match('#' . $regexp . '#isU', $html, $matches) != 0) {
                    $value = str_replace('"', '&quot;', stripslashes($matches[1]));

                    if ($header == 't_title' || $header == 't_image_url') { // Non-HTML
                        $value = str_replace(['&ndash;', '&mdash;'], ['-', '-'], $value);
                        $value = @html_entity_decode($value, ENT_QUOTES);
                        $value = trim($value);
                        $value = substr($value, 0, 255);
                    }

                    if ($value != '') {
                        $meta_details[$header] = $value;
                    }
                }
            }
        }

        if ($meta_details['t_image_url'] != '') {
            if (strlen($meta_details['t_image_url']) > 255) {
                $meta_details['t_image_url'] = ''; // We can't deal with really long URLs here
            }
        }

        if (($result[1] == 'application/octet-stream') || ($result[1] == '')) {
            // Generic, no real mime type - maybe the server is just not configured to know it - try and guess by using the file extension and our own Composr list
            require_code('mime_types');
            require_code('files');
            $meta_details['t_mime_type'] = get_mime_type(get_file_extension($url), true);
        } else {
            $meta_details['t_mime_type'] = $result[1];
        }

        $matches = [];
        $num_matches = preg_match_all('#<link\s+[^<>]*>#i', $html, $matches);
        for ($i = 0; $i < $num_matches; $i++) {
            $line = $matches[0][$i];
            $matches2 = [];
            if ((preg_match('#\srel=["\']?alternate["\']?#i', $line) != 0) && (preg_match('#\shref=["\']?([^"\']+)["\']?#i', $line, $matches2) != 0)) {
                if (preg_match('#\stype=["\']?application/json\+oembed["\']?#i', $line) != 0) {
                    $meta_details['t_json_discovery'] = @html_entity_decode($matches2[1], ENT_QUOTES);
                    if (strlen($meta_details['t_json_discovery']) > 255) {
                        $meta_details['t_json_discovery'] = ''; // We can't deal with really long URLs here
                    }
                }
                if (preg_match('#\stype=["\']?text/xml\+oembed["\']?#i', $line) != 0) {
                    $meta_details['t_xml_discovery'] = @html_entity_decode($matches2[1], ENT_QUOTES);
                    if (strlen($meta_details['t_xml_discovery']) > 255) {
                        $meta_details['t_xml_discovery'] = ''; // We can't deal with really long URLs here
                    }
                }
            }
        }

        $GLOBALS['SITE_DB']->query_insert('url_title_cache', $meta_details, false, true); // 'true' to stop race conditions
    } elseif (is_array($result)) {
        $meta_details['t_mime_type'] = $result[1];
    }

    $cache[$url] = $meta_details;
    return $meta_details;
}

/**
 * Return the file in the URL by downloading it over HTTP. If a byte limit is given, it will only download that many bytes. It outputs warnings, returning null, on error.
 *
 * @param  URLPATH $url The URL to download
 * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
 * @return object HttpDownloader object, which can be checked for return data
 */
function _cms_http_request(string $url, array $options = []) : object
{
    $curl = new HttpDownloaderCurl();
    $curl_priority = $curl->may_run_for($url, $options);

    $sockets = new HttpDownloaderSockets();
    $sockets_priority = $sockets->may_run_for($url, $options);

    $file_wrapper = new HttpDownloaderFileWrapper();
    $file_wrapper_priority = $file_wrapper->may_run_for($url, $options);

    $filesystem = new HttpDownloaderFilesystem();
    $filesystem_priority = $filesystem->may_run_for($url, $options);

    if (!empty($options['force_curl']) || empty($options['force_sockets']) && empty($options['force_file_wrapper']) && empty($options['force_filesystem']) && $curl_priority > $sockets_priority && $curl_priority > $file_wrapper_priority && $curl_priority > $filesystem_priority) {
        $test = $curl->run($url, $options);
        if ($test !== false) {
            return $curl;
        }
    }

    if (!empty($options['force_sockets']) || empty($options['force_file_wrapper']) && empty($options['force_filesystem']) && $sockets_priority > $file_wrapper_priority && $sockets_priority > $filesystem_priority) {
        $test = $sockets->run($url, $options);
        if ($test !== false) {
            return $sockets;
        }
    }

    if (!empty($options['force_file_wrapper']) || empty($options['force_filesystem']) && $file_wrapper_priority > $filesystem_priority) {
        $test = $file_wrapper->run($url, $options);
        if ($test !== false) {
            return $file_wrapper;
        }
    }

    $test = $filesystem->run($url, $options);
    if ($test === false) {
        fatal_exit(do_lang_tempcode('INTERNAL_ERROR'));
    }
    return $filesystem;
}

/**
 * Base HttpDownloader.
 *
 * @package core
 */
abstract class HttpDownloader
{
    // Run priorities
    protected const RUN_PRIORITY_NO = 1;
    protected const RUN_PRIORITY_LOW = 2;
    protected const RUN_PRIORITY_MEDIUM = 3;
    protected const RUN_PRIORITY_HIGH = 4;
    protected const RUN_PRIORITY_VERY_HIGH = 5;

    // Input parameters, coming from $options map
    protected $byte_limit = null; // ?integer. The number of bytes to download. This is not a guarantee, it is a minimum (null: all bytes)
    protected $trigger_error = true; // boolean. Whether to throw a Composr error, on error
    protected $no_redirect = false; // boolean. Whether to block redirects (returns null when found)
    protected $ua = 'Composr'; // ~?string. The user-agent to identify as (null: simulate Google Chrome) (false: none, useful to avoid filtering rules on the other end)
    protected $post_params = null; // ?array or string. An optional array of POST parameters to send or a string of a request body; if this is null, a GET request is used for the default for http_verb (null: none)
    protected $cookies = []; // array. An optional array of cookies to send
    protected $accept = null; // ?string. 'accept' header value (null: don't pass one)
    protected $accept_charset = null; // ?string. 'accept-charset' header value (null: don't pass one)
    protected $accept_language = null; // ?string. 'accept-language' header value (null: don't pass one)
    protected $write_to_file = null; // ?resource. File handle to write to (null: do not do that)
    protected $referer = null; // ?string. The HTTP referer (null: none)
    protected $auth = null; // ?array. A pair: authentication username and password (null: none)
    protected $timeout = 6.0; // float. The timeout (for connecting/stalling, not for overall download time); usually it is rounded up to the nearest second, depending on the downloader implementation
    protected $files = []; // array or string. Files to send, map between field name to file path; or a file path to use for the whole request body
    protected $extra_headers = []; // array. Extra headers to send
    protected $http_verb = null; // ?string. HTTP verb (null: auto-decide based on other parameters)
    protected $raw_content_type = null; // string. The content type to use if a raw HTTP post
    protected $ignore_http_status = false; // boolean. Return a result regardless of HTTP status
    protected $verifypeer_enabled = true; // boolean. Whether to check SSL certificates
    protected $convert_to_internal_encoding = false; // boolean. Whether to look at the character set of the request (from HTTP headers and BOMs) and convert it
    protected $default_charset = 'utf-8'; // string. The default character set to assume if none is specified in the response

    // Class processing configuration
    protected $add_content_type_header_manually = false;
    protected $add_files_manually = false;

    // Internal
    protected $divider = null;
    protected $url_parts = null;
    protected $connect_to = null;
    protected $do_ip_forwarding = null;
    protected $connecting_url = null;
    protected $raw_payload = null;
    protected $sending_request_content = false;
    protected $raw_post_handle = null;
    protected $raw_post_path = null;
    protected $timeout_before = null;

    // Output
    public $data = null; // ?string. The result returned from the last HTTP lookup.
    public $content_size = 0; // integer. The number of bytes we downloaded so far.

    public $download_mime_type = null; // ?ID_TEXT. The mime type returned from the last HTTP lookup.
    public $download_size = null; // ?ID_TEXT. The download size returned from the last HTTP lookup.
    public $download_url = null; // ?ID_TEXT. The URL for the last HTTP lookup.
    public $download_mtime = null; // ?ID_TEXT. The file modification time returned from the last HTTP lookup.

    /**
     * The status code returned.
     * This is usually the HTTP status. But if an error occurred prior to HTTP, it may be something different, like 'curl-error-#'.
     *
     * @var ?string The status code (null: no status code processed yet)
     */
    public $message = null;

    public $message_b = null; // ?string. The status message returned from the last HTTP lookup.
    public $new_cookies = []; // ?ID_TEXT. The cookies returned from the last HTTP lookup.
    public $filename = null; // ?ID_TEXT. The filename returned from the last HTTP lookup.
    public $charset = null; // ?ID_TEXT. The character set returned from the last HTTP lookup.
    public $headers = []; // Any HTTP headers collected.
    public $generation_time = 0; // How long the request took
    public $implementation_used = null; // For debugging.

    /**
     * See if this class may run.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return integer The execution priority
     */
    abstract public function may_run_for(string $url, array $options = []) : int;


    /**
     * Return the file in the URL by downloading it over HTTP. If a byte limit is given, it will only download that many bytes. It outputs warnings, returning null, on error.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return ~?string The data downloaded (null: error) (false: backend failed)
     */
    public function run(string $url, array $options = [])
    {
        global $DOWNLOAD_LEVEL;

        $this->implementation_used = get_class($this);

        $this->download_url = $url;

        // Parse options...

        $this->read_in_options($options);

        // Prevent DOS loop attack...

        if ((!empty($_SERVER['HTTP_USER_AGENT'])) && (is_string($this->ua))) {
            if ($_SERVER['HTTP_USER_AGENT'] == $this->ua) {
                $ua = 'Composr-recurse';
            }
            if ($_SERVER['HTTP_USER_AGENT'] == 'Composr-recurse') {
                return null;
            }
        }
        if ($DOWNLOAD_LEVEL == 8) {
            $this->data = null;
            return null;
        }

        // Normalise the URL...

        require_code('urls');
        $url = str_replace(' ', '%20', $url);
        if (url_is_local($url)) {
            $url = get_custom_base_url() . '/' . $url;
        }
        if ((strpos($url, '/') !== false) && (strrpos($url, '/') < 7)) {
            $url .= '/';
        }

        // HTTP authentication in URL
        if ($this->auth === null) {
            $matches = [];
            if (preg_match('#^https?://([^:@/]+):([^:@/]+)@#', $url, $matches) != 0) {
                $this->auth = [$matches[1], $matches[2]];
            }
        }

        // Work out what we'll be connecting to...

        $this->url_parts = @cms_parse_url_safe(normalise_idn_url($url));
        if (($this->url_parts === false) || (!isset($this->url_parts['host'])) || (!isset($this->url_parts['scheme']))) {
            if ($this->trigger_error) {
                warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_BAD_URL', escape_html($url)), false, true);
            } else {
                $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_BAD_URL', escape_html($url));
            }
            $DOWNLOAD_LEVEL--;
            $this->message = 'malconstructed-URL';
            return null;
        }
        if (!array_key_exists('host', $this->url_parts)) {
            $this->url_parts['host'] = get_base_url_hostname();
        }
        $this->connect_to = $this->url_parts['host'];

        $config_ip_forwarding = function_exists('get_option') ? get_option('ip_forwarding') : '';
        $no_proxy = ((!function_exists('get_option')) || (get_option('proxy') == ''));
        $this->do_ip_forwarding = ($config_ip_forwarding != '') && ($config_ip_forwarding != '0') && (is_our_server($this->connect_to)) && ($no_proxy);

        if ($this->do_ip_forwarding) { // For cases where we have IP-forwarding, and a strong firewall (i.e. blocked to our own domain's IP by default)
            if ($config_ip_forwarding == '1') {
                $server_ips = get_server_ips(true);
                $this->connect_to = $server_ips[0];
            } else {
                $protocol_end_pos = strpos($config_ip_forwarding, '://');
                if ($protocol_end_pos !== false) {
                    // Full with protocol
                    $url = preg_replace('#^(https?://)#', substr($config_ip_forwarding, 0, $protocol_end_pos + 3), $url);
                    $config_ip_forwarding = substr($config_ip_forwarding, $protocol_end_pos + 3);
                } else {
                    // IP address
                }
                $this->connect_to = $config_ip_forwarding;
            }
        } elseif (($this->url_parts['scheme'] == 'http') && ($no_proxy)) {
            $this->connect_to = cms_gethostbyname($this->connect_to); // for DNS caching
        }
        if (!array_key_exists('scheme', $this->url_parts)) {
            $this->url_parts['scheme'] = 'http';
        }

        if (strpos($this->connect_to, ':') !== false) {
            $connect_to_url_compat = '[' . $this->connect_to . ']';
        } else {
            $connect_to_url_compat = $this->connect_to;
        }
        $this->connecting_url = preg_replace('#^(https?://)' . preg_quote($this->url_parts['host'], '#') . '([/:]|$)#', '${1}' . $connect_to_url_compat . '${2}', $url);

        // More preprocessing...

        $this->raw_payload = ''; // Note that this will contain HTTP headers (it is appended directly after headers with no \r\n between -- so it contains \r\n\r\n itself when the content body is going to start)
        $this->sending_request_content = false;
        $this->raw_post_handle = null;
        $this->raw_post_path = null;
        if (($this->post_params !== null) || (!empty($this->files))) {
            if ($this->post_params === null) {
                $this->post_params = []; // POST is implied
            }

            if ((is_string($this->files)) && (!empty($this->post_params))) {
                fatal_exit('Internal error - you cannot do a PUT-style request from a file AND use post parameters at the same time');
            }

            $this->sending_request_content = true;

            if (is_string($this->files)) {
                $_postdetails_params = null; // Will use the first file instead
            } elseif (is_string($this->post_params)) {
                $_postdetails_params = $this->post_params;
            } elseif (empty($this->post_params)) {
                $_postdetails_params = '';
            } else {
                $_postdetails_params = http_build_query($this->post_params);
            }

            if ($_postdetails_params === null) { // Direct output from file
                $this->raw_post_path = $this->files;
                $this->raw_post_handle = fopen($this->files, 'rb');
                $this->raw_payload .= 'Content-Length: ' . strval(filesize($this->files)) . "\r\n";
            } elseif ((empty($this->files))) { // If no files, use simple application/x-www-form-urlencoded
                if (!$this->add_content_type_header_manually) {
                    if ($this->raw_content_type !== null) {
                        if (!isset($this->extra_headers['Content-Type'])) {
                            $this->raw_payload .= 'Content-Type: ' . $this->raw_content_type . "\r\n";
                        }
                    } else {
                        $this->raw_payload .= 'Content-Type: application/x-www-form-urlencoded; charset=' . get_charset() . "\r\n";
                    }
                    $this->raw_payload .= 'Content-Length: ' . strval(strlen($_postdetails_params)) . "\r\n";
                    $this->raw_payload .= "\r\n";
                }
                // ^ curl sets the above itself

                $this->raw_payload .= $_postdetails_params;
            } else { // If files, use more complex multipart/form-data
                $this->divider = uniqid('', true);
                $raw_payload_mime = '';
                foreach ($this->post_params as $key => $val) {
                    $raw_payload_mime .= '----cms' . $this->divider . "\r\n";
                    if ($this->raw_content_type !== null) {
                        if (!isset($this->extra_headers['Content-Type'])) {
                            $raw_payload_mime .= 'Content-Type: ' . $this->raw_content_type . "\r\n\r\n";
                        }
                    } else {
                        $raw_payload_mime .= 'Content-Disposition: form-data; name="' . str_replace('"', '\"', $key) . '"' . "\r\n\r\n";
                    }
                    $raw_payload_mime .= $val . "\r\n";
                }
                foreach ($this->files as $upload_field => $file_path) {
                    if ((!empty($this->post_params)) || (count($this->files) != 1)) {
                        $raw_payload_mime .= '----cms' . $this->divider . "\r\n";
                        if (strpos($upload_field, '/') === false) {
                            $raw_payload_mime .= 'Content-Disposition: form-data; name="' . str_replace('"', '\"', $upload_field) . '"; filename="' . str_replace('"', '\"', basename($file_path)) . '"' . "\r\n";

                            require_code('mime_types');
                            require_code('files');
                            $raw_payload_mime .= 'Content-Type: ' . get_mime_type(get_file_extension($file_path), true) . "; charset=" . get_charset() . "\r\n\r\n";
                        } else {
                            // mime-type given rather than file-name
                            $raw_payload_mime .= 'Content-Type: ' . $upload_field . "\r\n\r\n";
                        }
                    } else {
                        if ((strpos($upload_field, '/') === false) && ($this->raw_content_type == '')) {
                            $this->raw_content_type = $upload_field;
                        }
                    }
                    $raw_payload_mime .= cms_file_get_contents_safe($file_path, FILE_READ_LOCK);
                    if ((!empty($this->post_params)) || (count($this->files) != 1)) {
                        $raw_payload_mime .= "\r\n";
                    }
                    if ((!empty($this->post_params)) || (count($this->files) != 1)) {
                        $raw_payload_mime .= '----cms' . $this->divider . "--\r\n";
                    }
                }

                if ($this->add_files_manually) {
                    $this->raw_payload = $raw_payload_mime; // Other settings will be passed via cURL itself
                } else {
                    $this->raw_payload = 'Content-Type: multipart/form-data; boundary="--cms' . $this->divider . '"; charset=' . get_charset() . "\r\n";
                    $this->raw_payload .= 'Content-Length: ' . strval(strlen($raw_payload_mime)) . "\r\n";
                    $this->raw_payload .= "\r\n" . $raw_payload_mime;
                }
            }
        }

        if ($this->http_verb === null) {
            $this->http_verb = ((($this->post_params === null) && (empty($this->files))) ? (($this->byte_limit === 0) ? 'HEAD' : 'GET') : 'POST');
        }

        // Debugging
        $_log_file = get_custom_file_base() . '/data_custom/http.log';
        if (is_file($_log_file)) {
            require_code('files');

            $debug = [
                'byte_limit' => $this->byte_limit,
                'no_redirect' => $this->no_redirect,
                'ua' => $this->ua,
                'post_params' => $this->post_params,
                'cookies' => $this->cookies,
                'accept' => $this->accept,
                'accept_charset' => $this->accept_charset,
                'accept_language' => $this->accept_language,
                'referer' => $this->referer,
                'auth' => $this->auth,
                'timeout' => $this->timeout,
                'files' => $this->files,
                'extra_headers' => $this->extra_headers,
                'http_verb' => $this->http_verb,
                'raw_content_type' => $this->raw_content_type,
                'ignore_http_status' => $this->ignore_http_status,
                'verifypeer_enabled' => $this->verifypeer_enabled,
                'convert_to_internal_encoding' => $this->convert_to_internal_encoding,
                'default_charset' => $this->default_charset,
                'add_content_type_header_manually' => $this->add_content_type_header_manually,
                'add_files_manually' => $this->add_files_manually,
            ];
            $log_message = loggable_date() . ' REQUEST (' . $url . '): ' . serialize($debug) . "\n";
            $log_file = cms_fopen_text_write($_log_file, true, 'ab');
            fwrite($log_file, $log_message);
            flock($log_file, LOCK_UN);
            fclose($log_file);
        }

        // Call downloader method...

        $start_time = time();

        global $DOWNLOAD_LEVEL;
        $DOWNLOAD_LEVEL++;
        $this->data = $this->_run($url, $options);
        $DOWNLOAD_LEVEL--;
        $this->detect_character_encoding();

        $this->generation_time = time() - $start_time;

        // Post-processing...

        // Detect HTTP-status using http-equiv
        if (($this->download_mime_type !== null) && ($this->data !== null) && ($this->message !== null) && (substr($this->message, 0, 1) == '2')) {
            if (preg_match('#^(text/html|application/xhtml+xml)($|;)#', $this->download_mime_type) != 0) {
                $matches = [];
                if (preg_match('#<meta http-equiv=["\']?Status["\']? content=["\']?(\d+)#i', $this->data, $matches) != 0) {
                    $this->download_mime_type = $matches[1];
                }
            }
        }

        // Debugging
        $_log_file = get_custom_file_base() . '/data_custom/http.log';
        if (is_file($_log_file)) {
            require_code('files');

            $debug = [
                'data' => is_string($this->data) ? strlen($this->data) : $this->data,
                'content_size' => $this->content_size,
                'download_mime_type' => $this->download_mime_type,
                'download_size' => $this->download_size,
                'download_url' => $this->download_url,
                'download_mtime' => $this->download_mtime,
                'message' => $this->message,
                'message_b' => $this->message_b,
                'new_cookies' => $this->new_cookies,
                'filename' => $this->filename,
                'charset' => $this->charset,
                'headers' => $this->headers,
                'generation_time' => $this->generation_time,
                'implementation_used' => $this->implementation_used,
            ];
            $log_message = loggable_date() . ' RESPONSE (' . $url . '): ' . serialize($debug) . "\n";
            $log_file = cms_fopen_text_write($_log_file, true, 'ab');
            fwrite($log_file, $log_message);
            flock($log_file, LOCK_UN);
            fclose($log_file);
        }

        // Done...

        return $this->data;
    }

    /**
     * Fixup non-standard status codes to standard ones.
     *
     * @param  string $status Status returned
     * @return string Fixed status
     */
    protected function fix_non_standard_statuses(string $status) : string
    {
        $status = preg_replace('#^(\d\d\d)\.\d+$#', '$1', $status); // IIS
        $status = preg_replace('#^(\d\d\d)\s.*$#', '$1', $status); // E.g. "200 OK"; we just care about the 200
        return $status;
    }

    /**
     * Read in options.
     *
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     */
    protected function read_in_options(array $options)
    {
        if (array_key_exists('byte_limit', $options)) {
            $this->byte_limit = $options['byte_limit'];
        }

        if (array_key_exists('trigger_error', $options)) {
            $this->trigger_error = $options['trigger_error'];
        }

        if (array_key_exists('no_redirect', $options)) {
            $this->no_redirect = $options['no_redirect'];
        }

        if (array_key_exists('ua', $options)) {
            $this->ua = $options['ua'];

            if ($this->ua === null) {
                $this->ua = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.A.B.C Safari/525.13';
            }
        }

        if (array_key_exists('post_params', $options)) {
            $this->post_params = $options['post_params'];
        }

        if (array_key_exists('cookies', $options)) {
            $this->cookies = $options['cookies'];
        }

        if (array_key_exists('accept', $options)) {
            $this->accept = $options['accept'];
        }

        if (array_key_exists('accept_charset', $options)) {
            $this->accept_charset = $options['accept_charset'];
        }

        if (array_key_exists('accept_language', $options)) {
            $this->accept_language = $options['accept_language'];
        }

        if (array_key_exists('write_to_file', $options)) {
            $this->write_to_file = $options['write_to_file'];
        }

        if (array_key_exists('referer', $options)) {
            $this->referer = $options['referer'];
        }

        if (array_key_exists('auth', $options)) {
            $this->auth = $options['auth'];
        }

        if (array_key_exists('timeout', $options)) {
            $this->timeout = $options['timeout'];
        }

        if (array_key_exists('files', $options)) {
            $this->files = $options['files'];
        }

        if (array_key_exists('extra_headers', $options)) {
            $this->extra_headers = $options['extra_headers'];
        }

        if (array_key_exists('http_verb', $options)) {
            $this->http_verb = $options['http_verb'];
        }

        if (array_key_exists('raw_content_type', $options)) {
            $this->raw_content_type = $options['raw_content_type'];
        }

        if (array_key_exists('ignore_http_status', $options)) {
            $this->ignore_http_status = $options['ignore_http_status'];
        }

        if (array_key_exists('verifypeer_enabled', $options)) {
            $this->verifypeer_enabled = $options['verifypeer_enabled'];
        }

        if (array_key_exists('convert_to_internal_encoding', $options)) {
            $this->convert_to_internal_encoding = $options['convert_to_internal_encoding'];
        }
    }

    /**
     * Do the inner call using a particular downloader method.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return ~?string The data downloaded (null: error) (false: backend failed)
     */
    abstract protected function _run(string $url, array $options);


    /**
     * Find the cookie HTTP header contents.
     *
     * @return string The cookie line
     */
    protected function get_cookie_string() : string
    {
        // Prep cookies
        if (!empty($this->cookies)) {
            $cookies = '';
            $done_one_cookie = false;
            foreach ($this->cookies as $key => $val) {
                if ($done_one_cookie) {
                    $cookies .= '; ';
                }
                if (is_array($val)) {
                    foreach ($val as $key2 => $val2) {
                        if (!is_string($key2)) {
                            $key2 = strval($key2);
                        }
                        if ($done_one_cookie) {
                            $cookies .= '; ';
                        }
                        $cookies .= $key . '[' . $key2 . ']=' . rawurlencode($val2);
                        $done_one_cookie = true;
                    }
                } else {
                    $cookies .= $key . '=' . rawurlencode($val);
                }
                $done_one_cookie = true;
            }
        }
        return $cookies;
    }

    /**
     * Find the HTTP header contents.
     *
     * @return string The header contents
     */
    protected function get_header_string() : string
    {
        $headers = '';
        if (!empty($this->cookies)) {
            $headers .= 'Cookie: ' . $this->get_cookie_string() . "\r\n";
        }
        if (is_string($this->ua)) {
            $headers .= 'User-Agent: ' . rawurlencode($this->ua) . "\r\n";
        }
        if ($this->auth !== null) {
            $headers .= 'Authorization: Basic ' . base64_encode(implode(':', $this->auth)) . "==\r\n";
        }
        foreach ($this->extra_headers as $key => $val) {
            $headers .= $key . ': ' . $val . "\r\n";
        }
        if ($this->accept !== null) {
            $headers .= 'Accept: ' . rawurlencode($this->accept) . "\r\n";
        } else {
            $headers .= "Accept: */*\r\n"; // There's a mod_security rule that checks for this
        }
        if ($this->accept_charset !== null) {
            $headers .= 'Accept-Charset: ' . rawurlencode($this->accept_charset) . "\r\n";
        }
        if ($this->accept_language !== null) {
            $headers .= 'Accept-Language: ' . rawurlencode($this->accept_language) . "\r\n";
        }
        if ($this->referer !== null) {
            $headers .= 'Referer: ' . rawurlencode($this->referer) . "\r\n";
        }
        return $headers;
    }

    /**
     * Read in any HTTP headers that we probe for, from an HTTP line.
     *
     * @param  string $line The line
     */
    protected function read_in_headers(string $line)
    {
        $matches = [];
        if (preg_match("#Content-Disposition: [^\r\n]*filename=\"([^;\r\n]*)\"\r\n#i", $line, $matches) != 0) {
            $this->filename = $matches[1];
        }
        if (preg_match("#^Set-Cookie: ([^\r\n=]*)=([^\r\n]*)\r\n#i", $line, $matches) != 0) {
            $this->new_cookies[trim(rawurldecode($matches[1]))] = trim($matches[2]);

            $cookie_key = trim(rawurldecode($matches[1]));

            $cookie_value = trim($matches[2]);
            $_cookie_parts = explode('; ', $cookie_value);

            $cookie_parts = [];

            $cookie_parts['key'] = $cookie_key;
            $cookie_parts['value'] = trim(rawurldecode(array_shift($_cookie_parts)));

            foreach ($_cookie_parts as $i => $part) {
                $temp = explode('=', $part, 2);
                if (array_key_exists(1, $temp)) {
                    $cookie_parts[trim($temp[0])] = trim(rawurldecode($temp[1]));
                }
            }
            if (function_exists('get_cookie_domain')) {
                $cookie_parts['domain'] = get_cookie_domain();
            }

            $this->new_cookies[$cookie_key] = $cookie_parts;
        }
        if (preg_match("#^Content-Length: ([^;\r\n]*)\r\n#i", $line, $matches) != 0) {
            $this->download_size = intval($matches[1]);
        }
        if (preg_match("#^Last-Modified: ([^;\r\n]*)\r\n#i", $line, $matches) != 0) {
            $this->download_mtime = strtotime($matches[1]);
        }
        if (preg_match("#^Content-Type: ([^;\r\n]*)(;[^\r\n]*)?\r\n#i", $line, $matches) != 0) {
            $this->download_mime_type = $matches[1];
            if (array_key_exists(2, $matches)) {
                $_ct_more = explode(';', str_replace(' ', '', trim($matches[2])));
                foreach ($_ct_more as $ct_more) {
                    $ct_bits = explode('=', $ct_more, 2);
                    if ((count($ct_bits) == 2) && (cms_strtolower_ascii($ct_bits[0]) == 'charset')) {
                        $this->charset = trim($ct_bits[1]);
                    }
                }
            }
        }

        $this->headers[] = $line;
    }

    /**
     * Try a bit harder to detect the character encoding, in case it was not in an HTTP header.
     * Perform a data conversion is requested.
     */
    protected function detect_character_encoding()
    {
        if ((($this->charset === null) || (/*Fudge for if AddDefaultCharset is set*/($this->download_mime_type === 'text/plain') && ($this->download_url !== null) && (substr($this->download_url, -4) == '.txt'))) && ($this->data !== null) && ($this->data !== false)) {
            $matches = [];
            if (preg_match('#^\s*<' . '?xml[^<>]*\s+encoding="([^"]+)"#', $this->data, $matches) != 0) {
                $this->charset = trim($matches[1]);
            } elseif (preg_match('#<meta\s+http-equiv="Content-Type"\s+content="[^"]*;\s*charset=([^"]+)"#i', substr($this->data, 0, 2048), $matches) != 0) {
                $this->charset = trim($matches[1]);
            } else {
                list($this->charset, $bom) = detect_string_bom($this->data);
                if (($this->convert_to_internal_encoding) && ($bom !== null)) {
                    $this->data = substr($this->data, strlen($bom));
                }

                if ($this->charset === null) {
                    $this->charset = $this->default_charset;
                }
            }
        }

        if (($this->convert_to_internal_encoding) && ($this->charset !== null) && ($this->data !== null)) {
            require_code('character_sets');
            $this->data = convert_to_internal_encoding($this->data, $this->charset);
        }
    }

    /**
     * Copy response metadata from one object to another.
     * Does not copy response data, that is returned as a string.
     *
     * @param  object $from The child response
     * @param  object $to The parent response
     * @return ?string The data downloaded (null: error)
     */
    protected function copy_from_other(object $from, object $to) : ?string
    {
        $to->download_mime_type = $from->download_mime_type;
        $to->download_size = $from->download_size;
        $to->download_url = $from->download_url;
        $to->download_mtime = $from->download_mtime;
        $to->message = $from->message;
        $to->message_b = $from->message_b;
        $to->new_cookies = array_merge($to->new_cookies, $from->new_cookies);
        if ($to->filename !== null) {
            $to->filename = $from->filename;
        }
        $to->charset = $from->charset;
        $to->headers = $from->headers;
        $to->implementation_used = $from->implementation_used;

        return $from->data;
    }

    /**
     * Cleanup object.
     */
    public function __destruct()
    {
        // Cleanup
        if ($this->raw_post_handle !== null) {
            fclose($this->raw_post_handle);
        }
    }
}

/**
 * cURL-based HttpDownloader.
 *
 * @package core
 */
class HttpDownloaderCurl extends HttpDownloader
{
    // Class processing configuration
    protected $add_content_type_header_manually = true;
    protected $add_files_manually = true;

    // Data collection
    protected $curl_headers = [];
    protected $curl_body = null;
    protected $attempts_remaining = 3; // Used when CURL errors with a partial file

    /**
     * See if this class may run.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return integer The execution priority
     */
    public function may_run_for(string $url, array $options = []) : int
    {
        $this->url_parts = @cms_parse_url_safe(normalise_idn_url($url));
        $this->read_in_options($options);

        if (!function_exists('curl_init')) {
            return HttpDownloader::RUN_PRIORITY_NO;
        }

        if ((function_exists('get_value')) && (get_value('prefer_curl') === '0')) {
            return HttpDownloader::RUN_PRIORITY_NO;
        }

        if (isset($this->url_parts['scheme'])) {
            if ($this->url_parts['scheme'] != 'http') {
                $curl_version = curl_version();
                if (((is_string($curl_version)) && (strpos($curl_version, 'OpenSSL') !== false)) || ((is_array($curl_version)) && (array_key_exists('ssl_version', $curl_version)))) {
                    return HttpDownloader::RUN_PRIORITY_HIGH;
                }
            }

            if (($this->url_parts['scheme'] == 'http') && ((function_exists('get_value')) && (get_value('prefer_curl') === '1'))) {
                return HttpDownloader::RUN_PRIORITY_HIGH;
            }
        }

        return HttpDownloader::RUN_PRIORITY_LOW;
    }

    /**
     * Do the inner call using a particular downloader method.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return ~?string The data downloaded (null: error) (false: backend failed)
     */
    protected function _run(string $url, array $options)
    {
        $ch = curl_init($this->do_ip_forwarding ? $this->connecting_url : $url);

        // Cookie prep
        if (!empty($this->cookies)) {
            curl_setopt($ch, CURLOPT_COOKIE, $this->get_cookie_string());
        }

        // SSL prep
        $verifypeer_enabled =
            $this->verifypeer_enabled &&
            !$this->do_ip_forwarding &&
            ((!function_exists('get_value')) || (get_value('disable_ssl_for__' . $this->url_parts['host']) === '0'));
        if ($verifypeer_enabled) {
            curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
            curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 1);
        } else {
            curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
            curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
        }
        if (ini_get('curl.cainfo') == '') {
            $crt_path = get_file_base() . '/data/curl-ca-bundle.crt';
            curl_setopt($ch, CURLOPT_CAINFO, $crt_path);
        }
        curl_setopt($ch, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1); // https://jve.linuxwall.info/blog/index.php?post/TLS_Survey
        // ^ If you get errors about ciphers not matching up, it's possibly cURL being buggy and misreporting a firewall problem. The above config settings should be rock-solid on almost every single server.

        // Misc settings
        //if (!$this->no_redirect) @curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); // we can do better ourselves anyway and protect against file:// exploits.
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, intval($this->timeout));
        curl_setopt($ch, CURLOPT_TIMEOUT, intval($this->timeout));

        // Request type
        if (($this->http_verb !== null) && (cms_strtolower_ascii($this->http_verb) == 'head')) {
            curl_setopt($ch, CURLOPT_NOBODY, true); // Branch needed as doing a HEAD via CURLOPT_CUSTOMREQUEST can cause a timeout bug in cURL
        } else {
            curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $this->http_verb);
        }

        // Headers
        $curl_headers = [];
        if ($this->accept !== null) {
            $curl_headers[] = 'Accept: ' . $this->accept;
        }
        if ($this->accept_charset !== null) {
            $curl_headers[] = 'Accept-Charset: ' . $this->accept_charset;
        }
        if ($this->accept_language !== null) {
            $curl_headers[] = 'Accept-Language: ' . $this->accept_language;
        }
        foreach ($this->extra_headers as $key => $val) {
            $curl_headers[] = $key . ': ' . $val;
        }
        if ($this->raw_content_type !== null) {
            if (!isset($this->extra_headers['Content-Type'])) {
                $curl_headers[] = 'Content-Type: ' . $this->raw_content_type;
            }
        }

        if ($this->post_params !== null) {
            if ($this->raw_post_handle !== null) {
                curl_setopt($ch, CURLOPT_PUT, true);
                curl_setopt($ch, CURLOPT_INFILE, $this->raw_post_handle);
                curl_setopt($ch, CURLOPT_INFILESIZE, filesize($this->raw_post_path));
            } else {
                curl_setopt($ch, CURLOPT_POST, true);
                curl_setopt($ch, CURLOPT_POSTFIELDS, $this->raw_payload);
                if (!empty($this->files)) { // We will be doing a multipart/form-data call
                    $curl_headers[] = 'Content-Type: multipart/form-data; boundary="--cms' . $this->divider . '"; charset=' . get_charset();
                }
                $curl_headers[] = 'Expect:'; // Suppress  automatic Expect header
            }
        }
        if ($this->do_ip_forwarding) {
            $curl_headers[] = 'Host: ' . $this->url_parts['host'];
        }
        if (!empty($curl_headers)) {
            curl_setopt($ch, CURLINFO_HEADER_OUT, true);
            curl_setopt($ch, CURLOPT_HTTPHEADER, $curl_headers);
        }

        // cURL options that will create headers also
        if ($this->auth !== null) {
            curl_setopt($ch, CURLOPT_USERPWD, implode(':', $this->auth));
        }
        if ($this->referer !== null) {
            curl_setopt($ch, CURLOPT_REFERER, $this->referer);
        }
        if (is_string($this->ua)) {
            curl_setopt($ch, CURLOPT_USERAGENT, $this->ua);
        }
        if ($this->byte_limit !== null) {
            curl_setopt($ch, CURLOPT_RANGE, strval($this->content_size) . '-' . strval(($this->byte_limit == 0) ? 0 : ($this->byte_limit - 1)));
        } elseif ($this->content_size > 0) {
            curl_setopt($ch, CURLOPT_RANGE, strval($this->content_size) . '-');
        }
        if ($this->write_to_file !== null) {
            curl_setopt($ch, CURLOPT_FILE, $this->write_to_file);
        }

        // Proxy settings
        $proxy = function_exists('get_option') ? get_option('proxy') : '';
        if (($proxy != '') && (!is_local_machine($this->url_parts['host'])) && (!is_our_server($this->url_parts['host']))) {
            $port = get_option('proxy_port');
            curl_setopt($ch, CURLOPT_PROXY, $proxy . ':' . $port);
            $proxy_user = get_option('proxy_user');
            if ($proxy_user != '') {
                $proxy_password = get_option('proxy_password');
                curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxy_user . ':' . $proxy_password);
            }
        }

        // Data collection
        curl_setopt($ch, CURLOPT_HEADERFUNCTION, [$this, 'file_curl_headers']);
        curl_setopt($ch, CURLOPT_WRITEFUNCTION, [$this, 'file_curl_body']);

        // Response
        $curl_result = curl_exec($ch);
        $info = curl_getinfo($ch);

        // Debug logging
        $_log_file = get_custom_file_base() . '/data_custom/http.log';
        $log_message = loggable_date() . 'curl_getinfo: ' . serialize($info) . "\n";
        $log_message = loggable_date() . 'RESPONSE headers: ' . serialize($this->curl_headers) . "\n";
        if (is_file($_log_file)) {
            require_code('files');
            $log_file = cms_fopen_text_write($_log_file, true, 'ab');
            fwrite($log_file, $log_message);
            flock($log_file, LOCK_UN);
            fclose($log_file);
        }

        /*if ((count($curl_headers)!=0) && (!empty($this->files))) { // Useful for debugging
            var_dump(curl_getinfo($ch,CURLINFO_HEADER_OUT));exit();
        }*/

        if (($curl_result === false) && ((curl_errno($ch) != CURLE_WRITE_ERROR) || ($this->byte_limit === null))) {
            // Error
            $error = curl_error($ch);
            $curl_errno = curl_errno($ch);
            curl_close($ch);

            // Try again on a partial file error if we did not exceed our attempts remaining
            if (($curl_errno == CURLE_PARTIAL_FILE) && ($this->attempts_remaining > 0)) {
                $this->attempts_remaining--;
                return $this->_run($url, $options); // $this->content_size should be set to how many bytes we have so far
            }

            // If any of these error codes are returned, treat as an internal failure on the cURL end, thus we should use a different backend
            $possible_internal_curl_errors = [1, 2, 4, 5, 16, 34, 35, 41, 43, 45, 48, 52, 53, 54, 55, 56, 58, 59, 60, 64, 66, 77, 80, 81, 82, 83, 89, 90, 91, 92, 95, 96, 97, 99, 101];
            if (in_array($curl_errno, $possible_internal_curl_errors)) {
                return false;
            }

            // Probably a request/response error, not a cURL one
            if ($this->trigger_error) {
                warn_exit(protect_from_escaping($error), false, true);
            } else {
                $this->message = 'curl-error-' . protect_from_escaping(strval($curl_errno));
                $this->message_b = protect_from_escaping($error);
            }
            return null;
        }

        // Response metadata that cURL lets us gather easily
        $this->download_mime_type = curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
        if ($this->download_mime_type === false) {
            $this->download_mime_type = null;
        }
        $this->download_size = curl_getinfo($ch, CURLINFO_CONTENT_LENGTH_DOWNLOAD);
        if ($this->download_size == -1) {
            $this->download_size = null;
        }
        $this->download_url = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
        if ($this->download_url == $this->connecting_url) {
            $this->download_url = $url;
        }
        $this->message = $this->fix_non_standard_statuses(strval(curl_getinfo($ch, CURLINFO_HTTP_CODE)));
        if ($this->message == '206') {
            $this->message = '200'; // We don't care about partial-content return code, as Composr implementation gets ranges differently and we check '200' as a return result
        }
        if (($this->download_mime_type !== null) && (strpos($this->download_mime_type, ';') !== false) && (strpos($this->download_mime_type, 'charset=') !== false)) {
            $this->charset = substr($this->download_mime_type, 8 + strpos($this->download_mime_type, 'charset='));
            $this->download_mime_type = substr($this->download_mime_type, 0, strpos($this->download_mime_type, ';'));
        }

        curl_close($ch);

        // Process HTTP status
        switch ((substr($this->message, 0, 1) == '2') ? '200' : $this->message) {
            case '200':
            case '301':
            case '302':
            case '303':
            case '307':
                break;

            case '401':
            case '403':
                if (!$this->ignore_http_status) {
                    if ($this->trigger_error) {
                        warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNAUTHORIZED', escape_html($url)), false, true);
                    } else {
                        $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNAUTHORIZED', escape_html($url));
                    }
                }
                break;

            case '404':
            case '410':
                if (!$this->ignore_http_status) {
                    if ($this->trigger_error) {
                        warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_NOT_FOUND', escape_html($url)), false, true);
                    } else {
                        $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_NOT_FOUND', escape_html($url));
                    }
                }
                break;

            case '400':
            case '429':
                if (!$this->ignore_http_status) {
                    if ($this->trigger_error) {
                        warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_BAD_REQUEST', escape_html($url)), false, true);
                    } else {
                        $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_BAD_REQUEST', escape_html($url));
                    }
                }
                break;

            case '500':
            case '502':
            case '503':
            case '504':
                if (!$this->ignore_http_status) {
                    if ($this->trigger_error) {
                        warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_SERVER_ERROR', escape_html($url)), false, true);
                    } else {
                        $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_SERVER_ERROR', escape_html($url));
                    }
                }
                break;

            default:
                if (!$this->ignore_http_status) {
                    if ($this->trigger_error) {
                        warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNKNOWN', escape_html($url), escape_html($this->message)), false, true);
                    } else {
                        $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNKNOWN', escape_html($url), escape_html($this->message));
                    }
                }
                break;
        }

        // Receive headers
        foreach ($this->curl_headers as $header) {
            $matches = [];

            $this->read_in_headers($header);

            if (preg_match("#^Refresh: (\d*);(.*)\r\n#i", $header, $matches) != 0) {
                if ($this->filename === null) {
                    $this->filename = urldecode(basename($matches[1]));
                }

                if (strpos($matches[1], '://') === false) {
                    $matches[1] = qualify_url($matches[1], $url, true);
                }

                if ($this->no_redirect) {
                    $text = null;
                    $this->download_url = $matches[2];
                } else {
                    $_options = $options;
                    unset($_options['post_params']);
                    $inner_request = _cms_http_request($matches[2], $_options);
                    $text = $this->copy_from_other($inner_request, $this);
                }

                return $text;
            }

            if (preg_match("#^Location: (.*)\r\n#i", $header, $matches) != 0) {
                if ($this->filename === null) {
                    $this->filename = urldecode(basename($matches[1]));
                }

                if (strpos($matches[1], '://') === false) {
                    $matches[1] = qualify_url($matches[1], $url, true);
                }

                if (($matches[1] != $url) && (preg_match('#^3\d\d$#', $this->message) != 0)) {
                    if ($this->no_redirect) {
                        $text = null;
                        $this->download_url = $matches[1];
                    } else {
                        $_options = $options;
                        unset($_options['post_params']);
                        $inner_request = _cms_http_request($matches[1], $_options);
                        $text = $this->copy_from_other($inner_request, $this);
                    }

                    return $text;
                }
            }
        }

        // Receive body
        if ($this->curl_body === null) {
            $this->curl_body = '';
        }
        if ((substr($this->message, 0, 1) != '2') && (!$this->ignore_http_status)) {
            $this->curl_body = null;
        }

        return $this->curl_body;
    }

    /**
    * Callback for receiving cURL headers.
    *
    * @param  resource $ch cURL resource handle
    * @param  string $header Header
    * @return integer Length of header
    *
    * @ignore
    */
    protected function file_curl_headers($ch, string $header) : int
    {
        $this->curl_headers[] = $header;
        return strlen($header);
    }

    /**
    * Callback for receiving a part of the cURL body.
    *
    * @param  resource $ch cURL resource handle
    * @param  string $str Body part
    * @return integer Length of body part
    *
    * @ignore
    */
    protected function file_curl_body($ch, string $str) : int
    {
        if ($this->curl_body === null) {
            $this->curl_body = '';
        }

        if ($this->write_to_file !== null) {
            $_str = $str;

            if (($this->byte_limit !== null) && ((($this->content_size + strlen($str)) > $this->byte_limit))) {
                $_str = substr($_str, 0, ($this->byte_limit - $this->content_size));
            }
            fwrite($this->write_to_file, $_str);
            $this->content_size += strlen($_str);
            return strlen($_str);
        }

        $this->curl_body .= $str;
        $this->content_size = strlen($this->curl_body);
        if (($this->byte_limit !== null) && ($this->content_size > $this->byte_limit)) {
            $this->curl_body = substr($this->curl_body, 0, $this->byte_limit);
            $this->content_size = $this->byte_limit;
            return strlen($str) - ($this->content_size - $this->byte_limit);
        }
        return strlen($str);
    }
}

/**
 * Sockets-based HttpDownloader.
 *
 * @package core
 */
class HttpDownloaderSockets extends HttpDownloader
{
    /**
     * See if this class may run.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return integer The execution priority
     */
    public function may_run_for(string $url, array $options = []) : int
    {
        $this->url_parts = @cms_parse_url_safe(normalise_idn_url($url));
        $this->read_in_options($options);

        if (isset($this->url_parts['scheme']) && (!GOOGLE_APPENGINE) && (php_function_allowed('fsockopen'))) {
            if ($this->url_parts['scheme'] == 'http') {
                return HttpDownloader::RUN_PRIORITY_MEDIUM;
            }
            if ((defined('OPENSSL_VERSION_NUMBER')) && ($this->url_parts['scheme'] == 'https')) {
                return HttpDownloader::RUN_PRIORITY_MEDIUM;
            }
        }

        return HttpDownloader::RUN_PRIORITY_NO;
    }

    /**
     * Do the inner call using a particular downloader method.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return ~?string The data downloaded (null: error) (false: backend failed)
     */
    protected function _run(string $url, array $options)
    {
        static $has_ctype_xdigit = null;
        if ($has_ctype_xdigit === null) {
            $has_ctype_xdigit = function_exists('ctype_xdigit');
        }

        $errno = 0;
        $errstr = '';

        $proxy = function_exists('get_option') ? get_option('proxy') : null;
        if ($proxy == '') {
            $proxy = null;
        }
        if (($proxy !== null) && (!is_local_machine($this->connect_to)) && (!is_our_server($this->connect_to))) {
            $proxy_port = get_option('proxy_port');
            $mysock = @fsockopen($proxy, intval($proxy_port), $errno, $errstr, $this->timeout);
        } else {
            $connect_to = $this->connect_to;
            if (cms_strtolower_ascii(substr($url, 0, 8)) == 'https://') {
                $connect_to = 'tls://' . $this->connect_to;
                $default_port = 443;
            } else {
                $default_port = 80;
            }
            $port = array_key_exists('port', $this->url_parts) ? $this->url_parts['port'] : $default_port;
            $mysock = @fsockopen($connect_to, $port, $errno, $errstr, $this->timeout);
        }

        if ($mysock !== false) {
            if (function_exists('stream_set_timeout')) {
                if (@stream_set_timeout($mysock, intval($this->timeout), fmod($this->timeout, 1.0) / 1000000.0) === false) {
                    $mysock = false;
                }
            } elseif (function_exists('socket_set_timeout')) {
                if (@socket_set_timeout($mysock, intval($this->timeout), fmod($this->timeout, 1.0) / 1000000.0) === false) {
                    $mysock = false;
                }
            }
        }

        if ($mysock !== false) {
            $url2 = array_key_exists('path', $this->url_parts) ? $this->url_parts['path'] : '/';
            if (array_key_exists('query', $this->url_parts)) {
                $url2 .= '?' . $this->url_parts['query'];
            }

            if (($proxy !== null) && (!is_local_machine($this->connect_to)) && (!is_our_server($this->connect_to))) {
                $out = '';
                $out .= $this->http_verb . ' ' . escape_header($url) . " HTTP/1.1\r\n";
                $proxy_user = get_option('proxy_user');
                if ($proxy_user != '') {
                    $proxy_password = get_option('proxy_password');
                    $out .= 'Proxy-Authorization: Basic ' . base64_encode($proxy_user . ':' . $proxy_password) . "\r\n";
                }
            } else {
                $out = $this->http_verb . ' ' . escape_header($url2) . " HTTP/1.1\r\n";
            }
            $out .= 'Host: ' . $this->url_parts['host'] . "\r\n";
            $out .= 'Connection: Close' . "\r\n";
            $out .= $this->get_header_string();
            $out .= $this->raw_payload;
            $out .= "\r\n";

            @fwrite($mysock, $out);
            if ($this->raw_post_handle !== null) {
                while (!feof($this->raw_post_handle)) {
                    $data = @fread($this->raw_post_handle, 1024 * 100);
                    if (($data !== false) && ($data !== null)) {
                        @fwrite($mysock, $data);
                    } else {
                        break;
                    }
                }
            }
            $data_started = false;
            $input = '';
            $input_len = 0;
            $first_fail_time = null;
            $chunked = false;
            $buffer_unprocessed = '';
            $_frh = [$mysock];
            $_fwh = null;
            $time_init = time();
            $line = '';
            while ((!@feof($mysock)) || (($chunked) && ($buffer_unprocessed != ''))) { // @'d because socket might have died. If so fread will will return false and hence we'll break
                if ((empty($_frh)) || ((function_exists('stream_select')) && (!@stream_select($_frh, $_fwh, $_fwh, intval($this->timeout), intval(fmod($this->timeout, 1.0) / 1000000.0))))) {
                    // Detected timeout
                    if (($chunked) && ($buffer_unprocessed != '')) {
                        // Either a broken chunked stream or no input at all yet...

                        $line = false; // Manual timeout
                        if ($this->trigger_error) {
                            warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_CONNECTION_STALLED', escape_html($url)));
                        } else {
                            $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_CONNECTION_STALLED', escape_html($url));
                        }

                        $this->message = 'connection-stalled';

                        return null; // Termination case
                    } else {
                        // We don't know our stream is broken, so let's consider us done, just that the connection was not detected on our as properly closed with EOF...
                        break; // Termination case
                    }
                }

                $line = @fread($mysock, 32000);

                if (($input === '') && ($time_init + $this->timeout < time())) {
                    $line = false; // Manual timeout
                }
                if ($line === false) {
                    // Implicit EOF, so terminate
                    if ((!$chunked) || ($buffer_unprocessed == '')) {
                        $_frh = []; // Termination case: Will trigger a detected timeout on next loop cycle. Note stream_select also can empty $_frh, and we are simulating the same thing here but without any overhead on waiting for stream_select
                        $line = '';
                    } else {
                        break; // Termination case
                    }
                }
                if ($line == '') {
                    if ($first_fail_time !== null) {
                        if ($first_fail_time < time() - 5) {
                            break; // Termination case
                        }
                    } else {
                        $first_fail_time = time();
                    }
                } else {
                    $first_fail_time = null;
                }

                if ($data_started) {
                    $line = $buffer_unprocessed . $line;
                    $buffer_unprocessed = '';

                    if ($chunked) {
                        if ((isset($line[1])) && ($line[0] == "\r") && ($line[1] == "\n")) {
                            $line = @substr($line, 2);
                        }

                        $hexdec_chunk_details = '';
                        $chunk_line_length = strlen($line);
                        for ($hexdec_read = 0; $hexdec_read < $chunk_line_length; $hexdec_read++) {
                            $chunk_char = $line[$hexdec_read];
                            if ($chunk_char == "\r") {
                                break;
                            } else {
                                if ($has_ctype_xdigit) {
                                    $chunk_char_is_hex = ctype_xdigit($chunk_char);
                                } else {
                                    $chunk_char_ord = ord($chunk_char);
                                    $chunk_char_is_hex = ($chunk_char_ord >= 48 && $chunk_char_ord <= 57 || $chunk_char_ord >= 65 && $chunk_char_ord <= 90 || $chunk_char_ord >= 97 && $chunk_char_ord <= 122);
                                }
                            }
                            if ($chunk_char_is_hex) {
                                $hexdec_chunk_details .= $chunk_char;
                            } else {
                                break;
                            }
                        }
                        if ($hexdec_chunk_details == '') { // No data
                            // No usable data here...

                            if ($line == '') {
                                // We just have no data yet
                                continue;
                            }

                            // Corrupt, so terminate as empty
                            if ($this->write_to_file === null) {
                                $input = '';
                            } else {
                                ftruncate($this->write_to_file, 0);
                            }
                            $this->content_size = 0;
                            $input_len = 0;
                            $buffer_unprocessed = '';

                            $this->message = 'connection-stalled';

                            break; // Termination case
                        }
                        $chunk_end_pos = strpos($line, "\r\n");
                        if ($chunk_end_pos === false) {
                            $buffer_unprocessed = $line;
                            continue;
                        }
                        $amount_wanted = hexdec($hexdec_chunk_details);
                        $amount_available = $chunk_line_length - ($chunk_end_pos + 2);
                        if ($amount_available < $amount_wanted) { // Chunk was more than what we grabbed, so we need to iterate more (more fread) to parse
                            $buffer_unprocessed = $line;
                            continue;
                        }
                        $buffer_unprocessed = substr($line, $chunk_end_pos + 2 + $amount_wanted); // May be some more extra read
                        $line = substr($line, $chunk_end_pos + 2, $amount_wanted);
                        if ($line == '') {
                            // An empty chunk
                            break; // Termination case
                        }

                        $input_len += $amount_wanted;
                    } else {
                        $input_len += strlen($line);
                    }

                    if ($this->write_to_file === null) {
                        $input .= $line;
                        $this->content_size = $input_len;

                        if (($this->byte_limit !== null) && ($this->content_size >= $this->byte_limit)) {
                            $input = substr($input, 0, $this->byte_limit);
                            $input_len = $this->byte_limit;
                            $this->content_size = $this->byte_limit;
                            break; // Termination case: Success, got all the input we need
                        }
                    } else {
                        if (($this->byte_limit !== null) && ($input_len >= $this->byte_limit)) {
                            $_line = substr($line, 0, strlen($line) - ($input_len - $this->byte_limit));
                            fwrite($this->write_to_file, $_line);
                            $input_len = $this->byte_limit;
                            $this->content_size = $this->byte_limit;
                            break; // Termination case: Success, got all the input we need
                        }

                        fwrite($this->write_to_file, $line);
                        $this->content_size = $input_len;
                    }
                } elseif ($line != '') {
                    $old_line = $line;
                    $lines = explode("\r\n", $line);

                    $tally = 0;
                    foreach ($lines as $lno => $line) {
                        if (empty($line)) {
                            $tally += 2;

                            $data_started = true;
                            $buffer_unprocessed = @substr($old_line, $tally);
                            if ($buffer_unprocessed === false) {
                                $buffer_unprocessed = '';
                            }

                            if (($chunked) && (isset($buffer_unprocessed[1])) && ($buffer_unprocessed[0] == "\r") && ($buffer_unprocessed[1] == "\n")) {
                                // Explicit termination after headers, we should not try and wait for more
                                break 2; // Termination case
                            }

                            break;
                        }

                        $line .= "\r\n";

                        $tally += strlen($line);

                        $matches = [];
                        if (preg_match("#Transfer-Encoding: chunked\r\n#i", $line, $matches) != 0) {
                            $chunked = true;
                        }

                        $this->read_in_headers($line);

                        if (preg_match("#^Refresh: (\d*);(.*)\r\n#i", $line, $matches) != 0) {
                            if ($this->filename === null) {
                                $this->filename = urldecode(basename($matches[1]));
                            }

                            @fclose($mysock);

                            if (strpos($matches[1], '://') === false) {
                                $matches[1] = qualify_url($matches[1], $url, true);
                            }

                            if ($this->no_redirect) {
                                $text = null;
                                $this->download_url = $matches[2];
                            } else {
                                $_options = $options;
                                unset($_options['post_params']);
                                $inner_request = _cms_http_request($matches[2], $_options);
                                $text = $this->copy_from_other($inner_request, $this);
                            }

                            return $text; // Termination case (was delegated)
                        }

                        if (preg_match("#^Location: (.*)\r\n#i", $line, $matches) != 0) {
                            if ($this->filename === null) {
                                $this->filename = urldecode(basename($matches[1]));
                            }

                            if (strpos($matches[1], '://') === false) {
                                $matches[1] = qualify_url($matches[1], $url, true);
                            }

                            if ($matches[1] != $url) {
                                if (($matches[1] != $url) && (preg_match('#^3\d\d$#', $this->message) != 0)) {
                                    @fclose($mysock);

                                    if ($this->no_redirect) {
                                        $text = null;
                                        $this->download_url = $matches[1];
                                    } else {
                                        $_options = $options;
                                        unset($_options['post_params']);
                                        $inner_request = _cms_http_request($matches[1], $_options);
                                        $text = $this->copy_from_other($inner_request, $this);
                                    }

                                    return $text; // Termination case (was delegated)
                                }
                            }
                        }

                        if (preg_match("#HTTP/(\d*\.\d*) (\d*) #", $line, $matches) != 0) {
                            // 200=Ok
                            // 301/302/303/307=Redirected: Not good, we should not be here
                            // 401/403=Unauthorized
                            // 404=Not found
                            // 400/500=Internal error
                            // 405=Method not allowed

                            $this->message = $this->fix_non_standard_statuses($matches[2]);

                            switch ((substr($this->message, 0, 1) == '2') ? '200' : $this->message) {
                                case '200':
                                    // Good
                                    break;

                                case '301':
                                case '302':
                                case '303':
                                case '307':
                                    // We'll expect a location header
                                    break;

                                case '401':
                                case '403':
                                    if (!$this->ignore_http_status) {
                                        @fclose($mysock);

                                        if ($this->trigger_error) {
                                            warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNAUTHORIZED', escape_html($url)), false, true);
                                        } else {
                                            $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNAUTHORIZED', escape_html($url));
                                        }

                                        return null;
                                    }
                                    break;

                                case '404':
                                case '410':
                                    if (!$this->ignore_http_status) {
                                        @fclose($mysock);

                                        if ($this->trigger_error) {
                                            warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_NOT_FOUND', escape_html($url)), false, true);
                                        } else {
                                            $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_NOT_FOUND', escape_html($url));
                                        }

                                        return null;
                                    }
                                    break;

                                case '400':
                                case '429':
                                    if (!$this->ignore_http_status) {
                                        @fclose($mysock);

                                        if ($this->trigger_error) {
                                            warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_BAD_REQUEST', escape_html($url)), false, true);
                                        } else {
                                            $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_BAD_REQUEST', escape_html($url));
                                        }

                                        return null;
                                    }
                                    break;

                                case '500':
                                case '502':
                                case '503':
                                case '504':
                                    if (!$this->ignore_http_status) {
                                        @fclose($mysock);

                                        if ($this->trigger_error) {
                                            warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_SERVER_ERROR', escape_html($url)), false, true);
                                        } else {
                                            $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_SERVER_ERROR', escape_html($url));
                                        }

                                        return null;
                                    }
                                    break;

                                case '405':
                                    if ($this->byte_limit == 0 && !$this->no_redirect && empty($this->post_params)) { // Try again as non-HEAD request if we just did a HEAD request that got "Method not allowed"
                                        @fclose($mysock);

                                        if ($this->no_redirect) {
                                            $text = null;
                                            $this->download_url = $matches[1];
                                        } else {
                                            $_options = $options;
                                            $_options['byte_limit'] = 1;
                                            $inner_request = _cms_http_request($matches[1], $_options);
                                            $text = $this->copy_from_other($inner_request, $this);
                                        }

                                        return $text;
                                    }
                                    // no break

                                default:
                                    if (!$this->ignore_http_status) {
                                        @fclose($mysock);

                                        if ($this->trigger_error) {
                                            warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNKNOWN', escape_html($url), escape_html($this->message)), false, true);
                                        } else {
                                            $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_STATUS_UNKNOWN', escape_html($url), escape_html($this->message));
                                        }

                                        return null;
                                    }
                                    break;
                            }
                        }
                    }
                }
            }

            // (loop ended)

            // Process any non-chunked extra buffer (chunked would have been handled in main loop)
            if (!$chunked) {
                if ($buffer_unprocessed != '') {
                    $input_len += strlen($buffer_unprocessed);
                    $this->content_size = $input_len;

                    if ($this->write_to_file === null) {
                        $input .= $buffer_unprocessed;

                        if (($this->byte_limit !== null) && ($this->content_size >= $this->byte_limit)) {
                            $input = substr($input, 0, $this->byte_limit);
                            $this->content_size = $this->byte_limit;
                        }
                    } else {
                        $_buffer_unprocessed = $buffer_unprocessed;

                        if (($this->byte_limit !== null) && ($this->content_size >= $this->byte_limit)) {
                            $_buffer_unprocessed = substr($_buffer_unprocessed, 0, strlen($_buffer_unprocessed) - ($this->content_size - $this->byte_limit));
                            $this->content_size = $this->byte_limit;
                        }
                        fwrite($this->write_to_file, $_buffer_unprocessed);
                    }
                }
            }

            @fclose($mysock);

            if (!$data_started) {
                if ($this->byte_limit === 0) {
                    return ''; // Termination case
                }

                if ($this->trigger_error) {
                    warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_NO_SERVER', escape_html($url)), false, true);
                } else {
                    $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_NO_SERVER', escape_html($url));
                }
                $this->message = 'no-data';


                return null; // Termination case
            }

            $size_expected = $this->download_size;
            if ($this->byte_limit !== null) {
                if ($this->byte_limit < $this->download_size) {
                    $size_expected = $this->byte_limit;
                }
            }
            if ($input_len < $size_expected) {
                if ($this->trigger_error) {
                    warn_exit(do_lang_tempcode('HTTP_DOWNLOAD_CUT_SHORT', escape_html($url), escape_html(integer_format($size_expected)), escape_html(integer_format($input_len))), false, true);
                } else {
                    $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_CUT_SHORT', escape_html($url), escape_html(integer_format($size_expected)), escape_html(integer_format($input_len)));
                }
                $this->message = 'short-data';

                return $input; // Termination case
            }

            return $input; // Termination case: success
        }

        $errstr = cms_error_get_last();
        if ($this->trigger_error) {
            if ($errstr == '') {
                $errstr = strval($errno);
            }
            $error = do_lang_tempcode('_HTTP_DOWNLOAD_NO_SERVER', escape_html($url), escape_html($errstr));
            warn_exit($error, false, true);
        } else {
            $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_NO_SERVER', escape_html($url));
        }
        $this->message = 'could not connect to host'; // Could append  (' . $errstr . ') but only when debugging because we use this string like a constant in some places

        return null; // Termination case
    }
}

/**
 * File-wrapper-based HttpDownloader.
 *
 * @package core
 */
class HttpDownloaderFileWrapper extends HttpDownloader
{
    /**
     * See if this class may run.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return integer The execution priority
     */
    public function may_run_for(string $url, array $options = []) : int
    {
        return HttpDownloader::RUN_PRIORITY_LOW;
    }

    /**
     * Do the inner call using a particular downloader method.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return ~?string The data downloaded (null: error) (false: backend failed)
     */
    protected function _run(string $url, array $options)
    {
        // PHP streams method
        //  Imperfect, does not support $this->download_url
        $errno = 0;
        $errstr = '';
        if (($errno != 110) && (($errno != 10060) || (ini_get('default_socket_timeout') == '1')) && ((ini_get('allow_url_fopen') == '1') || (php_function_allowed('ini_set')))) {
            cms_ini_set('allow_url_fopen', '1');

            $this->timeout_before = ini_get('default_socket_timeout');
            cms_ini_set('default_socket_timeout', strval($this->timeout));

            if ($this->raw_post_handle !== null) {
                $this->raw_payload .= cms_file_get_contents_safe($this->raw_post_path, FILE_READ_LOCK);
            }

            $crt_path = get_file_base() . '/data/curl-ca-bundle.crt';

            $verifypeer_enabled =
                $this->verifypeer_enabled &&
                !$this->do_ip_forwarding &&
                ((!function_exists('get_value')) || (get_value('disable_ssl_for__' . $this->url_parts['host']) === '0'));

            $header = rtrim((($this->url_parts['host'] != $this->connect_to) ? ('Host: ' . $this->url_parts['host'] . "\r\n") : '') . $this->get_header_string());

            $proxy = function_exists('get_option') ? get_option('proxy') : '';
            if ($proxy != '') {
                $proxy_user = get_option('proxy_user');
                if ($proxy_user != '') {
                    $proxy_password = get_option('proxy_password');
                    $header .= "\r\n" . 'Proxy-Authorization: Basic ' . base64_encode($proxy_user . ':' . $proxy_password);
                }
            }

            $opts = [
                'http' => [
                    'method' => $this->http_verb,
                    'header' => $header,
                    'content' => $this->raw_payload,
                    'follow_location' => $this->no_redirect ? 0 : 1,
                    'ignore_errors' => $this->ignore_http_status,
                    'ssl' => [
                        'verify_peer' => $verifypeer_enabled,
                        'verify_peer_name' => $verifypeer_enabled,
                        'cafile' => $crt_path,
                        'SNI_enabled' => true,
                        'ciphers' => 'TLSv1',
                    ],
                ],
            ];

            if (is_string($this->ua)) {
                $opts['http']['user_agent'] = $this->ua;
            }

            if ($proxy != '') {
                $port = get_option('proxy_port');
                $opts['http']['proxy'] = 'tcp://' . $proxy . ':' . $port;
                $opts['http']['request_fulluri'] = true;
            }

            $context = stream_context_create($opts);

            if (($this->byte_limit === null) && ($this->write_to_file === null)) {
                if ($this->trigger_error) {
                    push_suppress_error_death(true); // We have to use this rather than '@' because stream errors don't go into cms_error_get_last() always and thus this is the only way to let those show (as logged/attached messages)
                    $read_file = file_get_contents($this->connecting_url, false, $context);
                    pop_suppress_error_death();
                } else {
                    $read_file = @file_get_contents($this->connecting_url, false, $context);
                }
            } else {
                if ($this->trigger_error) {
                    push_suppress_error_death(true); // We have to use this rather than '@' because stream errors don't go into cms_error_get_last() always and thus this is the only way to let those show (as logged/attached messages)
                    $_read_file = fopen($this->connecting_url, 'rb', false, $context);
                    pop_suppress_error_death();
                } else {
                    $_read_file = @fopen($this->connecting_url, 'rb', false, $context);
                }
                if ($_read_file !== false) {
                    $read_file = '';
                    while ((!feof($_read_file)) && (($this->byte_limit === null) || ($this->content_size < $this->byte_limit))) {
                        $data = fread($_read_file, 1024);
                        if ($data !== false) {
                            $this->content_size += strlen($data);
                            if (($this->byte_limit !== null) && ($data != ''/*substr would fail with false*/) && ($this->content_size >= $this->byte_limit)) {
                                $data = substr($data, 0, strlen($data) - ($this->content_size - $this->byte_limit));
                                $this->content_size = $this->byte_limit;
                            }

                            $read_file .= $data;
                            if ($this->write_to_file !== null) {
                                fwrite($this->write_to_file, $data);
                            }
                        }
                    }
                    fclose($_read_file);
                } else {
                    $read_file = false;
                }
            }

            cms_ini_set('allow_url_fopen', '0');
            cms_ini_set('default_socket_timeout', $this->timeout_before);

            if (isset($http_response_header)) {
                foreach ($http_response_header as $header) {
                    $this->read_in_headers($header . "\r\n");

                    if ($this->no_redirect) {
                        $matches = [];
                        if (preg_match("#^Refresh: (\d*);(.*)#i", $header, $matches) != 0) {
                            if ($this->filename === null) {
                                $this->filename = urldecode(basename($matches[1]));
                            }

                            if (strpos($matches[1], '://') === false) {
                                $matches[1] = qualify_url($matches[1], $url, true);
                            }

                            $read_file = null;
                            $this->download_url = $matches[2];
                        }

                        if (preg_match("#^Location: (.*)#i", $header, $matches) != 0) {
                            if ($this->filename === null) {
                                $this->filename = urldecode(basename($matches[1]));
                            }

                            if (strpos($matches[1], '://') === false) {
                                $matches[1] = qualify_url($matches[1], $url, true);
                            }

                            if (($matches[1] != $url) && (isset($this->headers[0])) && (preg_match('#\s3\d\d\s#', $this->headers[0]) != 0)) {
                                $read_file = null;
                                $this->download_url = $matches[1];
                            }
                        }
                    }
                }
            }

            if ($read_file !== false) {
                return $read_file;
            }
        }

        $errstr = cms_error_get_last();
        if ($this->trigger_error) {
            if ($errstr == '') {
                $errstr = strval($errno);
            }
            $error = do_lang_tempcode('_HTTP_DOWNLOAD_NO_SERVER', escape_html($url), escape_html($errstr));
            warn_exit($error, false, true);
        } else {
            $this->message_b = do_lang_tempcode('HTTP_DOWNLOAD_NO_SERVER', escape_html($url));
        }
        $this->message = 'could not connect to host'; // Could append  (' . $errstr . ') but only when debugging because we use this string like a constant in some places

        return null;
    }
}

/**
 * Filesystem-wrapper-based HttpDownloader.
 *
 * @package core
 */
class HttpDownloaderFilesystem extends HttpDownloader
{
    /**
     * See if this class may run.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return integer The execution priority
     */
    public function may_run_for(string $url, array $options = []) : int
    {
        $this->url_parts = @cms_parse_url_safe(normalise_idn_url($url));
        $this->read_in_options($options);

        $faux = function_exists('get_value') ? get_value('http_faux_loopback') : null;
        if ((!cms_empty_safe($faux)) && ($this->post_params === null) && (empty($this->files))) { // NB: Does not support cookies, accept headers, referers
            if (substr($faux, 0, 1) != '#') {
                $faux = '#' . $faux . '#i';
            }
            if (preg_match($faux, $url) != 0) {
                return HttpDownloader::RUN_PRIORITY_VERY_HIGH;
            }
        }

        return HttpDownloader::RUN_PRIORITY_NO;
    }

    /**
     * Do the inner call using a particular downloader method.
     *
     * @param  URLPATH $url The URL to download
     * @param  array $options Map of options (see the properties of the HttpDownloader class for what you may set)
     * @return ~?string The data downloaded (null: error) (false: backend failed)
     */
    protected function _run(string $url, array $options)
    {
        $parsed = cms_parse_url_safe(normalise_idn_url($url));
        $parsed_base_url = cms_parse_url_safe(get_custom_base_url());
        $file_base = get_custom_file_base();

        // URLs always use forward slashes, but on Windows our path could use backslashes. Change these to forward slashes so our preg_replace works.
        $file_base = str_replace('\\', '/', $file_base);

        $file_base = preg_replace('#' . preg_quote(urldecode($parsed_base_url['path'])) . '$#', '', $file_base);
        $file_path = $file_base . urldecode($parsed['path']);

        if ((php_function_allowed('escapeshellcmd')) && (php_function_allowed('shell_exec')) && (substr($file_path, -4) == '.php')) {
            $cmd = 'DOCUMENT_ROOT=' . cms_escapeshellarg(dirname(get_file_base()));
            $cmd .= ' PATH_TRANSLATED=' . cms_escapeshellarg($file_path);
            $cmd .= ' SCRIPT_NAME=' . cms_escapeshellarg($file_path);
            if (is_string($this->ua)) {
                $cmd .= ' HTTP_USER_AGENT=' . cms_escapeshellarg($this->ua);
            }
            $cmd .= ' QUERY_STRING=' . cms_escapeshellarg($parsed['query']);
            $cmd .= ' HTTP_HOST=' . cms_escapeshellarg($parsed['host']);
            $cmd .= ' ' . escapeshellcmd(find_php_path(true)) . ' ' . cms_escapeshellarg($file_path);
            $contents = shell_exec($cmd);
            $split_pos = strpos($contents, "\r\n\r\n");
            if ($split_pos !== false) {
                $_headers = explode("\r\n", substr($contents, 0, $split_pos));
                foreach ($_headers as $line) {
                    $this->read_in_headers($line);
                }
                $contents = substr($contents, $split_pos + 4);
            }
        } else {
            if ($this->trigger_error) {
                $contents = cms_file_get_contents_safe($file_path, FILE_READ_LOCK);
            } else {
                $contents = @cms_file_get_contents_safe($file_path, FILE_READ_LOCK);
            }

            require_code('mime_types');
            require_code('files');
            $this->download_mime_type = get_mime_type(get_file_extension($file_path), true);
            $this->download_size = filesize($file_path);
            $this->download_mtime = filemtime($file_path);
            $this->message = '200';
            $this->filename = basename($file_path);
        }

        if ($this->byte_limit !== null) {
            $contents = substr($contents, 0, $this->byte_limit);
        }

        $this->content_size = strlen($contents);

        if ($this->write_to_file !== null) {
            fwrite($this->write_to_file, $contents);
            return '';
        }
        return $contents;
    }
}
