<?php /*

 Composr
 Copyright (c) Christopher Graham, 2004-2024

 See docs/LICENSE.md for full licensing information.


 NOTE TO PROGRAMMERS:
   Do not edit this file. If you need to make changes, save your changed file to the appropriate *_custom folder
   **** If you ignore this advice, then your website upgrades (e.g. for bug fixes) will likely kill your changes ****

*/

/**
 * @license    http://opensource.org/licenses/cpal_1.0 Common Public Attribution License
 * @copyright  Christopher Graham
 * @package    health_check
 */

/**
 * Hook class.
 */
class Hook_health_check_mistakes_build extends Hook_Health_Check
{
    protected $category_label = 'Build mistakes';

    /**
     * Standard hook run function to run this category of health checks.
     *
     * @param  ?array $sections_to_run Which check sections to run (null: all)
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     * @param  boolean $show_unusable_categories Whether to include categories that might not be accessible for some reason
     * @return array A pair: category label, list of results
     */
    public function run(?array $sections_to_run, int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null, bool $show_unusable_categories = false) : array
    {
        $this->process_checks_section('testManualWebStandards', 'Manual checks for web standards', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testGuestAccess', 'Guest access (tests the configured/contextual pages only)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testBrokenLinks', 'Broken links (tests the configured/contextual pages only) (slow)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testIncompleteContent', 'Incomplete content (tests the configured/contextual pages only)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testLocalLinking', 'Local linking (tests the configured/contextual pages only)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testBrokenWebPostForms', 'Broken web POST forms (tests the configured/contextual pages only)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testForgottenIcons', 'Default icons', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testSpellingComcodePages', 'Spellchecking of all Comcode pages (slow)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testSpellingPages', 'Spellchecking of pages (tests the configured/contextual pages only)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testSpellingContent', 'Spellchecking of miscellaneous content', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testComcodePageHeadings', 'Comcode page headings (tests the configured/contextual pages only)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testComcodePageFormFields', 'Comcode page form fields (tests the configured/contextual pages only)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testWebStandards', 'Web standards (tests the configured/contextual pages only)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);
        $this->process_checks_section('testCommonMistakePatterns', 'Common mistake patterns (page_errors.xml) (tests the configured/contextual pages only)', $sections_to_run, $check_context, $manual_checks, $automatic_repair, $use_test_data_for_pass, $urls_or_page_links, $comcode_segments);

        return [$this->category_label, $this->results];
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testManualWebStandards(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }

        if (!$manual_checks) {
            return;
        }

        $this->stateCheckManual('Check [url="Open Graph metadata"]https://developers.facebook.com/tools/debug/[/url] on any key pages you expect to be shared');

        if ($check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS) {
            $this->log('Skipped; running on specific page links.');
            return;
        }

        // external_health_check (on maintenance sheet)

        $this->stateCheckManual('Check [url="HTML5 validation"]https://validator.w3.org/[/url] (take warnings with a pinch of salt, not every suggestion is appropriate)');
        $this->stateCheckManual('Check [url="CSS validation"]https://jigsaw.w3.org/css-validator/[/url] (take warnings with a pinch of salt, not every suggestion is appropriate)');
        $this->stateCheckManual('Check [url="WCAG validation"]https://achecker.achecks.ca/checker/index.php[/url] (take warnings with a pinch of salt, not every suggestion is appropriate)');

        $this->stateCheckManual('Check schema.org/microformats validation on either [url="Google"]https://developers.google.com/search/docs/appearance/structured-data[/url] or [url="Bing"]https://www.bing.com/webmasters/help/url-inspection-55a30305[/url] or [url="Yandex"]https://webmaster.yandex.com/tools/microtest/[/url], on any key pages you want to be semantic');

        $this->stateCheckManual('Do a [url="general check"]https://www.woorank.com/[/url] (take warnings with a pinch of salt, not every suggestion is appropriate)');
        $this->stateCheckManual('Do a [url="general check"]https://website.grader.com/[/url] (take warnings with a pinch of salt, not every suggestion is appropriate)');
        $this->stateCheckManual('Do a [url="general check"]https://webhint.io/[/url] (take warnings with a pinch of salt, not every suggestion is appropriate)');

        $this->stateCheckManual('Test in Mozilla Firefox');
        $this->stateCheckManual('Test in Mozilla Firefox (mobile)');
        $this->stateCheckManual('Test in IE10'); // TODO: remove when we no longer support it
        $this->stateCheckManual('Test in IE11'); // TODO: remove when we no longer support it
        $this->stateCheckManual('Test in Microsoft Edge');
        $this->stateCheckManual('Test in Google Chrome');
        $this->stateCheckManual('Test in Google Chrome (mobile)');
        $this->stateCheckManual('Test in Safari');
        $this->stateCheckManual('Test in Safari (mobile)');

        $this->stateCheckManual('Check the website would look good if printed');
        $this->stateCheckManual('Check the website would look good if generated as a PDF');
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testGuestAccess(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }
        if ($check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS) {
            $this->log('Skipped; running on specific page links.');
            return;
        }

        $page_links = $this->process_urls_into_page_links([':', ':login']);

        foreach ($page_links as $page_link) {
            $http_result = $this->get_page_http_content($page_link);

            $this->assertTrue(!in_array($http_result->message, ['401', '403']), '"' . $page_link . '" page is not allowing guest access');
        }
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testBrokenLinks(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }

        $page_links = $this->process_urls_into_page_links($urls_or_page_links);

        $html_segments = [];
        foreach ($page_links as $page_link) {
            $html = $this->get_page_content($page_link, $check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS);

            if ($html === null) {
                $this->stateCheckSkipped(do_lang('HC_PAGE_DOWNLOAD_ERROR', $page_link));
                continue;
            }

            $html_segments[] = $html;
        }

        if ($comcode_segments !== null) {
            foreach ($comcode_segments as $comcode_segment) {
                $html_segments[] = static_evaluate_tempcode(comcode_to_tempcode($comcode_segment));
            }
        }

        $urls = [];
        foreach ($html_segments as $html) {
            $urls = array_merge($urls, $this->get_embed_urls_from_data($html));
            $urls = array_merge($urls, $this->get_link_urls_from_data($html));
        }
        $urls = array_unique($urls);

        $_urls = [];
        foreach ($urls as $url) {
            // Exceptions
            if (preg_match('#^https://vk\.com/share\.php#', $url) != 0) {
                continue;
            }
            if (preg_match('#^https://twitter.com/intent/tweet#', $url) != 0) {
                // Twitter is blocking unknown user agents
                continue;
            }

            if (substr($url, 0, 2) == '//') {
                $url = 'http:' . $url;
            }

            // Don't check local URLs, we're interested in broken remote links (local validation is too much)
            if (substr($url, 0, strlen(get_base_url()) + 1) == get_base_url() . '/') {
                continue;
            }
            if (strpos($url, '://') === false) {
                continue;
            }

            $_urls[] = $url;
        }

        require_code('urls2');

        foreach ($_urls as $url) {
            // Check
            /*
            $data = http_get_contents($url, ['byte_limit' => 0, 'trigger_error' => false]);
            $ok = ($data !== null);
            */
            $message = '';
            $ok = check_url_exists($url, 60 * 60 * 24 * 1, true, 3, $message);
            if ($message === null) {
                $message = 'UNKNOWN';
            }
            $message_text = do_lang('BROKEN_LINK_PROBLEM__' . str_replace('-', '_', $message), $url, $message, null, null, false); // Support custom messages for specific status codes
            if ($message_text === null) {
                $message_text = do_lang('BROKEN_LINK_PROBLEM', $url, $message);
            }
            $this->assertTrue($ok, $message_text);
        }
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testIncompleteContent(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }

        if ((!$manual_checks) && ($comcode_segments === null)) {
            $this->stateCheckSkipped('Will not check automatically because there could be false positives');
            return;
        }

        $page_links = $this->process_urls_into_page_links($urls_or_page_links);

        $html_segments = [];
        foreach ($page_links as $page_link) {
            $html = $this->get_page_content($page_link, $check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS);

            if ($html === null) {
                $this->stateCheckSkipped(do_lang('HC_PAGE_DOWNLOAD_ERROR', $page_link));
                continue;
            }

            $html_segments['[tt]' . $page_link . '[/tt]'] = $html;
        }

        if ($comcode_segments !== null) {
            foreach ($comcode_segments as $field_title => $comcode_segment) {
                $html_segments[$field_title] = static_evaluate_tempcode(comcode_to_tempcode($comcode_segment));
            }
        }

        foreach ($html_segments as $field_title => $html) {
            $check_for = ['TODO', 'FIXME', 'Lorem Ipsum'];
            foreach ($check_for as $c) {
                $this->assertTrue(strpos($html, $c) === false, do_lang('INCOMPLETE_CONTENT_PROBLEM', $c, $field_title));
            }
        }
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testLocalLinking(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }

        if (is_local_machine(get_base_url_hostname())) {
            return;
        }

        $page_links = $this->process_urls_into_page_links($urls_or_page_links);

        $html_segments = [];
        foreach ($page_links as $page_link) {
            $html = $this->get_page_content($page_link, $check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS);

            if ($html === null) {
                $this->stateCheckSkipped(do_lang('HC_PAGE_DOWNLOAD_ERROR', $page_link));
                continue;
            }

            $html_segments['[tt]' . $page_link . '[/tt]'] = $html;
        }

        if ($comcode_segments !== null) {
            foreach ($comcode_segments as $field_title => $comcode_segment) {
                $html_segments[$field_title] = static_evaluate_tempcode(comcode_to_tempcode($comcode_segment));
            }
        }

        $problematic_host_prefixes = [];
        foreach (get_localhost_names_and_ips() as $ip_or_hostname) {
            $problematic_host_prefixes[] = preg_quote($ip_or_hostname, '#') . '($|:|/)';
        }
        $problematic_host_prefixes[] = '10\.';
        $problematic_host_prefixes[] = '192\.168\.';
        if (!health_check__is_test_site()) {
            $problematic_host_prefixes[] = '(staging|sandbox)\.' . preg_quote(preg_replace('#^www\.#', '', get_base_url_hostname()), '#') . '($|:|/)';
        }

        foreach ($html_segments as $field_title => $html) {
            $regexp = '#https?://(';
            foreach ($problematic_host_prefixes as $i => $host_prefix) {
                if ($i != 0) {
                    $regexp .= '|';
                }
                $regexp .= $host_prefix;
            }
            $regexp .= ')#';

            $this->assertTrue(preg_match($regexp, $html) == 0, do_lang('LOCAL_LINKING_PROBLEM', $field_title));
        }
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testBrokenWebPostForms(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context != CHECK_CONTEXT__SPECIFIC_PAGE_LINKS) {
            $this->log('Skipped; not running on specific page links.');
            return;
        }

        $page_links = $this->process_urls_into_page_links($urls_or_page_links);

        $html_segments = [];
        foreach ($page_links as $page_link) {
            $html = $this->get_page_content($page_link, $check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS);

            if ($html === null) {
                $this->stateCheckSkipped(do_lang('HC_PAGE_DOWNLOAD_ERROR', $page_link));
                continue;
            }

            $html_segments['[tt]' . $page_link . '[/tt]'] = $html;
        }

        if ($comcode_segments !== null) {
            foreach ($comcode_segments as $field_title => $comcode_segment) {
                $html_segments[$field_title] = static_evaluate_tempcode(comcode_to_tempcode($comcode_segment));
            }
        }

        foreach ($html_segments as $field_title => $html) {
            if (stripos($html, '<form') !== false) {
                $matches = [];
                $num_matches = preg_match_all('#<form[^<>]*method="POST">#i', $html, $matches);
                for ($i = 0; $i < $num_matches; $i++) {
                    $match = $matches[0][$i];

                    $matches_action = [];
                    $has_action = (preg_match('#action=["\']([^"\']*)["\']#i', $match, $matches_action) != 0);
                    $this->assertTrue($has_action, do_lang('FORM_ACTION_PROBLEM'));

                    if ($has_action) {
                        $url = html_entity_decode($matches_action[1], ENT_QUOTES);

                        if ($url == '#' || $url == '#!') {
                            continue;
                        }

                        $is_absolute_url = (strpos($url, '://') !== false);
                        $this->assertTrue($is_absolute_url, do_lang('FORM_ACTION_RELATIVE_PROBLEM'));

                        if ($is_absolute_url) {
                            $result = cms_http_request($url, ['trigger_error' => false]);
                            $this->assertTrue($result->message == '400', do_lang('FORM_ACTION_ERROR_HANDLING_PROBLEM', $url, $result->message));
                        }
                    }
                }
            }
        }
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testForgottenIcons(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }
        if ($check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS) {
            $this->log('Skipped; running on specific page links.');
            return;
        }

        $test = find_theme_image('favicon', false, true, $GLOBALS['FORUM_DRIVER']->get_theme(''));
        $this->assertTrue($test != 'themes/default/images/favicon.ico', 'Still using default favicon (the favicon theme image needs editing)');

        $test = find_theme_image('webclipicon', false, true, $GLOBALS['FORUM_DRIVER']->get_theme(''));
        $this->assertTrue($test != 'themes/default/images/webclipicon.ico', 'Still using default webclipicon (the webclipicon theme image needs editing)');
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testSpellingComcodePages(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }
        if ($check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS) {
            $this->log('Skipped; running on specific page links.');
            return;
        }

        require_code('files2');
        require_code('spelling');

        if (_find_spell_checker() === null) {
            $this->assertTrue(false, 'pspell/enchant required for this test');
            return;
        }

        $paths = ['pages/comcode_custom/' . user_lang()];
        $zones = find_all_zones();
        foreach ($zones as $zone) {
            if ($zone != 'docs') { // Big, and we check this in an automated test
                $paths[] = $zone . '/pages/comcode_custom/' . user_lang();
            }
        }
        $all_files = [];
        foreach ($paths as $_path) {
            $files = get_directory_contents(get_file_base() . '/' . $_path, $_path, null, false, true, ['txt']);
            $all_files = array_merge($all_files, $files);
        }

        foreach ($all_files as $path) {
            $c = cms_file_get_contents_safe(get_file_base() . '/' . $path);
            $text = clean_comcode_for_spellcheck($c);

            $misspellings = run_spellcheck($text, null, true, false);
            if ($misspellings === null) {
                $misspellings = []; // Ignore spell check errors
            }
            foreach (array_keys($misspellings) as $word) {
                $this->assertTrue(false, do_lang('SPELLING_PROBLEM', $word, $path));
            }
        }

        spellchecker_shutdown();
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testSpellingPages(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }

        require_code('spelling');

        if (_find_spell_checker() === null) {
            $this->assertTrue(false, 'pspell/enchant required for this test');
            return;
        }

        if (($comcode_segments !== null) && (get_option('enable_spell_check') !== '1')) {
            return;
        }

        $page_links = $this->process_urls_into_page_links($urls_or_page_links);

        $html_segments = [];
        foreach ($page_links as $page_link) {
            $html = $this->get_page_content($page_link, $check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS);

            if ($html === null) {
                $this->stateCheckSkipped(do_lang('HC_PAGE_DOWNLOAD_ERROR', $page_link));
                continue;
            }

            $html_segments['[tt]' . $page_link . '[/tt]'] = $html;
        }

        if ($comcode_segments !== null) {
            foreach ($comcode_segments as $field_title => $comcode_segment) {
                $html_segments[$field_title] = static_evaluate_tempcode(comcode_to_tempcode($comcode_segment));
            }
        }

        foreach ($html_segments as $field_title => $html) {
            $text = clean_html_for_spellcheck($html);

            $misspellings = run_spellcheck($text, null, true, false);
            if ($misspellings === null) {
                $misspellings = []; // Ignore spell check errors
            }
            foreach (array_keys($misspellings) as $word) {
                $this->assertTrue(false, do_lang('SPELLING_PROBLEM', $word, $field_title));
            }
        }

        spellchecker_shutdown();
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testSpellingContent(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }
        if ($check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS) {
            $this->log('Skipped; running on specific page links.');
            return;
        }

        require_code('spelling');
        require_code('database_relations');

        if (_find_spell_checker() === null) {
            $this->assertTrue(false, 'pspell/enchant required for this test');
            return;
        }

        $okay_words = [
            // Stuff there by default
            'american',
            'cms',
            'composr',
            'ecommerce',
            'facebook',
            'firstname',
            'lastname',
            'ocproducts',
            'semihtml',
            'sku',
            'skype',
            'sr',
            'usergroup',
            'viewable',
            'yy',
            'amazon',
            'wishlist',
            'bandcamp',
            'dailymotion',
            'discord',
            'dropbox',
            'flattr',
            'github',
            'gitlab',
            'tiktok',
            'gitlab',
            'instagram',
            'linkedin',
            'patreon',
            'paypal',
            'pinterest',
            'quora',
            'reddit',
            'slashdot',
            'snapchat',
            'soundcloud',
            'spotify',
            'tumblr',
            'vimeo',
            'whatsapp',
            'youtube',
            'vk',
            'baidu',
            'tieba',
            'sina',
            'weibo',
            'tencentweibo',
            'viber',
            'wechat',
            'qzone',
            'playstation',
            'xbox',
            'steemit',
            'dtube',
            'mastodon',
            'diaspora',
            'bitcoin',
            'utopian',
        ];

        $spell_link = spellcheck_initialise();
        add_spellchecker_words_temp($spell_link, $okay_words);

        $field_types_wanted = [
            'SHORT_TEXT',
            '*SHORT_TEXT',
            'LONG_TEXT',
            '*LONG_TEXT',
            'SHORT_TRANS',
            '*SHORT_TRANS',
            'LONG_TRANS',
            '*LONG_TRANS',
        ];
        $or_list = '';
        foreach ($field_types_wanted as $field_type) {
            if ($or_list != '') {
                $or_list .= ' OR ';
            }
            $or_list .= db_string_equal_to('m_type', $field_type);
        }

        $db_fields = $GLOBALS['SITE_DB']->query_select('db_meta', ['m_table', 'm_name'], [], 'WHERE (' . $or_list . ')');
        foreach ($db_fields as $db_field) {
            $table = $db_field['m_table'];
            $name = $db_field['m_name'];

            if (in_array($table, [
                // User data
                'f_topics',
                'f_posts',

                // Irrelevant
                'config',
                'values',
                'f_emoticons',
                'url_id_monikers',
                'theme_images',
                'wordfilter', // Don't want to swear at the user running this either!

                // Will come in elsewhere
                'translate',
                'comcode_pages',
                'cached_comcode_pages',
            ])) {
                continue;
            }

            if (table_has_purpose_flag($table, TABLE_PURPOSE__FLUSHABLE | TABLE_PURPOSE__FLUSHABLE_AGGRESSIVE | TABLE_PURPOSE__AUTOGEN_STATIC | TABLE_PURPOSE__MISC_NO_MERGE)) {
                continue;
            }

            if (in_array($name, [
                // Irrelevant
                'author',
                'm_timezone_offset',
            ])) {
                continue;
            }

            $start = 0;
            $max = 100;
            do {
                $old_limit = cms_extend_time_limit(TIME_LIMIT_EXTEND__SLUGGISH);

                push_db_scope_check(true);
                $rows = $GLOBALS['SITE_DB']->query_select($table, [$name], [], '', $max, $start);
                pop_db_scope_check();

                foreach ($rows as $row) {
                    $_c = $row[$name];
                    if (is_integer($_c)) {
                        $c = get_translated_text($_c);
                    } else {
                        $c = $_c;
                    }
                    $text = clean_comcode_for_spellcheck($c);

                    $misspellings = run_spellcheck($text, null, true, false);
                    if ($misspellings === null) {
                        $misspellings = []; // Ignore spell check errors
                    }
                    foreach (array_keys($misspellings) as $word) {
                        $this->assertTrue(false, 'Misspelling: ' . $word . ' (' . $table . ')');
                    }
                }

                cms_set_time_limit($old_limit);

                $start += $max;
            } while (count($rows) == $max);
        }

        spellchecker_shutdown();
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testComcodePageHeadings(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }

        $page_links = $this->process_urls_into_page_links($urls_or_page_links);

        $_comcode_segments = [];
        foreach ($page_links as $page_link) {
            $page_content = $this->get_comcode_page_content($page_link);

            if ($page_content === null) {
                $this->stateCheckSkipped(do_lang('HC_PAGE_DOWNLOAD_ERROR', $page_link));
                continue;
            }

            list($comcode, $html, , $page) = $page_content;

            $_comcode_segments['[tt]' . $page_link . '[/tt]'] = [$comcode, $html, $page];
        }

        if (($comcode_segments !== null) && (get_page_name() == 'cms_comcode_pages')) {
            foreach ($comcode_segments as $comcode_segment) {
                $_comcode_segments[do_lang('PREVIEW')] = [$comcode_segment, static_evaluate_tempcode(comcode_to_tempcode($comcode_segment)), get_param_string('page', '')];
            }
        }

        foreach ($_comcode_segments as $field_title => $page_content) {
            list($comcode, $html, $page) = $page_content;

            if ((substr($page, 0, 1) != '_') && (substr($page, 0, 6) != 'panel_') && (trim($comcode) != '') && (trim($html) != '')) {
                $found_l1 = ((strpos($html, '<h1') !== false) || (strpos($comcode, '[title]') !== false) || (strpos($comcode, '[title="1"]') !== false));
                $this->assertTrue($found_l1, do_lang('LEVEL_1_HEADERS_PROBLEM', $field_title));

                $matches = [];
                $okay_l2 = (strpos($html, '<h2') !== false) || (preg_match_all('#\n\[(b|font|size)\][^\.]+\[/(b|font|size)\]\r?\n#', $comcode, $matches) < 2);
                $this->assertTrue($okay_l2, do_lang('LEVEL_2_HEADERS_PROBLEM', $field_title));
            }
        }
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testComcodePageFormFields(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }

        $page_links = $this->process_urls_into_page_links($urls_or_page_links);

        $_comcode_segments = [];
        foreach ($page_links as $page_link) {
            $page_content = $this->get_comcode_page_content($page_link);

            if ($page_content === null) {
                $this->stateCheckSkipped(do_lang('HC_PAGE_DOWNLOAD_ERROR', $page_link));
                continue;
            }

            list(, $html) = $page_content;

            $_comcode_segments['[tt]' . $page_link . '[/tt]'] = $html;
        }

        if ($comcode_segments !== null) {
            foreach ($comcode_segments as $comcode_segment) {
                $_comcode_segments[do_lang('PREVIEW')] = static_evaluate_tempcode(comcode_to_tempcode($comcode_segment));
            }
        }

        foreach ($_comcode_segments as $field_title => $html) {
            $html = preg_replace('#<form.*</form>#Uis', '', $html);
            $this->assertTrue(preg_match('#<(select|textarea|input|button)#i', $html) == 0, do_lang('FORM_ELEMENT_PROBLEM', $field_title));
        }
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testWebStandards(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }

        $page_links = $this->process_urls_into_page_links($urls_or_page_links);

        $html_segments = [];
        foreach ($page_links as $page_link) {
            $html = $this->get_page_content($page_link, $check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS);

            if ($html === null) {
                $this->stateCheckSkipped(do_lang('HC_PAGE_DOWNLOAD_ERROR', $page_link));
                continue;
            }

            $html_segments['[tt]' . $page_link . '[/tt]'] = $html;
        }

        if ($comcode_segments !== null) {
            foreach ($comcode_segments as $field_title => $comcode_segment) {
                $html_segments[$field_title] = static_evaluate_tempcode(comcode_to_tempcode($comcode_segment));
            }
        }

        $ignore_errors = explode("\n", get_option('hc_webstandards_safelist'));

        foreach ($html_segments as $field_title => $html) {
            require_lang('webstandards');
            require_code('crypt');
            require_code('webstandards');
            $results = check_xhtml($html, false, true, true, true, true, true, false);

            $errors = [];
            if ($results !== null) {
                foreach ($results['errors'] as $error) {
                    $ignore = false;
                    foreach ($ignore_errors as $ignore_error) {
                        $ignore_error = trim($ignore_error);

                        if (($ignore_error != '') && (strpos($error['error'], $ignore_error) !== false)) {
                            $ignore = true;
                            break;
                        }
                    }

                    if (!$ignore) {
                        $errors[] = $error['error'];
                    }
                }
            }

            $this->assertTrue(empty($errors), do_lang('WEB_STANDARDS_PROBLEM', $field_title, '[html]' . implode('; ', $errors) . '[/html]'));
        }
    }

    /**
     * Run a section of health checks.
     *
     * @param  integer $check_context The current state of the website (a CHECK_CONTEXT__* constant)
     * @param  boolean $manual_checks Mention manual checks
     * @param  boolean $automatic_repair Do automatic repairs where possible
     * @param  ?boolean $use_test_data_for_pass Should test data be for a pass [if test data supported] (null: no test data)
     * @param  ?array $urls_or_page_links List of URLs and/or page-links to operate on, if applicable (null: those configured)
     * @param  ?array $comcode_segments Map of field names to Comcode segments to operate on, if applicable (null: N/A)
     */
    public function testCommonMistakePatterns(int $check_context, bool $manual_checks = false, bool $automatic_repair = false, ?bool $use_test_data_for_pass = null, ?array $urls_or_page_links = null, ?array $comcode_segments = null)
    {
        if ($check_context == CHECK_CONTEXT__INSTALL) {
            $this->log('Skipped; we are running from installer.');
            return;
        }

        $patterns = [];

        require_code('xml');

        $pattern_path = get_file_base() . '/data_custom/xml_config/page_errors.xml';
        if (!is_file($pattern_path)) {
            $pattern_path = get_file_base() . '/data/xml_config/page_errors.xml';
        }
        $parsed = new CMS_simple_xml_reader(cms_file_get_contents_safe($pattern_path, FILE_READ_LOCK | FILE_READ_BOM));

        list($root_tag, $root_attributes, , $this_children) = $parsed->gleamed;
        if ($root_tag == 'pageErrors') {
            foreach ($this_children as $_child) {
                if (!is_array($_child)) {
                    continue;
                }
                list($row_tag, $row_attributes, $row_value, $row_children) = $_child;

                $regexp = null;
                $description = null;
                if ($row_tag == 'pattern') {
                    foreach ($row_children as $__child) {
                        if (!is_array($__child)) {
                            continue;
                        }
                        list($row_tag, $row_attributes, $row_value, $row_children) = $__child;
                        switch ($row_tag) {
                            case 'regexp':
                                $regexp = $row_value;
                                break;
                            case 'description':
                                $description = $row_value;
                                break;
                        }
                    }
                }

                if ($regexp !== null) {
                    if ($description === null) {
                        $description = '[tt]' . $regexp . '[/tt]';
                    }

                    $patterns[$regexp] = $description;
                }
            }
        }

        $page_links = $this->process_urls_into_page_links($urls_or_page_links);

        $html_segments = [];
        foreach ($page_links as $page_link) {
            $html = $this->get_page_content($page_link, $check_context == CHECK_CONTEXT__SPECIFIC_PAGE_LINKS);

            if ($html === null) {
                $this->stateCheckSkipped(do_lang('HC_PAGE_DOWNLOAD_ERROR', $page_link));
                continue;
            }

            foreach ($patterns as $regexp => $_description) {
                $matches = [];
                $result = @preg_match($regexp, $html, $matches);
                if ($result === false) {
                    attach_message('Bad regexp: ' . $regexp, 'warn', false, true);
                    continue;
                }
                /*if ($result != 0) { Debugging
                    @header('Content-Type: text/plain');@var_dump($matches);exit();
                }*/

                // Description could be a language string Tempcode
                require_code('tempcode_compiler');
                $description = template_to_tempcode($_description);

                $this->assertTrue($result == 0, static_evaluate_tempcode($description) . ' (' . $page_link . ')');
            }
        }
    }
}
