<?php /*

 Composr
 Copyright (c) ocProducts, 2004-2016

 See text/EN/licence.txt for full licencing information.


 NOTE TO PROGRAMMERS:
   Do not edit this file. If you need to make changes, save your changed file to the appropriate *_custom folder
   **** If you ignore this advice, then your website upgrades (e.g. for bug fixes) will likely kill your changes ****

*/

/*EXTRA FUNCTIONS: glob*/

/**
 * @license    http://opensource.org/licenses/cpal_1.0 Common Public Attribution License
 * @copyright  ocProducts Ltd
 * @package    core_database_drivers
 */

/*
    Known (intentional) issues in SQL support (we are targeting rough MySQL-4.3 parity, similar to SQL-92)
        We support a few SQL/MySQL functions, the ones the Composr db_function outputs and the very basic operators. However we prefix with 'X_' so that we don't accidentally code in assumptions about MySQL.
        We do not support SQL data types, we use Composr ones instead.
        We do not have any special table/field naming escaping support-- so you need to use names that aren't awkward
        MySQL-style auto-increment is supported, but actually done as key randomisation, once install has finished
        Indexes are not supported
        We ARE type strict, unlike MySQL (even MySQL strict mode won't complain if a type conversion is always lossless, such as integer to string)
        Data Control Language (DCL) is not supported
        Information schemas are not supported
        Semi-colons to split queries are not supported at the driver level
        Temporary tables are not supported
        Views are not supported
        Integrity checking (field constraints, CHECK) is not supported
        Transactions are not supported
        Full-text-search is not supported
        Special foreign key support is not supported
        INTERSECT and EXCEPT are not supported
        JOIN's are not supported in DELETE or UPDATE queries
        Character set support is just whatever Composr is set to; there is no special supported
        SELECT INTO is not supported
        LIMIT's on UPDATE queries not supported
        Default values for fields are not supported
        Field naming for things like COUNT(*) will not be consistent with MySQL
        You must specify the field names in INSERT queries
        Expressions in ORDER BY clauses will be ignored
    This database system is intended only for Composr, and not as a general purpose database. In Composr our philosophy is to write logic in PHP, not SQL, hence the subset supported.
    Also as we have to target MySQL-4.3 we can't implement some more sophisticated featured, in case programmers rely on them!
*/

/**
 * Standard code module initialisation function.
 *
 * @ignore
 */
function init__database__xml()
{
    global $SCHEMA_CACHE, $DIR_CONTENTS_CACHE;
    $SCHEMA_CACHE = array();
    $DIR_CONTENTS_CACHE = array();

    global $DELIMITERS_FLIPPED, $DELIMITERS, $SYMBOL_DELIMITER, $DELIMITERS_ALPHA;
    $DELIMITERS = array_merge(array("\t", ' ', "\n"), _get_sql_keywords());
    sort($DELIMITERS);
    $DELIMITERS_FLIPPED = array_flip($DELIMITERS);
    $SYMBOL_DELIMITER = array_flip(array("\t", ' ', "\n", '+', '-', '*', '/', '>', '<', '=', "'", '"', "\\'", '(', ')', ','));
    foreach ($DELIMITERS as $d) {
        if (!isset($DELIMITERS_ALPHA[$d[0]])) {
            $DELIMITERS_ALPHA[$d[0]] = array();
        }
        $DELIMITERS_ALPHA[$d[0]][] = $d;
    }

    global $TABLE_BASES;
    $TABLE_BASES = array();

    global $INT_TYPES, $STRING_TYPES;
    $INT_TYPES = array('REAL', 'AUTO', 'AUTO_LINK', 'INTEGER', 'UINTEGER', 'SHORT_INTEGER', 'BINARY', 'MEMBER', 'GROUP', 'TIME', 'integer');
    if (multi_lang_content()) {
        $INT_TYPES[] = 'SHORT_TRANS';
        $INT_TYPES[] = 'LONG_TRANS';
        $INT_TYPES[] = 'SHORT_TRANS__COMCODE';
        $INT_TYPES[] = 'LONG_TRANS__COMCODE';
    }
    $STRING_TYPES = array(
        'SHORT_TEXT' => 255,
        'LONG_TEXT' => null,
        'ID_TEXT' => 80,
        'MINIID_TEXT' => 40,
        'IP' => 40,
        'LANGUAGE_NAME' => 5,
        'URLPATH' => 255,
        'UINTEGER' => 10, // Fudge as we need to send in unsigned integers using strings, as PHP can't hold them
    );
    if (!multi_lang_content()) {
        $STRING_TYPES['SHORT_TRANS'] = 255;
        $STRING_TYPES['LONG_TRANS'] = null;
        $STRING_TYPES['SHORT_TRANS__COMCODE'] = 255;
        $STRING_TYPES['LONG_TRANS__COMCODE'] = null;
    }

    require_code('xml');

    if (php_function_allowed('set_time_limit')) {
        @set_time_limit(100); // XML DB is *slow*
    }
}

/**
 * Get a list of all SQL keywords
 *
 * @return array List of keywords
 *
 * @ignore
 */
function _get_sql_keywords()
{
    return array(
        'LEFT', 'RIGHT', // Join types
        'X_CONCAT', 'X_LENGTH', 'X_REPLACE', 'X_COALESCE',
        'X_SUBSTR', 'X_RAND', 'X_LEAST', 'X_GREATEST', 'X_MOD',
        'WHERE',
        'SELECT', 'FROM', 'AS', 'UNION', 'ALL', 'DISTINCT',
        'INSERT', 'INTO', 'VALUES', 'UPDATE', 'SET', 'DELETE',
        'ALTER', 'CREATE', 'X_CREATE_TABLE', 'DROP', 'X_DROP_TABLE', 'ADD', 'CHANGE', 'RENAME', 'DEFAULT', 'TABLE', 'PRIMARY', 'KEY',
        'LIKE', 'IF', 'NOT', 'IS', 'NULL', 'AND', 'OR', 'BETWEEN', 'IN', 'EXISTS',
        'GROUP', 'BY', 'ORDER', 'ASC', 'DESC',
        'JOIN', 'OUTER', 'INNER', 'ON',
        'COUNT', 'SUM', 'AVG', 'MAX', 'MIN', 'X_GROUP_CONCAT',
        'LIMIT',
        '+', '-', '*', '/',
        '<>', '>', '<', '>=', '<=', '=',
        '"', "'", "\\'",
        '(', ')', ',',
        // Anything else is put into a value token
        // Tokens are delimited by white space or one of the symbol tokens
    );
}

/**
 * Database Driver.
 *
 * @package    core_database_drivers
 */
class Database_Static_xml
{
    /**
     * Find whether the database may run GROUP BY unfettered with restrictions on the SELECT'd fields having to be represented in it or aggregate functions
     *
     * @return boolean Whether it can
     */
    public function can_arbitrary_groupby()
    {
        return true;
    }

    /**
     * Get the default user for making db connections (used by the installer as a default).
     *
     * @return string The default user for db connections
     */
    public function db_default_user()
    {
        return '';
    }

    /**
     * Get the default password for making db connections (used by the installer as a default).
     *
     * @return string The default password for db connections
     */
    public function db_default_password()
    {
        return '';
    }

    /**
     * Find whether subquery support is present
     *
     * @param  array $db A DB connection
     * @return boolean Whether it is
     */
    public function db_has_subqueries($db)
    {
        return true;
    }

    /**
     * Get a map of Composr field types, to actual database types.
     *
     * @return array The map
     */
    public function db_get_type_remap()
    {
        $type_remap = array(
            'AUTO' => 'AUTO',
            'AUTO_LINK' => 'AUTO_LINK',
            'INTEGER' => 'INTEGER',
            'UINTEGER' => 'UINTEGER',
            'SHORT_INTEGER' => 'SHORT_INTEGER',
            'REAL' => 'REAL',
            'BINARY' => 'BINARY',
            'MEMBER' => 'MEMBER',
            'GROUP' => 'GROUP',
            'TIME' => 'TIME',
            'LONG_TRANS' => 'LONG_TRANS',
            'SHORT_TRANS' => 'SHORT_TRANS',
            'LONG_TRANS__COMCODE' => 'integer',
            'SHORT_TRANS__COMCODE' => 'integer',
            'SHORT_TEXT' => 'SHORT_TEXT',
            'LONG_TEXT' => 'LONG_TEXT',
            'ID_TEXT' => 'ID_TEXT',
            'MINIID_TEXT' => 'MINIID_TEXT',
            'IP' => 'IP',
            'LANGUAGE_NAME' => 'LANGUAGE_NAME',
            'URLPATH' => 'URLPATH',
        );
        return $type_remap;
    }

    /**
     * Get SQL for creating a table index.
     *
     * @param  ID_TEXT $table_name The name of the table to create the index on
     * @param  ID_TEXT $index_name The index name (not really important at all)
     * @param  string $_fields Part of the SQL query: a comma-separated list of fields to use on the index
     * @param  array $db The DB connection to make on
     * @param  ID_TEXT $raw_table_name The table name with no table prefix
     * @param  string $unique_key_fields The name of the unique key field for the table
     * @return array List of SQL queries to run
     */
    public function db_create_index($table_name, $index_name, $_fields, $db, $raw_table_name, $unique_key_fields)
    {
        // Indexes not supported
        return array();
    }

    /**
     * Change the primary key of a table.
     *
     * @param  ID_TEXT $table_name The name of the table to create the index on
     * @param  array $new_key A list of fields to put in the new key
     * @param  array $db The DB connection to make on
     */
    public function db_change_primary_key($table_name, $new_key, $db)
    {
        $this->db_query('UPDATE db_meta SET m_type=X_REPLACE(m_type,\'*\',\'\') WHERE ' . db_string_equal_to('m_table', $table_name), $db);
        foreach ($new_key as $_new_key) {
            $this->db_query('UPDATE db_meta SET m_type=' . db_function('CONCAT', array('\'*\'', 'm_type')) . ' WHERE ' . db_string_equal_to('m_table', $table_name) . ' AND ' . db_string_equal_to('m_name', $_new_key), $db);
        }
    }

    /**
     * Get the ID of the first row in an auto-increment table (used whenever we need to reference the first).
     *
     * @return integer First ID used
     */
    public function db_get_first_id()
    {
        return 1;
    }

    /**
     * Get SQL for creating a new table.
     *
     * @param  ID_TEXT $table_name The table name
     * @param  array $fields A map of field names to Composr field types (with *#? encodings)
     * @param  array $db The DB connection to make on
     * @param  ID_TEXT $raw_table_name The table name with no table prefix
     * @param  boolean $save_bytes Whether to use lower-byte table storage, with tradeoffs of not being able to support all unicode characters; use this if key length is an issue
     * @return array List of SQL queries to run
     */
    public function db_create_table($table_name, $fields, $db, $raw_table_name, $save_bytes = false)
    {
        return array("X_CREATE_TABLE '" . $this->db_escape_string(serialize(array($table_name, $fields, $raw_table_name, $save_bytes))) . "'");
    }

    /**
     * Encode an SQL statement fragment for a conditional to see if two strings are equal.
     *
     * @param  ID_TEXT $attribute The attribute
     * @param  string $compare The comparison
     * @return string The SQL
     */
    public function db_string_equal_to($attribute, $compare)
    {
        return $attribute . "='" . $this->db_escape_string($compare) . "'";
    }

    /**
     * Encode an SQL statement fragment for a conditional to see if two strings are not equal.
     *
     * @param  ID_TEXT $attribute The attribute
     * @param  string $compare The comparison
     * @return string The SQL
     */
    public function db_string_not_equal_to($attribute, $compare)
    {
        return $attribute . "<>'" . $this->db_escape_string($compare) . "'";
    }

    /**
     * This function is internal to the database system, allowing SQL statements to be build up appropriately. Some databases require IS NULL to be used to check for blank strings.
     *
     * @return boolean Whether a blank string IS NULL
     */
    public function db_empty_is_null()
    {
        return false;
    }

    /**
     * Delete a table.
     *
     * @param  ID_TEXT $table_name The table name
     * @param  array $db The DB connection to delete on
     * @return array List of SQL queries to run
     */
    public function db_drop_table_if_exists($table_name, $db)
    {
        return array('X_DROP_TABLE ' . $table_name);
    }

    /**
     * Find whether table truncation support is present
     *
     * @return boolean Whether it is
     */
    public function db_supports_truncate_table()
    {
        return true;
    }

    /**
     * Determine whether the database is a flat file database, and thus not have a meaningful connect username and password.
     *
     * @return boolean Whether the database is a flat file database
     */
    public function db_is_flat_file_simple()
    {
        return true;
    }

    /**
     * Encode a LIKE string comparision fragement for the database system. The pattern is a mixture of characters and ? and % wildcard symbols.
     *
     * @param  string $pattern The pattern
     * @return string The encoded pattern
     */
    public function db_encode_like($pattern)
    {
        return $this->db_escape_string($pattern);
    }

    /**
     * Close the database connections. We don't really need to close them (will close at exit), just disassociate so we can refresh them.
     */
    public function db_close_connections()
    {
    }

    /**
     * Get a database connection. This function shouldn't be used by you, as a connection to the database is established automatically.
     *
     * @param  boolean $persistent Whether to create a persistent connection
     * @param  string $db_name The database name
     * @param  string $db_host The database host (the server)
     * @param  string $db_user The database connection username
     * @param  string $db_password The database connection password
     * @param  boolean $fail_ok Whether to on error echo an error and return with a null, rather than giving a critical error
     * @return ?array A database connection (null: failed)
     */
    public function db_get_connection($persistent, $db_name, $db_host, $db_user, $db_password, $fail_ok = false)
    {
        if ((strpos($db_name, '\\') === false) && (strpos($db_name, '/') === false)) {
            $db_name = get_custom_file_base() . '/uploads/website_specific/' . $db_name;
        }
        if (!file_exists($db_name)) { // Will create on first usage
            mkdir($db_name, 0777);
            require_code('files');
            fix_permissions($db_name);
            sync_file($db_name);
        }

        return array($db_name);
    }

    /**
     * Find whether full-text-search is present
     *
     * @param  array $db A DB connection
     * @return boolean Whether it is
     */
    public function db_has_full_text($db)
    {
        return false;
    }

    /**
     * Assemble part of a WHERE clause for doing full-text search
     *
     * @param  string $content Our match string (assumes "?" has been stripped already)
     * @param  boolean $boolean Whether to do a boolean full text search
     * @return string Part of a WHERE clause for doing full-text search
     */
    public function db_full_text_assemble($content, $boolean)
    {
        return '';
    }

    /**
     * Find whether full-text-boolean-search is present
     *
     * @return boolean Whether it is
     */
    public function db_has_full_text_boolean()
    {
        return false;
    }

    /**
     * Escape a string so it may be inserted into a query. If SQL statements are being built up and passed using db_query then it is essential that this is used for security reasons. Otherwise, the abstraction layer deals with the situation.
     *
     * @param  string $string The string
     * @return string The escaped string
     */
    public function db_escape_string($string)
    {
        $string = fix_bad_unicode($string);

        return addslashes($string);
    }

    /**
     * Adjust an SQL query to apply offset/limit restriction.
     *
     * @param  string $query The complete SQL query
     * @param  ?integer $max The maximum number of rows to affect (null: no limit)
     * @param  ?integer $start The start row to affect (null: no specification)
     */
    public function apply_sql_limit_clause(&$query, $max = null, $start = 0)
    {
        if (($max !== null) && ($start !== null)) {
            $query .= ' LIMIT ' . strval($start) . ',' . strval($max);
        } elseif ($max !== null) {
            $query .= ' LIMIT ' . strval($max);
        } elseif ($start !== null) {
            $query .= ' LIMIT ' . strval($start) . ',30000000';
        }
    }

    /**
     * This function is a very basic query executor. It shouldn't usually be used by you, as there are abstracted versions available.
     *
     * @param  string $query The complete SQL query
     * @param  array $db A DB connection
     * @param  ?integer $max The maximum number of rows to affect (null: no limit)
     * @param  ?integer $start The start row to affect (null: no specification)
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  boolean $get_insert_id Whether to get the autoincrement ID created for an insert query
     * @param  boolean $no_syndicate Whether to force the query to execute on the XML database driver (won't optimise by using MySQL). Useful for calls happening for multi-part queries from within this DB driver
     * @param  boolean $save_as_volatile Whether we are saving as a 'volatile' file extension
     * @return ?mixed The results (null: no results), or the insert ID
     */
    public function db_query($query, $db, $max = null, $start = null, $fail_ok = false, $get_insert_id = false, $no_syndicate = false, $save_as_volatile = false)
    {
        global $DELIMITERS_FLIPPED, $DELIMITERS, $SYMBOL_DELIMITER;

        // LEXING STAGE
        // ------------

        $i = 0;
        $query .= ' '; // Cheat so that we do not have to handle the end state differently
        $len = strlen($query);
        $tokens = array();
        $current_token = '';
        $doing_symbol_delimiter = true;
        while ($i < $len) {
            $next = $query[$i];

            if (($next == "'") || ($next == '"')) {
                if (trim($current_token) != '') {
                    if (isset($DELIMITERS_FLIPPED[strtoupper($current_token)])) {
                        $tokens[] = strtoupper($current_token);
                    } else {
                        $tokens[] = $current_token;
                    }
                }
                $current_token = '';

                $i++;
                while ($i < $len) {
                    $next = $query[$i];

                    if ($next == '\\') {
                        $i++;
                        $next = $query[$i];
                        $current_token .= $next;
                    } else {
                        if (($next == "'") || ($next == '"')) {
                            $tokens[] = "'";
                            $tokens[] = $current_token;
                            $tokens[] = "'";
                            break;
                        } else {
                            $current_token .= $next;
                        }
                    }

                    $i++;
                }
                $current_token = '';
                $doing_symbol_delimiter = true;
            } else {
                $symbol_delimiter_coming = ((isset($SYMBOL_DELIMITER[$next])) && ((isset($DELIMITERS_FLIPPED[$next])) || (($i + 1 < $len) && (isset($DELIMITERS_FLIPPED[$next . $query[$i + 1]]))))); //  (NB: symbol delimiters are a maximum of two in length)
                if ( /*When token ends, which is..*/
                    ($symbol_delimiter_coming || $doing_symbol_delimiter) /*Case of toggling from symbol to text or vice-versa and we find a delimiter is coming. When symbol delimiter arrives or we are doing a symbol deliminator */
                    &&
                    (!$this->is_start_of_delimiter($current_token . $next)) /*And the next character does not fit onto the end of our current token*/
                ) {
                    if (trim($current_token) != '') {
                        if (isset($DELIMITERS_FLIPPED[strtoupper($current_token)])) {
                            $tokens[] = strtoupper($current_token);
                        } else {
                            $tokens[] = $current_token;
                        }
                    }
                    $current_token = $next;
                    $doing_symbol_delimiter = (isset($SYMBOL_DELIMITER[$next]));
                } else {
                    $current_token .= $next;
                    if ($doing_symbol_delimiter) {
                        $doing_symbol_delimiter = isset($SYMBOL_DELIMITER[$next]);
                    }
                }
            }

            $i++;
        }

        $query = substr($query, 0, $len - 1);

        // PARSING/EXECUTION STAGE
        // -----------------------

        switch ($tokens[0]) {
            case 'ALTER':
                return $this->_do_query_alter($tokens, $query, $db, $fail_ok);

            case 'X_CREATE_TABLE':
                return $this->_do_query_x_create($tokens, $query, $db, $fail_ok);

            case 'CREATE':
                return $this->_do_query_create($tokens, $query, $db, $fail_ok);

            case 'INSERT':
                $random_key = mt_rand(0, mt_getrandmax());
                return $this->_do_query_insert($tokens, $query, $db, $fail_ok, $get_insert_id, $random_key, $save_as_volatile);

            case 'UPDATE':
                return $this->_do_query_update($tokens, $query, $db, $max, $start, $fail_ok);

            case 'DELETE':
                return $this->_do_query_delete($tokens, $query, $db, $max, $start, $fail_ok);
            case 'TRUNCATE':
                return $this->_do_query_truncate($tokens, $query, $db, $fail_ok);

            case '(':
            case 'SELECT':
                $at = 0;
                $results = $this->_do_query_select($tokens, $query, $db, $max, $start, $fail_ok, $at);
                return $results;

            case 'X_DROP_TABLE':
                return $this->_do_query_x_drop($tokens, $query, $db, $fail_ok);

            case 'DROP':
                return $this->_do_query_drop($tokens, $query, $db, $fail_ok);
        }

        return $this->_bad_query($query, $fail_ok, 'Unrecognised query type, ' . $tokens[0]);
    }

    /**
     * See if an item is a prefix to something in the delimiter array.
     *
     * @param  string $looking The item
     * @return boolean Whether it is
     */
    public function is_start_of_delimiter($looking)
    {
        global $DELIMITERS_FLIPPED, $DELIMITERS, $DELIMITERS_ALPHA;

        $len = strlen($looking);
        $looking = strtoupper($looking);
        if ($len == 1) {
            return isset($DELIMITERS_FLIPPED[$looking]);
        }
        if (isset($DELIMITERS_ALPHA[$looking[0]])) {
            foreach ($DELIMITERS_ALPHA[$looking[0]] as $d) {
                if (substr($d, 0, $len) == $looking) {
                    return true;
                }
            }
        }
        return false;
    }

    /**
     * Find the schema for a table.
     *
     * @param  array $db The database
     * @param  string $table_name The table name
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?array The schema map (null: not found)
     */
    protected function _read_schema($db, $table_name, $fail_ok = false)
    {
        global $SCHEMA_CACHE;
        if (array_key_exists($table_name, $SCHEMA_CACHE)) {
            return $SCHEMA_CACHE[$table_name];
        }

        $table_prefix = get_table_prefix();

        $schema_query = 'SELECT m_name,m_type FROM ' . $table_prefix . 'db_meta WHERE ' . db_string_equal_to('m_table', substr($table_name, strlen($table_prefix)));

        if ($table_name == $table_prefix . 'db_meta') {
            $fields = array(
                array(
                    'm_name' => 'm_table',
                    'm_type' => '*ID_TEXT',
                ),
                array(
                    'm_name' => 'm_name',
                    'm_type' => '*ID_TEXT',
                ),
                array(
                    'm_name' => 'm_type',
                    'm_type' => 'ID_TEXT',
                ),
            );
        } elseif ($table_name == $table_prefix . 'db_meta_indices') {
            $fields = array(
                array(
                    'm_name' => 'i_table',
                    'm_type' => '*ID_TEXT',
                ),
                array(
                    'm_name' => 'i_name',
                    'm_type' => '*ID_TEXT',
                ),
                array(
                    'm_name' => 'i_fields',
                    'm_type' => '*ID_TEXT',
                ),
            );
        } else {
            $fields = $this->db_query($schema_query, $db, null, null, $fail_ok);
            if (is_null($fields)) {
                return array(); // Can happen during installation
            }
        }

        $schema = array();
        foreach ($fields as $f) {
            $schema[$f['m_name']] = $f['m_type'];

            if (substr($f['m_type'], -9) == '__COMCODE') {
                if (!multi_lang_content()) {
                    $schema[$f['m_name'] . '__text_parsed'] = 'LONG_TEXT';
                    $schema[$f['m_name'] . '__source_user'] = 'MEMBER';
                }
            }
        }

        if (count($schema) == 0) {
            if (!$fail_ok) {
                fatal_exit('Internal error: missing schema for ' . $table_name);
            } else {
                return null;
            }
        }

        $SCHEMA_CACHE[$table_name] = $schema;

        return $schema;
    }

    /**
     * Type check some data destined to go into a table.
     *
     * @param  array $schema The schema
     * @param  array $record The data
     * @param  string $query Query that was executed
     */
    protected function _type_check($schema, $record, $query)
    {
        global $INT_TYPES, $STRING_TYPES;

        foreach ($record as $key => $val) {
            if (!isset($schema[$key])) {
                fatal_exit('Unrecognised key, ' . $key);
            }
            $schema_type = preg_replace('#[^\w]#', '', $schema[$key]);

            if (is_integer($val)) {
                if (!in_array($schema_type, $INT_TYPES)) {
                    $this->_bad_query($query, false, 'Database type strictness error: ' . $schema_type . ' wanted for ' . $key . ' field, but integer was given');
                }

                if (($val < 0) && ($schema_type == 'UINTEGER')) {
                    $this->_bad_query($query, false, 'Database type strictness error: ' . $schema_type . ' wanted for ' . $key . ' field (negative number given)');
                }

                if (($val != 0) && ($val != 1) && ($schema_type == 'BINARY')) {
                    $this->_bad_query($query, false, 'Database type strictness error: ' . $schema_type . ' wanted for ' . $key . ' field (number given was not 0 or 1)');
                }
            } elseif (is_string($val)) {
                if (!in_array($schema_type, array_keys($STRING_TYPES))) {
                    $this->_bad_query($query, false, 'Database type strictness error: ' . $schema_type . ' wanted for ' . $key . ' field, but string (' . $val . ') was given');
                }

                $max_length = $STRING_TYPES[$schema_type];
                if ((!is_null($max_length)) && (strlen($val) > $max_length)) {
                    $this->_bad_query($query, false, 'Database type strictness error: ' . $schema_type . ' wanted for ' . $key . ' field (text too long, maximum is ' . integer_format($max_length) . ')');
                }
            } elseif (is_float($val)) {
                if (!in_array($schema_type, array('REAL'))) {
                    $this->_bad_query($query, false, 'Database type strictness error: ' . $schema_type . ' wanted for ' . $key . ' field, but float was given');
                }
            } elseif (is_null($val)) {
                if (strpos($schema[$key], '?') === false) {
                    $this->_bad_query($query, false, 'Database type strictness error: ' . $schema_type . ' wanted for ' . $key . ' field, but NULL was given');
                }
            } else {
                $this->_bad_query($query, false, 'Database type strictness error: ' . $schema_type . ' wanted for ' . $key . ' field, but ' . gettype($val) . ' was given');
            }
        }
    }

    /**
     * Read in all the records from a table.
     *
     * @param  array $db Database connection
     * @param  string $table_name The table name
     * @param  string $table_as What the table will be renamed to (blank: N/A)
     * @param  ?array $schema Schema to type-set against (null: do not do type-setting)
     * @param  ?array $where_expr Expression filtering results (used for optimisation, seeing if we can get a quick key match) (null: no data to filter with)
     * @param  array $bindings Bindings available in the execution scope
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  string $query Query that was executed
     * @param  boolean $include_unused_fields Whether to include fields that are present in the actual records but not in our schema
     * @param  ?integer $max The maximum number of rows to read (null: no limit / there's a sort clause meaning we need to read all)
     * @return ?array The collected records (null: error)
     */
    protected function _read_all_records($db, $table_name, $table_as, $schema, $where_expr, $bindings, $fail_ok, $query, $include_unused_fields = false, $max = null)
    {
        $records = array();
        $key_fragments = ''; // We can do a filename substring search to stop us having to parse ALL
        $must_contain = array();
        if ((!is_null($schema)) && (!is_null($where_expr))) { // Try for an efficient filename-based lookup
            $keys = array();
            foreach ($schema as $key => $type) {
                if (strpos($type, '*') !== false) {
                    $keys[] = $key;
                }
            }

            foreach ($keys as $i => $key) {
                if (strpos($key, '.') !== false) {
                    if (substr($key, 0, strpos($key, '.')) == $table_as) {
                        $key = substr($key, strpos($key, '.') + 1);
                    } else {
                        continue;
                    }
                }
                $keys[$i] = $key;
            }
            $keys = array_unique($keys);

            $key_lookup = true;
            $where_expr_compressed = $this->_turn_where_expr_to_map($where_expr, $table_as, $schema);
            $where_expr_compressed_b = $this->_turn_where_expr_to_map($where_expr, $table_as, $schema, true);
            $must_contain = array();

            foreach ($where_expr_compressed_b as $key => $val) {
                $new_val = mixed();
                if (is_string($val)) {
                    $new_val = $val;
                } elseif (is_integer($val)) {
                    $new_val = strval($val);
                } elseif (is_float($val)) {
                    $new_val = float_to_raw_string($val);
                } elseif (is_array($val)) {
                    $new_val = array();
                    foreach ($val as $_val) {
                        if (is_string($_val)) {
                            $new_val[] = $_val;
                        } elseif (is_integer($_val)) {
                            $new_val[] = strval($_val);
                        } elseif (is_float($_val)) {
                            $new_val[] = float_to_raw_string($_val);
                        }
                    }
                } else {
                    $new_val = '';
                }

                $must_contain[$key] = $new_val;
            }
            sort($keys);
            foreach ($keys as $key) {
                if ($key_fragments != '') {
                    $key_fragments .= ',';
                }
                if ($key != 'id') {
                    $key_fragments .= preg_quote($key . '=', '#');
                }

                if (!in_array($key, array_keys($where_expr_compressed))) {
                    $key_lookup = false;
                    $key_fragments .= '.*';
                } else {
                    $val = $where_expr_compressed[$key];
                    $new_val = '';
                    if (is_array($val)) {
                        if (count($val) == 1) {
                            $val = $val[0];
                            $where_expr_compressed[$key] = $val;
                        } else {
                            $key_lookup = false;
                            $key_fragments .= '(';
                            foreach ($val as $i => $possible) {
                                if ($i != 0) {
                                    $key_fragments .= '|';
                                }

                                if (is_string($possible)) {
                                    $new_val = $possible;
                                } elseif (is_integer($possible)) {
                                    $new_val = strval($possible);
                                } elseif (is_float($possible)) {
                                    $new_val = float_to_raw_string($possible);
                                }
                                $key_fragments .= preg_quote($this->_escape_name($new_val), '#');
                            }
                            $key_fragments .= ')';
                            continue;
                        }
                    }
                    if (is_string($val)) {
                        $new_val = $val;
                    } elseif (is_integer($val)) {
                        $new_val = strval($val);
                    } elseif (is_float($val)) {
                        $new_val = float_to_raw_string($val);
                    }

                    $key_fragments .= preg_quote($this->_escape_name($new_val), '#');
                }
            }

            $key_buildup = $this->_guid($schema, $where_expr_compressed);

            if (($key_lookup) && ($key_buildup != '')) {
                $file_exists_xml = file_exists($db[0] . '/' . $table_name . '/' . $key_buildup . '.xml');
                $file_exists_xml_volatile = file_exists($db[0] . '/' . $table_name . '/' . $key_buildup . '.xml-volatile');
                if (($file_exists_xml) || ($file_exists_xml_volatile)) {
                    $the_key = preg_replace('#\.[\w\-]+$#', '', $key_buildup);
                    $suffix = $file_exists_xml ? '.xml' : '.xml-volatile';
                    $test = $this->_read_record($db[0] . '/' . $table_name . '/' . $key_buildup . $suffix, $schema, null, $include_unused_fields, $fail_ok);
                    if ($test === null) {
                        return array();
                    }
                    $records[$the_key] = $test;
                    if ($table_name == get_table_prefix() . 'translate') {
                        $sup_file = $db[0] . '/' . $table_name . '/sup/' . $key_buildup . '.xml-volatile';
                        if (file_exists($sup_file)) {
                            $sup_record = $this->_read_record($sup_file, $schema, null, $include_unused_fields, $fail_ok);
                            $records[$the_key]['text_parsed'] = $sup_record['text_parsed'];
                        }
                    }
                    return $records;
                } else {
                    return array();
                }
            }
        }

        global $DIR_CONTENTS_CACHE;
        if (true/*We can't do this, the directory contents may change in another HTTP request*/ || !isset($DIR_CONTENTS_CACHE[$table_name])) {
            if (!file_exists($db[0] . '/' . $table_name)) {
                mkdir($db[0] . '/' . $table_name, 0777);
                require_code('files');
                fix_permissions($db[0] . '/' . $table_name);
                sync_file($db[0] . '/' . $table_name);
            }
            @chdir($db[0] . '/' . $table_name);
            $dh = @glob('{,.}*.{xml,xml-volatile}', GLOB_NOSORT | GLOB_BRACE);
            if ($dh === false) {
                $dh = array();
            }
            @chdir(get_file_base());
            if (file_exists($db[0] . '/' . $table_name . '/.xml')) {
                $dh[] = '.xml';
            } elseif (file_exists($db[0] . '/' . $table_name . '/.xml-volatile')) {
                $dh[] = '.xml-volatile';
            }
            $DIR_CONTENTS_CACHE[$table_name] = $dh;
        } else {
            $dh = $DIR_CONTENTS_CACHE[$table_name];
        }
        if (($dh === false) && ($fail_ok)) {
            return null;
        }
        if ($dh === false) {
            critical_error('PASSON', 'Failure to read table ' . $table_name);
        }
        $regexp = '#^' . $key_fragments . '(' . preg_quote('.xml') . '|' . preg_quote('.xml-volatile') . ')$#';

        foreach ($dh as $file) {
            if ($key_fragments != '') {
                if (preg_match($regexp, $file) == 0) {
                    continue;
                }
            }
            $full_path = $db[0] . '/' . $table_name . '/' . $file;
            if ((strlen($full_path) >= 255) && (stripos(PHP_OS, 'WIN') === 0)) {
                continue; // :(
            }
            $read = $this->_read_record($full_path, $schema, $must_contain, $include_unused_fields, $fail_ok);
            if (!is_null($read)) {
                $the_key = preg_replace('#\.[\w\-]+$#', '', $file);
                $records[$the_key] = $read;

                if ($table_name == get_table_prefix() . 'translate') {
                    $sup_file = $db[0] . '/' . $table_name . '/sup/' . preg_replace('#\.\w+$#', '', $file) . '.xml-volatile';
                    if (file_exists($sup_file)) {
                        $test = $this->_read_record($sup_file, $schema, null, $include_unused_fields, $fail_ok);
                        if ($test !== null) {
                            $records[$the_key] += $test;
                        }
                    }
                }

                if (($max !== null) && (count($records) >= $max) && (($where_expr === null) || ($where_expr == array('LITERAL', true)))) {
                    break;
                }
            }
        }

        return $records;
    }

    /**
     * Take an expression and do our best to collapse it into a fixed mapping of stuff we know we are going to AND.
     *
     * @param  array $where_expr The expression parse tree
     * @param  string $table_as What the table will be renamed to (blank: N/A)
     * @param  ?array $schema Schema to type-set against (null: do not do type-setting)
     * @param  boolean $not_full_accuracy Whether to do a not-full-accurate search
     * @return array AND map
     */
    protected function _turn_where_expr_to_map($where_expr, $table_as, $schema = null, $not_full_accuracy = false)
    {
        if ($where_expr[0] == 'BRACKETED') {
            return $this->_turn_where_expr_to_map($where_expr[1], $table_as, $schema, $not_full_accuracy);
        }
        if ($where_expr[0] == 'AND') {
            return array_merge($this->_turn_where_expr_to_map($where_expr[1], $table_as, $schema, $not_full_accuracy), $this->_turn_where_expr_to_map($where_expr[2], $table_as, $schema, $not_full_accuracy));
        }
        if ($where_expr[0] == 'OR') {
            $alpha = $this->_turn_where_expr_to_map($where_expr[1], $table_as, $schema, $not_full_accuracy);
            $beta = $this->_turn_where_expr_to_map($where_expr[2], $table_as, $schema, $not_full_accuracy);
            $_alpha = array_keys($alpha);
            $_beta = array_keys($beta);
            if ((count($alpha) == 1) && (count($beta) == 1) && ($_alpha == $_beta)) {
                $alpha[$_alpha[0]] = array_merge(is_array($alpha[$_alpha[0]]) ? $alpha[$_alpha[0]] : array($alpha[$_alpha[0]]), is_array($beta[$_beta[0]]) ? $beta[$_beta[0]] : array($beta[$_beta[0]]));
                return $alpha;
            }
        }
        if (($where_expr[0] == '=') && ($where_expr[1][0] == 'LITERAL') && ($where_expr[2][0] == 'FIELD')) {
            $where_expr = array($where_expr[0], $where_expr[2], $where_expr[1]);
        }
        if (($where_expr[0] == 'LIKE') && ($where_expr[1][0] == 'FIELD') && ($where_expr[2][0] == 'LITERAL') && ($not_full_accuracy)) {
            $key = $where_expr[1][1];
            if ($table_as != '') {
                $key = preg_replace('#^' . preg_quote($table_as, '#') . '\.#', '', $key);
                if (strpos($key, '.') !== false) {
                    return array(); // Not for our table
                }
            }

            if ((!is_null($schema)) && (!array_key_exists($key, $schema))) {
                return array(); // Not in our table (join involved. must be in other join)
            }

            if (substr($where_expr[2][1], 0, 1) == '%') {
                $where_expr[2][1] = substr($where_expr[2][1], 1);
            }
            if (substr($where_expr[2][1], -1) == '%') {
                $where_expr[2][1] = substr($where_expr[2][1], 0, strlen($where_expr[2][1]) - 1);
            }
            if ((strpos($where_expr[2][1], '%') !== false) || (strpos($where_expr[2][1], '?') !== false)) {
                return array();
            }

            return array($key => $where_expr[2][1]);
        }
        if (($where_expr[0] == '=') && ($where_expr[1][0] == 'FIELD') && ($where_expr[2][0] == 'LITERAL')) {
            $key = $where_expr[1][1];
            if ($table_as != '') {
                $key = preg_replace('#^' . preg_quote($table_as, '#') . '\.#', '', $key);
                if (strpos($key, '.') !== false) {
                    return array(); // Not for our table
                }
            }
            if ((!is_null($schema)) && (!array_key_exists($key, $schema))) {
                return array(); // Not in our table (join involved. must be in other join)
            }
            return array($key => $where_expr[2][1]);
        }
        return array();
    }

    /**
     * Read a record from an XML file.
     *
     * @param  PATH $path The file path
     * @param  ?array $schema Schema to type-set against (null: do not do type-setting)
     * @param  ?array $must_contain_strings Substrings to check it is in, used for performance (null: none)
     * @param  boolean $include_unused_fields Whether to include fields that are present in the actual records but not in our schema
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?array The record map (null: does not contain requested substrings / error)
     */
    protected function _read_record($path, $schema = null, $must_contain_strings = null, $include_unused_fields = false, $fail_ok = false)
    {
        if ($fail_ok && !is_file($path)) {
            return null;
        }

        $file_contents = file_get_contents($path);

        if (!is_null($must_contain_strings)) {
            foreach ($must_contain_strings as $match) {
                if (is_array($match)) {
                    $found = 0;
                    $possible_matches = $match;
                    foreach ($possible_matches as $match2) {
                        if ($match2 == '') {
                            continue;
                        }

                        if (strpos($file_contents, xmlentities($match2)) !== false) {
                            $found++;
                            break;
                        }
                    }
                    if ($found == 0) {
                        return null;
                    }
                } else {
                    if ($match == '') {
                        continue;
                    }

                    if (strpos($file_contents, xmlentities($match)) === false) {
                        return null;
                    }
                }
            }
        }

        /* Too slow
        $ob = new xml_file_parse($file_contents);
        if (!is_null($ob->error)) {
            fatal_exit($ob->error);
        }
        $_record = $ob->output;
        */
        // This is much faster, even though it's a bit of a hack as it assumes all records are as Composr would write them
        $bits = preg_split('#</?([^>]*)>#', $file_contents, -1, PREG_SPLIT_DELIM_CAPTURE);
        $_record = array();
        $bc = count($bits) - 2;
        $i = 0;
        if (trim($bits[$i]) == '') {
            $i++; // Whitespace between tags
        }
        if ((!isset($bits[$i])) || ($bits[$i] != 'composr')) {
            warn_exit('Unrecognised XML in ' . $path);
        }
        $i++; // Skip past "Composr"
        if (trim($bits[$i]) == '') {
            $i++; // Whitespace between tags
        }
        while ($i < $bc) {
            $field = $bits[$i];
            $i++;
            $data = $bits[$i];
            $i++;
            $i++; // Skip past closing tag
            if (trim($bits[$i]) == '') {
                $i++; // Whitespace between tags
            }
            $_record[$field] = html_entity_decode($data, ENT_QUOTES, get_charset());
        }

        // Even if we did serialize with type information (we don't) we would still need to do type checking, because when we do add_table_field/alter_table_field we can't assume it will alter all non-committed records on other people's systems
        if (is_null($schema)) {
            return $_record;
        } else {
            global $INT_TYPES;
            $record = array();
            foreach ($_record as $key => $val) {
                $new_val = mixed();
                if ((!array_key_exists($key, $schema)) && (!$include_unused_fields)) {
                    continue; // Been deleted
                }
                $type = $schema[$key];
                $schema_type = preg_replace('#[^\w]#', '', $type);

                if (in_array($schema_type, $INT_TYPES)) {
                    if (((is_null($val)) || ($val === '')) && (substr($type, 0, 1) == '?')) {
                        $new_val = null;
                    } else {
                        $new_val = @intval($val);
                    }
                } elseif (in_array($schema_type, array('REAL'))) {
                    if (((is_null($val)) || ($val === '')) && (substr($type, 0, 1) == '?')) {
                        $new_val = null;
                    } else {
                        $new_val = @floatval($val);
                    }
                } else {
                    $new_val = $val;
                }

                $record[$key] = $new_val;

                unset($schema[$key]);
            }

            global $INT_TYPES;
            foreach ($schema as $key => $type) {
                $schema_type = preg_replace('#[^\w]#', '', $type);

                if (in_array($schema_type, $INT_TYPES)) {
                    if (substr($type, 0, 1) == '?') {
                        $record[$key] = null;
                    } else {
                        $record[$key] = 0;
                    }
                } elseif (in_array($schema_type, array('REAL'))) {
                    if (substr($type, 0, 1) == '?') {
                        $record[$key] = 0.0;
                    } else {
                        $record[$key] = 0.0;
                    }
                } else {
                    $record[$key] = '';
                }
            }
        }

        return $record;
    }

    /**
     * Write in all the records to a table.
     *
     * @param  array $db Database connection
     * @param  string $table_name The table name
     * @param  array $records The list of record maps
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     */
    protected function _write_records($db, $table_name, $records, $fail_ok = false)
    {
        foreach ($records as $guid => $record) {
            if (!is_string($guid)) {
                $guid = strval($guid); // As PHP can use type for array keys
            }
            $this->_write_record($db, $table_name, $guid, $record, $fail_ok);
        }
    }

    /**
     * Write a record to an XML file.
     *
     * @param  array $db Database connection
     * @param  string $table_name The table name
     * @param  string $guid The GUID
     * @param  array $record The record map
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  boolean $save_as_volatile Whether we are saving as a 'volatile' file extension
     */
    protected function _write_record($db, $table_name, $guid, $record, $fail_ok = false, $save_as_volatile = false)
    {
        $suffix = $save_as_volatile ? '.xml-volatile' : '.xml';

        if (!file_exists($db[0] . '/' . $table_name)) {
            mkdir($db[0] . '/' . $table_name, 0777);
            require_code('files');
            fix_permissions($db[0] . '/' . $table_name);
            sync_file($db[0] . '/' . $table_name);
        }

        $path = $db[0] . '/' . $table_name . '/' . $guid . $suffix;

        if ($table_name == get_table_prefix() . 'translate') { // Special code to store volatile text_parsed attribute externally
            $record_copy = $record;
            unset($record_copy['text_original']);
            unset($record_copy['source_user']);
            unset($record_copy['broken']);
            unset($record_copy['importance_level']);
            $record['text_parsed'] = '';
            $this->_write_record($db, $table_name . '/sup', $guid, $record_copy, $fail_ok, true);
        }

        if ((strlen($path) > 255) && (stripos(PHP_OS, 'WIN') === 0)) {
            attach_message('File path too long on Windows (' . $path . ')', 'warn');
            return;
        }

        require_code('files');
        $contents = '';
        $contents .= "<composr>\n";
        $val = mixed();
        foreach ($record as $key => $val) {
            if (is_integer($val)) {
                $val = strval($val);
            }
            elseif (is_float($val)) {
                $val = float_to_raw_string($val);
            }
            elseif (is_null($val)) {
                $val = '';
            }
            $contents .= "\t<" . $key . ">" . xmlentities($val) . "</" . $key . ">\n";
        }
        $contents .= "</composr>\n";
        cms_file_put_contents_safe($path, $contents, FILE_WRITE_FIX_PERMISSIONS | FILE_WRITE_SYNC_FILE);

        $schema = $this->_read_schema($db, preg_replace('#/sup$#', '', $table_name), $fail_ok);
        if (!is_null($schema)) {
            $new_guid = $this->_guid($schema, $record);
            $new_path = $db[0] . '/' . $table_name . '/' . $new_guid . $suffix;
            if ($path != $new_path) {
                rename($path, $new_path);
                sync_file_move($path, $new_path);
            }
        }
    }

    /**
     * Write a record to an XML file.
     *
     * @param  PATH $path The file path
     * @param  array $db Database connection
     */
    protected function _delete_record($path, $db)
    {
        if (file_exists($path)) {
            @unlink($path);
            sync_file($path);
        }
    }

    /**
     * Check to see if there is a key conflict problem.
     *
     * @param  array $db Database connection
     * @param  string $table_name The table name
     * @param  array $schema The schema
     * @param  array $record The record
     * @param  string $query Query that was executed
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  ?string $existing_identity The GUID representing what we have now (so we don't think we're conflicting with ourself) (null: not yet added)
     * @return boolean Whether there was a conflict
     */
    protected function _key_conflict_check($db, $table_name, $schema, $record, $query, $fail_ok, $existing_identity = null)
    {
        $where = '';
        foreach ($schema as $key => $type) {
            if (strpos($type, '*') === false) {
                continue;
            }

            if ($where != '') {
                $where .= ' AND ';
            }

            $value = $record[$key];
            if (is_null($value)) {
                $where .= $key . 'IS NULL';
            } else {
                if (is_float($value)) {
                    $where .= $key . '=' . float_to_raw_string($value);
                } elseif (is_integer($value)) {
                    $where .= $key . '=' . strval($value);
                } else {
                    $where .= $key . '=\'' . db_escape_string($value) . '\'';
                }
            }
        }

        $test_results = $this->db_query('SELECT * FROM ' . $table_name . ' WHERE ' . $where, $db, 2, null, $fail_ok, false, true);
        if (count($test_results) == 0) {
            return false;
        }
        if (count($test_results) > 1) {
            return true;
        }
        if ((count($test_results) == 1) && (is_null($existing_identity))) {
            return true;
        }
        $is_different = ($this->_guid($schema, $test_results[0]) != $existing_identity);
        return $is_different;
    }

    /**
     * Execute a DROP query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?mixed The results (null: no results)
     */
    protected function _do_query_x_drop($tokens, $query, $db, $fail_ok)
    {
        $at = 0;
        if (!$this->_parsing_expects($at, $tokens, 'X_DROP_TABLE', $query)) {
            return null;
        }

        $table_name = $this->_parsing_read($at, $tokens, $query);

        $file_path = $db[0] . '/' . $table_name;
        $dh = @opendir($file_path);
        if ($dh !== false) {
            while (($file = readdir($dh)) !== false) {
                if ((substr($file, -4) == '.xml') || (substr($file, -13) == '.xml-volatile')) {
                    unlink($file_path . '/' . $file);
                    sync_file($file_path . '/' . $file);
                }
            }
            closedir($dh);
            @rmdir($file_path);
            sync_file($file_path);
        }

        global $SCHEMA_CACHE;
        unset($SCHEMA_CACHE[$table_name]);

        return null;
    }

    /**
     * Execute a DROP query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?mixed The results (null: no results)
     */
    protected function _do_query_drop($tokens, $query, $db, $fail_ok)
    {
        $at = 0;
        if (!$this->_parsing_expects($at, $tokens, 'DROP', $query)) {
            return null;
        }
        $type = $this->_parsing_read($at, $tokens, $query);
        if ($type == 'INDEX') {
            $index_name = $this->_parsing_read($at, $tokens, $query);
            if (!$this->_parsing_expects($at, $tokens, 'ON', $query)) {
                return null;
            }
            $table_name = $this->_parsing_read($at, $tokens, $query);
            // We don't actually do indexes, so do nothing
        } elseif ($type == 'TABLE') {
            if ($tokens[$at] == 'IF') {
                $this->_parsing_read($at, $tokens, $query);
                if (!$this->_parsing_expects($at, $tokens, 'EXISTS', $query)) {
                    return null;
                }
            }
            $table_name = $this->_parsing_read($at, $tokens, $query);

            $queries = $this->db_drop_table_if_exists($table_name, $db);
            foreach ($queries as $sql) {
                $this->db_query($sql, $db, null, null, true); // Might already exist so suppress errors
            }
        } else {
            return $this->_bad_query($query, $fail_ok, 'Unrecognised DROP type, ' . $type);
        }

        if (!$this->_parsing_check_ended($at, $tokens, $query)) {
            return null;
        }

        return null;
    }

    /**
     * Execute an ALTER query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?mixed The results (null: no results)
     */
    protected function _do_query_alter($tokens, $query, $db, $fail_ok)
    {
        global $SCHEMA_CACHE;

        $at = 0;
        if (!$this->_parsing_expects($at, $tokens, 'ALTER', $query)) {
            return null;
        }
        if (!$this->_parsing_expects($at, $tokens, 'TABLE', $query)) {
            return null;
        }
        $table_name = $this->_parsing_read($at, $tokens, $query);
        $op = $this->_parsing_read($at, $tokens, $query);
        switch ($op) {
            case 'RENAME':
                $new_table_name = $this->_parsing_read($at, $tokens, $query);
                rename($db[0] . '/' . $table_name, $db[0] . '/' . $new_table_name);
                sync_file_move($db[0] . '/' . $table_name, $db[0] . '/' . $new_table_name);
                break;
            case 'CHANGE':
            case 'ADD':
                // Parse
                $column_name = $this->_parsing_read($at, $tokens, $query);
                if ($op == 'CHANGE') {
                    $new_column_name = $this->_parsing_read($at, $tokens, $query);
                } else {
                    $new_column_name = $column_name;
                }
                $data_type = $this->_parsing_read($at, $tokens, $query);
                $next = $this->_parsing_read($at, $tokens, $query, true);
                if ($next == 'DEFAULT') {
                    $_default = $this->_parsing_read_expression($at, $tokens, $query, $db, false, false, $fail_ok);
                    $default = $this->_execute_expression($_default, array(), $query, $db, $fail_ok);
                } else {
                    $default = false;

                    if (!is_null($next)) {
                        $at--;
                    }
                }
                $next = $this->_parsing_read($at, $tokens, $query, true);
                $allow_null = null; // No change
                if (!is_null($next)) {
                    if ($next == 'NOT') {
                        if (!$this->_parsing_expects($at, $tokens, 'NULL', $query)) {
                            return null;
                        }
                    } elseif ($next == 'NULL') {
                        $allow_null = true;
                    } else {
                        $at--;
                    }
                }

                if ($default === false) {
                    if ($allow_null) {
                        $default = null;
                    } elseif ($op == 'ADD') {
                        return $this->_bad_query($query, false, 'No DEFAULT given and NULL not allowed');
                    }
                }

                // Execute
                if ($op == 'ADD') {
                    $records = $this->_read_all_records($db, $table_name, '', null, null, array(), $fail_ok, $query);
                    if (is_null($records)) {
                        return null;
                    }
                    foreach (array_keys($records) as $guid) {
                        $records[$guid][$column_name] = $default;
                    }
                    $this->_write_records($db, $table_name, $records, $fail_ok);

                    $this->_read_schema($db, $table_name); // Workaround with caching. It might be the directory contents cache for the db_meta table is not updated at exactly the right times, as the execution order can be off. If the directory contents cache is not updated and the schema is read, it may miss the new field. Therefore we need to force it to read now, then extend it.
                    if (array_key_exists($table_name, $SCHEMA_CACHE)) {
                        $SCHEMA_CACHE[$table_name][$column_name] = $data_type;
                    }
                } elseif ($op == 'CHANGE') {
                    // Actually we type-convert in real-time so no change actually needed. Composr would have updated the meta stuff separately
                    if (array_key_exists($table_name, $SCHEMA_CACHE)) {
                        unset($SCHEMA_CACHE[$table_name][$column_name]);
                        $SCHEMA_CACHE[$table_name][$new_column_name] = $data_type;
                    }

                    if ($new_column_name != $column_name) {
                        $records = $this->_read_all_records($db, $table_name, '', null, null, array(), $fail_ok, $query, true);
                        if (is_null($records)) {
                            return null;
                        }
                        foreach (array_keys($records) as $guid) {
                            $records[$guid][$new_column_name] = $records[$guid][$column_name];
                            unset($records[$guid][$column_name]);
                        }
                        $this->_write_records($db, $table_name, $records, $fail_ok);
                    }
                }

                if (!$this->_parsing_check_ended($at, $tokens, $query)) {
                    return null;
                }

                return null;

            case 'DROP':
                // Parse
                if (!$this->_parsing_expects($at, $tokens, 'COLUMN', $query)) {
                    return null;
                }
                $column_name = $this->_parsing_read($at, $tokens, $query);

                // Execute
                $records = $this->_read_all_records($db, $table_name, '', null, null, array(), $fail_ok, $query);
                if (is_null($records)) {
                    return null;
                }
                foreach (array_keys($records) as $guid) {
                    unset($records[$guid][$column_name]);
                }
                $this->_write_records($db, $table_name, $records, $fail_ok);

                unset($SCHEMA_CACHE[$table_name][$column_name]);

                if (!$this->_parsing_check_ended($at, $tokens, $query)) {
                    return null;
                }

                return null;
        }

        return $this->_bad_query($query, false, 'Expected ALTER TABLE ADD or ALTER TABLE DROP or ALTER TABLE CHANGE or ALTER TABLE RENAME');
    }

    /**
     * Execute a CREATE query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?mixed The results (null: no results)
     */
    protected function _do_query_x_create($tokens, $query, $db, $fail_ok)
    {
        $at = 0;
        if (!$this->_parsing_expects($at, $tokens, 'X_CREATE_TABLE', $query)) {
            return null;
        }

        if (!$this->_parsing_expects($at, $tokens, "'", $query)) {
            return null;
        }

        $parameters = $this->_parsing_read($at, $tokens, $query);

        if (!$this->_parsing_expects($at, $tokens, "'", $query)) {
            return null;
        }

        list($table_name, $fields, $raw_table_name, $save_bytes) = unserialize($parameters);

        $path = $db[0] . '/' . $table_name;

        if (file_exists($path)) {
            return;
        }

        $found_key = false;
        foreach ($fields as $type) {
            if (strpos($type, '*') !== false) {
                $found_key = true;
            }
        }
        if (!$found_key) {
            fatal_exit('No key specified for table ' . $table_name);
        }

        @mkdir($path, 0777);
        require_code('files');
        fix_permissions($path);
        sync_file($path);

        return null;
    }

    /**
     * Execute a CREATE query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?mixed The results (null: no results)
     */
    protected function _do_query_create($tokens, $query, $db, $fail_ok)
    {
        $at = 0;
        if (!$this->_parsing_expects($at, $tokens, 'CREATE', $query)) {
            return null;
        }
        if (!$this->_parsing_expects($at, $tokens, 'TABLE', $query)) {
            return null;
        }
        $table_name = $this->_parsing_read($at, $tokens, $query);
        $if_not_exists = false;
        if ($table_name == 'IF') {
            if (!$this->_parsing_expects($at, $tokens, 'NOT', $query)) {
                return null;
            }
            if (!$this->_parsing_expects($at, $tokens, 'EXISTS', $query)) {
                return null;
            }
            $table_name = $this->_parsing_read($at, $tokens, $query);
            $if_not_exists = true;
        }

        if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
            return null;
        }
        $fields = array();
        do {
            $column_name = $this->_parsing_read($at, $tokens, $query);
            if ($column_name == 'PRIMARY') {
                if (!$this->_parsing_expects($at, $tokens, 'KEY', $query)) {
                    return null;
                }
                if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                    return null;
                }
                do {
                    $token = $this->_parsing_read($at, $tokens, $query);
                    $fields[$token] = '*' . $fields[$token];
                    $token = $this->_parsing_read($at, $tokens, $query);
                } while ($token == ',');
                $at--;
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
            } else {
                $null = true;
                $type = $this->_parsing_read($at, $tokens, $query);

                $token = $this->_parsing_read($at, $tokens, $query);
                if ($token == 'NOT') {
                    if (!$this->_parsing_expects($at, $tokens, 'NULL', $query)) {
                        return null;
                    }
                    $null = false;
                } elseif ($token == 'PRIMARY') {
                    if (!$this->_parsing_expects($at, $tokens, 'KEY', $query)) {
                        return null;
                    }
                    $type = '*' . $type;
                } else {
                    $at--;
                }

                $fields[$column_name] = ($null ? '?' : '') . $type;
            }

            $next = $this->_parsing_read($at, $tokens, $query);
        } while ($next == ',');
        $at--;
        if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
            return null;
        }

        $queries = $this->db_create_table($table_name, $fields, $db);
        foreach ($queries as $sql) {
            $this->db_query($sql, $db);
        }

        if (!$this->_parsing_check_ended($at, $tokens, $query)) {
            return null;
        }
    }

    /**
     * Wrapper to execute an INSERT query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  boolean $get_insert_id Whether to get the autoincrement ID created for an insert query
     * @param  ?integer $random_key The random key that we would use (null: not generated yet)
     * @param  boolean $save_as_volatile Whether we are saving as a 'volatile' file extension
     * @return ?mixed The insert ID (null: not requested / error)
     */
    protected function _do_query_insert($tokens, $query, $db, $fail_ok, $get_insert_id, &$random_key, $save_as_volatile = false)
    {
        $_inserts = $this->_do_query_insert__parse($tokens, $query, $db, $fail_ok);
        if (is_null($_inserts)) {
            return null;
        }
        list($table_name, $inserts) = $_inserts;
        return $this->_do_query_insert__execute($inserts, $table_name, $query, $db, $fail_ok, $get_insert_id, $random_key, $save_as_volatile);
    }

    /**
     * Parse an INSERT query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?array A pair: the table, and the rows to insert (null: error)
     */
    protected function _do_query_insert__parse($tokens, $query, $db, $fail_ok)
    {
        // Parse
        $at = 0;
        if (!$this->_parsing_expects($at, $tokens, 'INSERT', $query)) {
            return null;
        }
        if (!$this->_parsing_expects($at, $tokens, 'INTO', $query)) {
            return null;
        }
        $table_name = $this->_parsing_read($at, $tokens, $query);
        if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
            return null;
        }
        $record_basic = array();
        $reverse_index = array();
        do {
            $token = $this->_parsing_read($at, $tokens, $query);
            $record_basic[$token] = null;
            $reverse_index[] = $token;
            $token = $this->_parsing_read($at, $tokens, $query);
        } while ($token == ',');
        $at--;
        if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
            return null;
        }
        if (!$this->_parsing_expects($at, $tokens, 'VALUES', $query)) {
            return null;
        }
        $inserts = array();
        do {
            if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                return null;
            }
            $record = $record_basic;
            $i = 0;
            do {
                $expr = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
                $result = $this->_execute_expression($expr, array(), $query, $db, $fail_ok);
                $record[$reverse_index[$i]] = $result;
                $i++;
                $token = $this->_parsing_read($at, $tokens, $query);
            } while ($token == ',');
            $at--;
            if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                return null;
            }

            // Store in our list
            $inserts[] = $record;

            // Continue
            $token = $this->_parsing_read($at, $tokens, $query, true);
        } while ($token === ',');
        if (!is_null($token)) {
            $at--;
        }

        if (!$this->_parsing_check_ended($at, $tokens, $query)) {
            return null;
        }

        return array($table_name, $inserts);
    }

    /**
     * Execute an INSERT query.
     *
     * @param  array $inserts Rows being inserted
     * @param  ID_TEXT $table_name Table name we're inserting into
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  boolean $get_insert_id Whether to get the autoincrement ID created for an insert query
     * @param  ?integer $random_key The random key that we would use (null: not generated yet)
     * @param  boolean $save_as_volatile Whether we are saving as a 'volatile' file extension
     * @return ?mixed The insert ID (null: not requested / error)
     */
    protected function _do_query_insert__execute($inserts, $table_name, $query, $db, $fail_ok, $get_insert_id, &$random_key, $save_as_volatile = false)
    {
        global $TABLE_BASES;

        // Execute
        foreach ($inserts as $record_num => $record) {
            $insert_id = null;
            $schema = $this->_read_schema($db, $table_name, $fail_ok);
            if (is_null($schema)) {
                return null;
            }
            $no_key_conflict_check = false;
            foreach ($schema as $key => $val) {
                if (!array_key_exists($key, $record)) { // Possibly an auto-generated key
                    if (substr($key, -10) == '__text_parsed') {
                        $record[$key] = '';
                    } elseif (substr($key, -13) == '__source_user') {
                        $record[$key] = db_get_first_id();
                    } elseif (preg_replace('#[^\w]#', '', $val) == 'AUTO') {
                        $record[$key] = isset($TABLE_BASES[$table_name]) ? $TABLE_BASES[$table_name] : $this->db_get_first_id(); // We always want first record as '1', because we often reference it in a hard-coded way
                        while ((file_exists($db[0] . '/' . $table_name . '/' . strval($record[$key]) . '.xml')) || (file_exists($db[0] . '/' . $table_name . '/' . $this->_guid($schema, $record) . '.xml')) || (file_exists($db[0] . '/' . $table_name . '/' . strval($record[$key]) . '.xml-volatile')) || (file_exists($db[0] . '/' . $table_name . '/' . $this->_guid($schema, $record) . '.xml-volatile'))) {
                            if ($GLOBALS['IN_MINIKERNEL_VERSION']) { // In particular the f_groups/f_forum_groupings/calendar_types usage of tables references ID numbers for things. But let's just make all installer stuff linear
                                $record[$key]++;
                                $TABLE_BASES[$table_name] = $record[$key] + 1;
                            } else {
                                if ($record_num != 0) {
                                    $random_key = mt_rand(0, mt_getrandmax());
                                }
                                $record[$key] = $random_key; // We don't use auto-increment, we use randomisation. As otherwise when people sync over revision control there'd be conflicts
                            }
                        }
                        $insert_id = $record[$key];
                        if ($val == '*AUTO') {
                            $no_key_conflict_check = true;
                        }
                    } else {
                        return $this->_bad_query($query, false, 'No default value provided for ' . $key);
                    }
                }
            }
            $guid = $this->_guid($schema, $record);
            $this->_type_check($schema, $record, $query);
            if (!$no_key_conflict_check) {
                if ($this->_key_conflict_check($db, $table_name, $schema, $record, $query, $fail_ok)) {
                    return $this->_bad_query($query, $fail_ok, 'A record already exists with this key');
                }
            }
            $this->_write_record($db, $table_name, $guid, $record, $fail_ok, $save_as_volatile);
        }

        return $get_insert_id ? $insert_id : null;
    }

    /**
     * Parse an SQL expression.
     *
     * @param  integer $at Our offset counter
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $look_for_connectives Whether to work as a connection point to seek out logic connection expression parts
     * @param  boolean $look_for_any_connectives Whether to work as a connection point to seek out arithmetic connection expression parts
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?array The expression (null: error)
     */
    protected function _parsing_read_expression(&$at, $tokens, $query, $db, $look_for_connectives = true, $look_for_any_connectives = true, $fail_ok = false)
    {
        $token = $this->_parsing_read($at, $tokens, $query);
        $expr = array();
        $doing_not = false;
        switch ($token) {
            // Aggregate expressions...

            case 'DISTINCT':
                $expr = array('DISTINCT', array());
                $d = $this->_parsing_read($at, $tokens, $query);
                if ($d == '(') {
                    $d = $this->_parsing_read($at, $tokens, $query);
                    if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                        return null;
                    }
                    $expr[1][] = $d;
                } else {
                    $at--;
                    do {
                        $d = $this->_parsing_read($at, $tokens, $query);
                        $expr[1][] = $d;
                        $_token = $this->_parsing_read($at, $tokens, $query);
                    } while ($_token == ',');
                    $at--;
                }
                break;

            case 'COUNT':
                if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                    return null;
                }
                $expr = array($token, $this->_parsing_read($at, $tokens, $query));
                if ($expr[1] == 'DISTINCT') {
                    $expr[1] = array('DISTINCT');
                    do {
                        $d = $this->_parsing_read($at, $tokens, $query);
                        $expr[1][] = $d;
                        $_token = $this->_parsing_read($at, $tokens, $query);
                    } while ($_token == ',');
                    $at--;
                }
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
                break;

            case 'MAX':
            case 'MIN':
            case 'SUM':
            case 'X_GROUP_CONCAT':
            case 'AVG':
                if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                    return null;
                }
                $expr = array($token);
                $next = $this->_parsing_read($at, $tokens, $query);
                if ($next == 'DISTINCT') {
                    $distinct = true;
                } else {
                    $at--;
                    $distinct = false;
                }
                $_expr = $this->_parsing_read_expression($at, $tokens, $query, $db, false, true, $fail_ok);
                if ($distinct) {
                    $expr[1] = array('DISTINCT', $_expr);
                } else {
                    $expr[1] = $_expr;
                }
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
                break;

            // Conventional expressions...

            case 'X_RAND':
                if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                    return null;
                }
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
                $expr = array($token);
                break;

            case 'X_MOD':
                if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                    return null;
                }
                $expr1 = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
                if (!$this->_parsing_expects($at, $tokens, ',', $query)) {
                    return null;
                }
                $expr2 = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
                $expr = array($token, $expr1, $expr2);
                break;

            case 'X_LEAST':
            case 'X_GREATEST':
            case 'X_COALESCE':
            case 'X_CONCAT':
                if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                    return null;
                }
                $exprx = array();
                do {
                    $exprx[] = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
                    if (!$this->_parsing_expects($at, $tokens, ',', $query, true)) {
                        $at--;
                        break;
                    }
                }
                while (true);
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
                $expr = array($token, $exprx);
                break;

            case 'CAST':
                if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                    return null;
                }
                $expr = $this->_parsing_read_expression($at, $tokens, $query, $db, false, true, $fail_ok);
                if (!$this->_parsing_expects($at, $tokens, 'AS', $query)) {
                    return null;
                }
                $type = $this->_parsing_read($at, $tokens, $query);
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
                $expr = array($token, $expr, $type);
                break;

            case 'X_REPLACE':
            case 'X_SUBSTR':
                if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                    return null;
                }
                $expr1 = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
                if (!$this->_parsing_expects($at, $tokens, ',', $query)) {
                    return null;
                }
                $expr2 = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
                if (!$this->_parsing_expects($at, $tokens, ',', $query)) {
                    return null;
                }
                $expr3 = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
                $expr = array($token, $expr1, $expr2, $expr3);
                break;

            case 'X_LENGTH':
                if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                    return null;
                }
                $expr1 = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
                $expr = array($token, $expr1);
                break;

            case 'EXISTS':
                if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                    return null;
                }
                $results = $this->_parse_query_select($tokens, $query, $db, 1, 0, $fail_ok, $at, false);
                if ($results === null) {
                    return null;
                }
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
                $expr = array($token, $results);
                break;

            case '(':
                $next_token = $this->_parsing_read($at, $tokens, $query);
                $at--;
                if ($next_token == 'SELECT') { // subquery
                    $subquery = $this->_parse_query_select($tokens, $query, $db, null, null, $fail_ok, $at, false);
                    if ($subquery === null) {
                        return null;
                    }
                    $expr = array('SUBQUERY_VALUE', $subquery);
                } else {
                    $expr = array('BRACKETED', $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok));
                }
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
                break;
            case ')':
                $at--;
                break;

            case '"':
            case "'":
                $token = $this->_parsing_read($at, $tokens, $query);
                $expr = array('LITERAL', $token);
                if ((!$this->_parsing_expects($at, $tokens, "'", $query)) && (!$this->_parsing_expects($at, $tokens, '"', $query))) {
                    return null;
                }
                break;

            case 'NULL':
                $expr = array('NULL');
                break;

            case 'NOT':
                $expr = array('NOT', $this->_parsing_read_expression($at, $tokens, $query, $db, false, true, $fail_ok));
                break;

            default: // infix op
                $all_keywords = _get_sql_keywords();
                if (!in_array($token, $all_keywords)) { // Must be a field reference then
                    if (is_numeric($token)) {
                        if (strpos($token, '.') !== false) {
                            $expr = array('LITERAL', floatval($token));
                        } else {
                            $expr = array('LITERAL', intval($token));
                        }
                    } else {
                        if (substr($token, -1) == '.') {
                            $token .= $this->_parsing_read($at, $tokens, $query);
                        }
                        $expr = array('FIELD', $token);
                    }
                } elseif ($token == '-') {
                    $token = $this->_parsing_read($at, $tokens, $query);
                    if (strpos($token, '.') !== false) {
                        $expr = array('LITERAL', -floatval($token));
                    } else {
                        $expr = array('LITERAL', -intval($token));
                    }
                } else {
                    $this->_bad_query($query, false, 'Unexpected token (' . $token . ') in expression');
                }

                break;
        }

        // Find the operation now linking across (NB: We're not implementing BODMAS, we assume SQL calculations are very simple and this we'll just do ltr order)
        if ($look_for_any_connectives) {
            $token = $this->_parsing_read($at, $tokens, $query, true);

            if ($token == 'NOT') {
                $doing_not = true;
                $token = $this->_parsing_read($at, $tokens, $query, true);
            }

            switch ($token) {
                case '+':
                case '-':
                case '/':
                case '>':
                case '<':
                case '>=':
                case '<=':
                case '=':
                case '<>':
                case 'LIKE':
                    $expr = array($token, $expr, $this->_parsing_read_expression($at, $tokens, $query, $db, false, true, $fail_ok));
                    break;

                case '*':
                    $expr = array('MULTI', $expr, $this->_parsing_read_expression($at, $tokens, $query, $db, false, true, $fail_ok));
                    break;

                case 'IS':
                    $token = $this->_parsing_read($at, $tokens, $query);
                    if ($token == 'NULL') {
                        $expr = array('IS_NULL', $expr);
                    } else {
                        $at--;
                        if (!$this->_parsing_expects($at, $tokens, 'NOT', $query)) {
                            return null;
                        }
                        if (!$this->_parsing_expects($at, $tokens, 'NULL', $query)) {
                            return null;
                        }
                        $expr = array('IS_NOT_NULL', $expr);
                    }
                    break;

                case 'BETWEEN':
                    $expr1 = $this->_parsing_read_expression($at, $tokens, $query, $db, false, true, $fail_ok);
                    if (!$this->_parsing_expects($at, $tokens, 'AND', $query)) {
                        return null;
                    }
                    $expr2 = $this->_parsing_read_expression($at, $tokens, $query, $db, false, true, $fail_ok);
                    $expr = array('BETWEEN', $expr, $expr1, $expr2);
                    break;

                case 'IN':
                    if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                        return null;
                    }

                    $token = $this->_parsing_read($at, $tokens, $query);

                    $at--;

                    if ($token == 'SELECT') {
                        $test = $this->_parse_query_select($tokens, $query, $db, null, 0, $fail_ok, $at, false);
                        if ($test === null) {
                            return null;
                        }

                        $expr = array('IN_SUBQUERY', $expr, $test);
                    } else {
                        $or_list = array();
                        do {
                            $expr_in = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
                            if (is_null($expr_in)) { // Force an exit
                                break;
                            }
                            $or_list[] = $expr_in;
                            $token = $this->_parsing_read($at, $tokens, $query);
                        } while ($token == ',');
                        $at--;

                        $expr = array('IN', $expr, $or_list);
                    }
                    if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                        return null;
                    }
                    break;

                default:
                    if (!is_null($token)) {
                        $at--;
                    }
                    break;
            }

            if ($doing_not) {
                $expr = array('NOT', $expr);
            }
        }

        // More connectives?
        if ($look_for_connectives) {
            $token = $this->_parsing_read($at, $tokens, $query, true);
            $tail = &$expr;
            while (!is_null($token)) {
                switch ($token) {
                    case 'AND':
                        $next_expr = $this->_parsing_read_expression($at, $tokens, $query, $db, false, true, $fail_ok);
                        $tail = array('AND', $tail, $next_expr);
                        break;

                    case 'OR':
                        $next_expr = $this->_parsing_read_expression($at, $tokens, $query, $db, false, true, $fail_ok);
                        $expr = array('OR', $expr, $next_expr);
                        $tail = &$next_expr;
                        break;

                    default:
                        $at--;
                        break 2;
                }
                $token = $this->_parsing_read($at, $tokens, $query, true);
            }
        }

        return $expr;
    }

    /**
     * Execute an expression.
     *
     * @param  array $expr The expression
     * @param  array $bindings Bindings available in the execution scope
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  ?array $full_set The full record set within a HAVING scope (null: not in a HAVING scope)
     * @return ?mixed The result (null: error/NULL)
     */
    protected function _execute_expression($expr, $bindings, $query, $db, $fail_ok, $full_set = null)
    {
        switch ($expr[0]) {
            // Aggregate expressions...

            case 'COUNT':
            case 'MAX':
            case 'MIN':
            case 'SUM':
            case 'X_GROUP_CONCAT':
            case 'AVG':
                if ($full_set === null) {
                    return $this->_bad_query($query, $fail_ok, 'Cannot use aggregate function (' . $expr[0] . ') outside SELECT/HAVING scope');
                }

                $temp = $this->_function_set_scoping($full_set, array($expr), $bindings, $query, $db, $fail_ok);
                if ($temp === null) {
                    return null;
                }

                $temp = array_values($temp);
                return $temp[count($temp) - 1];

            // Conventional expressions...

            case 'X_COALESCE':
                $vals = array();
                $val = null;
                foreach ($expr[1] as $_expr) {
                    $val = $this->_execute_expression($_expr, $bindings, $query, $db, $fail_ok, $full_set);
                    if ($val !== null) {
                        break;
                    }
                }
                return $val;

            case 'CAST':
                $result = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                switch ($expr[2]) {
                    case 'CHAR':
                        $result = strval($result);
                        break;

                    case 'INT':
                        $result = intval($result);
                        break;

                    case 'FLOAT':
                        $result = floatval($result);
                        break;

                    default:
                        return $this->_bad_query($query, $fail_ok, 'Unrecognised CAST type' . $expr[2]);
                }
                return $result;

            case 'BRACKETED':
                return $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);

            case 'LITERAL':
                return $expr[1];

            case 'NULL':
                return null;

            case 'FIELD':
                return $bindings[$expr[1]];

            case '+':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a + $b);

            case '-':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a - $b);

            case 'MULTI':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a * $b);

            case '/':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a / $b);

            case '>':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a > $b);

            case '<':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a < $b);

            case '>=':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a >= $b);

            case '<=':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a <= $b);

            case '=':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a == $b);

            case '<>':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a != $b);

            case 'LIKE':
                $value = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                if ($value === null) {
                    return null;
                }
                $expr_eval = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                return simulated_wildcard_match($value, $expr_eval, true);

            case 'EXISTS':
                list($exists_select, $exists_as, $exists_joins, $exists_where_expr, $exists_group_by, $exists_having, $exists_orders, $exists_unions, $exists_start, $exists_max) = $expr[1];
                $exists_results = $this->_execute_query_select($exists_select, $exists_as, $exists_joins, $exists_where_expr, $exists_group_by, $exists_having, $exists_orders, $exists_unions, $query, $db, $exists_max, $exists_start, $bindings, $fail_ok);
                if ($exists_results === null) {
                    return null;
                }
                return count($exists_results) != 0;

            case 'NOT':
                $value = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                if ($value === null) {
                    return null;
                }
                return @!$value;

            case 'AND':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a && $b);

            case 'OR':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) && ($b === null)) {
                    return null;
                }
                if (($a === false) && ($b === null)) {
                    return null;
                }
                if (($a === null) && ($b === false)) {
                    return null;
                }
                return @($a || $b);

            case 'IS_NULL':
                return ($this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set) === null);

            case 'IS_NOT_NULL':
                return ($this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set) !== null);

            case 'BETWEEN':
                $comp = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $a = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[3], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @(($comp >= $a) && ($comp <= $b));

            case 'X_REPLACE':
                $search = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                $replace = $this->_execute_expression($expr[3], $bindings, $query, $db, $fail_ok);
                $subject = $this->_execute_expression($expr[1], $bindings, $query, $fail_ok, $full_set);
                if (($search === null) || ($replace === null) || ($subject === null)) {
                    return null;
                }
                return str_replace($search, $replace, $subject);

            case 'X_CONCAT':
                $vals = array();
                foreach ($expr[1] as $_expr) {
                    $value = $this->_execute_expression($_expr, $bindings, $query, $db, $fail_ok, $full_set);
                    if ($value === null) {
                        return null;
                    }
                    $vals[] = $value;
                }
                return implode('', $vals);

            case 'X_LENGTH':
                $value = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                if ($value === null) {
                    return null;
                }
                return cms_mb_strlen($value);

            case 'SUBQUERY_VALUE':
                list($subquery_select, $subquery_as, $subquery_joins, $subquery_where_expr, $subquery_group_by, $subquery_having, $subquery_orders, $subquery_unions, $subquery_start, $subquery_max) = $expr[1];
                $subquery = $this->_execute_query_select($subquery_select, $subquery_as, $subquery_joins, $subquery_where_expr, $subquery_group_by, $subquery_having, $subquery_orders, $subquery_unions, $query, $db, $subquery_max, $subquery_start, $bindings, $fail_ok);
                if ($subquery === null) {
                    return null;
                }
                return isset($subquery[0]) ? array_shift($subquery[0]) : null;

            case 'X_RAND':
                return mt_rand(0, mt_getrandmax());

            case 'X_SUBSTR':
                $string = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $start = min(0, $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set) - 1);
                $length = $this->_execute_expression($expr[3], $bindings, $query, $db, $fail_ok, $full_set);
                if (($string === null) || ($start === null) || ($length === null)) {
                    return null;
                }
                return cms_mb_substr($string, $start, $length);

            case 'X_LEAST':
                $vals = array();
                foreach ($expr[1] as $_expr) {
                    $value = $this->_execute_expression($_expr, $bindings, $query, $db, $fail_ok, $full_set);
                    if ($value === null) {
                        return null;
                    }
                    $vals[] = $value;
                }
                return call_user_func_array('min', $vals);

            case 'X_GREATEST':
                $vals = array();
                foreach ($expr[1] as $_expr) {
                    $value = $this->_execute_expression($_expr, $bindings, $query, $db, $fail_ok, $full_set);
                    if ($value === null) {
                        return null;
                    }
                    $vals[] = $value;
                }
                return call_user_func_array('max', $vals);

            case 'X_MOD':
                $a = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                $b = $this->_execute_expression($expr[2], $bindings, $query, $db, $fail_ok, $full_set);
                if (($a === null) || ($b === null)) {
                    return null;
                }
                return @($a % $b);

            case 'IN':
                $val = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);
                foreach ($expr[2] as $in) {
                    if ($val == $this->_execute_expression($in, $bindings, $query, $db, $fail_ok, $full_set)) {
                        return true;
                    }
                }
                return false;

            case 'IN_SUBQUERY':
                $val = $this->_execute_expression($expr[1], $bindings, $query, $db, $fail_ok, $full_set);

                list($subquery_select, $subquery_as, $subquery_joins, $subquery_where_expr, $subquery_group_by, $subquery_having, $subquery_orders, $subquery_unions, $subquery_start, $subquery_max) = $expr[2];
                $results = $this->_execute_query_select($subquery_select, $subquery_as, $subquery_joins, $subquery_where_expr, $subquery_group_by, $subquery_having, $subquery_orders, $subquery_unions, $query, $db, $subquery_max, $subquery_start, $bindings, $fail_ok);

                $or_list = array();
                foreach ($results as $result) {
                    $result = array_values($result);
                    $or_list[] = $result[0];
                }

                foreach ($or_list as $in) {
                    if ($val == $in) {
                        return true;
                    }
                }
                return false;
        }

        $this->_bad_query($query, false, 'Internal error evaluating expression, ' . $expr[0] . ' not recognised in evaluation context');
        return null;
    }

    /**
     * Execute an UPDATE query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  ?integer $max The maximum number of rows to affect (null: no limit)
     * @param  ?integer $start The start row to affect (null: no specification)
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?mixed The results (null: no results)
     */
    protected function _do_query_update($tokens, $query, $db, $max, $start, $fail_ok)
    {
        // Parse
        $at = 0;
        if (!$this->_parsing_expects($at, $tokens, 'UPDATE', $query)) {
            return null;
        }
        $table_name = $this->_parsing_read($at, $tokens, $query);
        if (!$this->_parsing_expects($at, $tokens, 'SET', $query)) {
            return null;
        }
        $set = array();
        do {
            $token = $this->_parsing_read($at, $tokens, $query);

            if (!$this->_parsing_expects($at, $tokens, '=', $query)) {
                return null;
            }
            $expr = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
            if (is_null($expr)) { // Force an exit
                break;
            }

            $set[$token] = $expr;

            $token = $this->_parsing_read($at, $tokens, $query, true);
        } while ($token === ',');
        if (!is_null($token)) {
            $at--;
        }
        $token = $this->_parsing_read($at, $tokens, $query, true);
        if ($token === 'WHERE') {
            $where_expr = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
        } else {
            $where_expr = array('LITERAL', true);
            if (!is_null($token)) {
                $at--;
            }
        }

        // Execute
        $schema = $this->_read_schema($db, $table_name, $fail_ok);
        if (is_null($schema)) {
            return null;
        }
        $records = $this->_read_all_records($db, $table_name, '', $schema, $where_expr, array(), $fail_ok, $query, false, $max);
        if (is_null($records)) {
            return null;
        }
        $i = 0;
        $done = 0;
        foreach ($records as $guid => $record) {
            if (!is_string($guid)) {
                $guid = strval($guid); // As PHP can use type for array keys
            }
            $test = $this->_execute_expression($where_expr, $record, $query, $db, $fail_ok);
            if ($test) {
                if ($i >= $start) {
                    $record_new = array();
                    foreach ($set as $column_name => $expr) {
                        $record_new[$column_name] = $this->_execute_expression($expr, $record, $query, $db, $fail_ok);
                    }
                    $this->_type_check($schema, $record_new, $query);
                    $record = $record_new + $record;
                    if ($this->_key_conflict_check($db, $table_name, $schema, $record, $query, $fail_ok, $guid)) {
                        return $this->_bad_query($query, $fail_ok, 'A record already exists with a key we are updating to');
                    }
                    $this->_write_record($db, $table_name, $guid, $record, $fail_ok);
                    $done++;
                    if ((!is_null($max)) && ($done > $max)) {
                        break;
                    }
                }
                $i++;
            }
        }

        if (!$this->_parsing_check_ended($at, $tokens, $query)) {
            return null;
        }
        return null;
    }

    /**
     * Execute a DELETE query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  ?integer $max The maximum number of rows to affect (null: no limit)
     * @param  ?integer $start The start row to affect (null: no specification)
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?mixed The results (null: no results)
     */
    protected function _do_query_delete($tokens, $query, $db, $max, $start, $fail_ok)
    {
        // Parse
        $at = 0;
        if (!$this->_parsing_expects($at, $tokens, 'DELETE', $query)) {
            return null;
        }
        if (!$this->_parsing_expects($at, $tokens, 'FROM', $query)) {
            return null;
        }
        $table_name = $this->_parsing_read($at, $tokens, $query);
        $token = $this->_parsing_read($at, $tokens, $query, true);
        if ($token === 'WHERE') {
            $where_expr = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
        } else {
            $where_expr = array('LITERAL', true);
            if (!is_null($token)) {
                $at--;
            }
        }

        // Execute
        $schema = $this->_read_schema($db, $table_name, $fail_ok);
        if (is_null($schema)) {
            return null;
        }
        $records = $this->_read_all_records($db, $table_name, '', $schema, $where_expr, array(), $fail_ok, $query, false, $max);
        if (is_null($records)) {
            return null;
        }
        $i = 0;
        $done = 0;
        foreach ($records as $guid => $record) {
            if (!is_string($guid)) {
                $guid = strval($guid); // As PHP can use type for array keys
            }
            $test = $this->_execute_expression($where_expr, $record, $query, $db, $fail_ok);
            if ($test) {
                if ($i >= $start) {
                    $path = $db[0] . '/' . $table_name . '/' . $guid . '.xml-volatile';
                    if (!file_exists($path)) {
                        $path = $db[0] . '/' . $table_name . '/' . $guid . '.xml';
                    }
                    $this->_delete_record($path, $db);
                    $done++;
                    if ((!is_null($max)) && ($done > $max)) {
                        break;
                    }
                }
                $i++;
            }
        }

        if (!$this->_parsing_check_ended($at, $tokens, $query)) {
            return null;
        }
    }

    /**
     * Execute a TRUNCATE query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?mixed The results (null: no results)
     */
    protected function _do_query_truncate($tokens, $query, $db, $fail_ok)
    {
        $at = 0;
        if (!$this->_parsing_expects($at, $tokens, 'TRUNCATE', $query)) {
            return null;
        }

        $table_name = $this->_parsing_read($at, $tokens, $query);

        $_path = $db[0] . '/' . $table_name;
        $dh = opendir($_path);
        while (($file = readdir($dh)) !== false) {
            $ext = get_file_extension($file);
            if (($ext == 'xml') || ($ext == 'xml-volatile')) {
                $path = $_path . '/' . $file;
                $this->_delete_record($path, $db);
            }
        }
        closedir($dh);

        if (!$this->_parsing_check_ended($at, $tokens, $query)) {
            return null;
        }

        return null;
    }

    /**
     * Execute a SELECT query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  ?integer $max The maximum number of rows to affect (null: no limit)
     * @param  ?integer $start The start row to affect (null: no specification)
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  integer $at Our offset counter
     * @param  boolean $do_end_check Whether to not do the check to make sure we've parsed everything
     * @return ?mixed The results (null: no results)
     */
    protected function _do_query_select($tokens, $query, $db, $max, $start, $fail_ok, &$at, $do_end_check = true)
    {
        $test = $this->_parse_query_select($tokens, $query, $db, $max, $start, $fail_ok, $at, $do_end_check);
        if ($test === null) {
            return null;
        }
        list($select, $as, $joins, $where_expr, $group_by, $having, $orders, $unions, $start, $max) = $test;
        return $this->_execute_query_select($select, $as, $joins, $where_expr, $group_by, $having, $orders, $unions, $query, $db, $max, $start, array(), $fail_ok);
    }

    /**
     * Parse a SELECT query.
     *
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  ?integer $max The maximum number of rows to affect (null: no limit)
     * @param  ?integer $start The start row to affect (null: no specification)
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  integer $at Our offset counter
     * @param  boolean $do_end_check Whether to not do the check to make sure we've parsed everything
     * @return ?array A tuple of query parts (null: error)
     */
    protected function _parse_query_select($tokens, $query, $db, $max, $start, $fail_ok, &$at, $do_end_check = true)
    {
        $all_keywords = _get_sql_keywords();

        // SELECT

        $as = null;

        if ($tokens[$at] == '(') {
            if (!$this->_parsing_expects($at, $tokens, '(', $query)) {
                return null;
            }

            $is_bracketed = true;
        } else {
            $is_bracketed = false;
        }

        if (!$this->_parsing_expects($at, $tokens, 'SELECT', $query)) {
            return null;
        }
        $select = array();
        do {
            $token = $this->_parsing_read($at, $tokens, $query);
            if (substr($token, -1) == '.') {
                $token .= $this->_parsing_read($at, $tokens, $query);
            }

            if ($token == '*') {
                $select[] = array('*');
            } elseif (substr($token, -2) == '.*') {
                $select[] = array('*', substr($token, 0, strlen($token) - 2));
            } else {
                $at--;
                $expression = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);

                $as_token = $this->_parsing_read($at, $tokens, $query, true);
                if ($as_token === ')') {
                    $at--;
                    $as_token = null;
                }
                if ($as_token === null) { // reached end of query
                    $select[] = $expression;
                } else {
                    if ($as_token == 'AS') {
                        $as = $this->_parsing_read($at, $tokens, $query);
                        $select[] = array('AS', $expression, $as);
                    } elseif (($as_token == '*') && (substr($token, -1) == '.')) {
                        $select[] = array('*', substr($token, 0, strlen($token) - 1));
                    } else {
                        $at--;
                        $select[] = $expression;
                    }
                }
            }

            $token = $this->_parsing_read($at, $tokens, $query, true);
        } while ($token === ',');
        if ($token !== null) {
            $at--;
        }

        // FROM

        if ($this->_parsing_expects($at, $tokens, 'FROM', $query, true)) {
            $closing_brackets_needed = 0;
            $table_name = $this->_parsing_read($at, $tokens, $query);
            if ($table_name == '(') { // subquery
                $table_name = $this->_parse_query_select($tokens, $query, $db, null, null, $fail_ok, $at, false);
                if ($table_name === null) {
                    return null;
                }
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
            }
            $as_test = $this->_parsing_read($at, $tokens, $query, true);
            if ((!is_null($as_test)) && ($as_test != 'ON') && ($as_test != ')') && ($as_test != 'LIMIT') && ($as_test != 'GROUP') && ($as_test != 'ORDER') && ($as_test != 'WHERE') && ($as_test != 'LEFT') && ($as_test != 'RIGHT') && ($as_test != 'INNER') && ($as_test != 'JOIN')) {
                $as = $as_test;
            } else {
                $as = is_array($table_name) ? 'x' : $table_name;
                if (!is_null($as_test)) {
                    $at--;
                }
            }

            for ($i = 0; $i < $closing_brackets_needed; $i++) {
                $br = $this->_parsing_read($at, $tokens, $query, true);
                if ($br === ')') {
                    $i--;
                    $closing_brackets_needed--;
                } else {
                    $at--;
                    break;
                }
            }

            $joins = array(array('SIMPLE', $table_name, $as));
            do {
                $test = $this->_read_join($at, $tokens, $query, $db, $fail_ok, $closing_brackets_needed);
                if (!is_null($test)) {
                    $joins[] = $test;
                }
            } while (!is_null($test));

            for ($i = 0; $i < $closing_brackets_needed; $i++) {
                if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                    return null;
                }
            }
        } else {
            $joins = array();
            $at--;
        }

        // WHERE

        $token = $this->_parsing_read($at, $tokens, $query, true);
        if ($token === 'WHERE') {
            $where_expr = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
            if ($where_expr === null) {
                return null;
            }
        } else {
            $where_expr = array('LITERAL', true);
            if (!is_null($token)) {
                $at--;
            }
        }

        // GROUP BY

        $having = null;
        $token = $this->_parsing_read($at, $tokens, $query, true);
        if ($token === 'GROUP') {
            if (!$this->_parsing_expects($at, $tokens, 'BY', $query)) {
                return null;
            }
            $group_by = array();
            do {
                $group_by[] = $this->_parsing_read($at, $tokens, $query);
                $test = $this->_parsing_read($at, $tokens, $query, true);
            } while ($test === ',');
            if (!is_null($test)) {
                $at--;
            }

            // HAVING

            $token = $this->_parsing_read($at, $tokens, $query, true);
            if ($token === 'HAVING') {
                $having = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
                if ($having === null) {
                    return null;
                }
            } else {
                if ($token !== null) {
                    $at--;
                }
            }
        } else {
            $group_by = null;
            if (!is_null($token)) {
                $at--;
            }
        }

        // ORDER

        $token = $this->_parsing_read($at, $tokens, $query, true);
        if ($token === 'ORDER') {
            if (!$this->_parsing_expects($at, $tokens, 'BY', $query)) {
                return null;
            }
            $orders = '';
            do {
                $order = $this->_parsing_read($at, $tokens, $query);
                $token = $this->_parsing_read($at, $tokens, $query, true);
                $reverse = false;
                if (($token === 'ASC') || ($token === 'DESC') || ($token === ',') || (!array_key_exists($token, $GLOBALS['DELIMITERS_FLIPPED']))) {
                    if ($token == 'DESC') {
                        $reverse = true;
                    }
                    if ($token == ',') {
                        $at--;
                    }
                } else {
                    if (($token == 'LIMIT') || ($token == 'UNION')) {
                        if (!is_null($token)) {
                            $at--;
                        }
                    } elseif (!is_null($token)) {
                        // Ignore complex order bys
                        $orders = null;
                        $token = null;
                        $at = count($tokens);
                        break;
                    }
                }
                if ($orders != '') {
                    $orders .= ',';
                }
                if ($reverse) {
                    $orders .= '!';
                }
                $orders .= $order;
                $test = $this->_parsing_read($at, $tokens, $query, true);
            } while ($test === ',');
            if (!is_null($token)) {
                $at--;
            }
        } else {
            $orders = null;
            if (!is_null($token)) {
                $at--;
            }
        }

        // LIMIT

        $token = $this->_parsing_read($at, $tokens, $query, true);
        if (!is_null($token)) {
            if ($token == 'LIMIT') {
                $max = intval($this->_parsing_read($at, $tokens, $query));
                $token = $this->_parsing_read($at, $tokens, $query, true);
                if (!is_null($token)) {
                    if ($token == ',') {
                        $start = $max;
                        $max = intval($this->_parsing_read($at, $tokens, $query));
                    } else {
                        $at--;
                    }
                }
            } else {
                $at--;
            }
        }

        if ($is_bracketed) {
            if (!$this->_parsing_expects($at, $tokens, ')', $query)) {
                return null;
            }
        }

        // UNION clause?
        $unions = array();
        $token = $this->_parsing_read($at, $tokens, $query, true);
        if ($token === 'UNION') {
            $token = $this->_parsing_read($at, $tokens, $query);
            if ($token == 'ALL') {
                $de_dupe = false;
            } else {
                $de_dupe = true;
                $at--;
            }

            $test = $this->_parse_query_select($tokens, $query, $db, $max, $start, $fail_ok, $at, $do_end_check);
            if ($test === null) {
                return null;
            }

            $unions[] = array($test, $de_dupe);
        } else {
            if (!is_null($token)) {
                $at--;
            }
            if ($do_end_check) {
                if (!$this->_parsing_check_ended($at, $tokens, $query)) {
                    return null;
                }
            }
        }

        // ---

        return array($select, $as, $joins, $where_expr, $group_by, $having, $orders, $unions, $start, $max);
    }

    /**
     * Execute a parsed SELECT query.
     *
     * @param  array $select Select constructs
     * @param  ?string $as The renaming of our table, so we can recognise it in the join condition (null: no renaming)
     * @param  array $joins Join constructs
     * @param  array $where_expr Where constructs
     * @param  ?array $group_by Grouping by constructs (null: none)
     * @param  ?array $having Having construct (null: none)
     * @param  ?string $orders Ordering string for sort_maps_by (null: none)
     * @param  array $unions Union constructs
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  ?integer $max The maximum number of rows to affect (null: no limit)
     * @param  ?integer $start The start row to affect (null: no specification)
     * @param  array $bindings Bindings available in the execution scope
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?mixed The results (null: no results)
     */
    protected function _execute_query_select($select, $as, $joins, $where_expr, $group_by, $having, $orders, $unions, $query, $db, $max, $start, $bindings, $fail_ok)
    {
        // Execute to get records
        $done = 0;
        if (count($joins) == 0) {
            $records = array(array());
        }
        elseif ((count($joins) == 1) && (!is_array($joins[0][1])) && ($where_expr == array('LITERAL', true)) && ($select === array(array('COUNT', '*'))) && ($orders === null)) { // Quick fudge to get fast table counts
            global $DIR_CONTENTS_CACHE;
            if (!isset($DIR_CONTENTS_CACHE[$joins[0][1]])) {
                if (is_dir($db[0] . '/' . $joins[0][1])) {
                    chdir($db[0] . '/' . $joins[0][1]);
                    $dh = @glob('{,.}*.{xml,xml-volatile}', GLOB_NOSORT | GLOB_BRACE);
                    if ($dh === false) {
                        $dh = array();
                    }
                    @chdir(get_file_base());
                } else {
                    $dh = array();
                }
                $DIR_CONTENTS_CACHE[$joins[0][1]] = $dh;
            } else {
                $dh = $DIR_CONTENTS_CACHE[$joins[0][1]];
            }
            $records = array();
            foreach ($dh as $d) {
                $records[] = array('file' => $d);
            }
        } else {
            foreach ($joins as $join) {
                if ($join[0] == 'SIMPLE') {
                    $joined_as = $join[2];

                    if (is_array($join[1])) {
                        $schema = array();

                        list($join_select, $join_as, $join_joins, $join_where_expr, $join_group_by, $join_having, $join_orders, $join_unions, $join_start, $join_max) = $join[1];
                        $records = $this->_execute_query_select($join_select, $join_as, $join_joins, $join_where_expr, $join_group_by, $join_having, $join_orders, $join_unions, $query, $db, $join_max, $join_start, $bindings, $fail_ok);
                    } else {
                        $schema = $this->_read_schema($db, $join[1], $fail_ok);

                        if (is_null($schema)) {
                            return null;
                        }
                        $records = $this->_read_all_records($db, $join[1], $joined_as, $schema, $where_expr, $bindings, $fail_ok, $query, false, ((count($joins) == 1) && ($orders === null)) ? $max : null);
                        if (is_null($records)) {
                            return null;
                        }

                        foreach ($schema as $k => $v) {
                            $schema[$joined_as . '.' . $k] = $v; // Needed so all scoped variables can be put in place as NULL's in a right variable
                        }
                    }

                    // Handle the join as condition
                    foreach ($records as $guid => $record) {
                        if (!is_string($guid)) {
                            $guid = strval($guid); // As PHP can use type for array keys
                        }
                        $new_record = $record;
                        foreach ($record as $key => $val) {
                            $new_record[$joined_as . '.' . $key] = $val;
                        }
                        $records[$guid] = $new_record;
                    }
                } else {
                    $result = $this->_execute_join($db, $as, $join, $query, $records, $schema, $where_expr, $bindings, $fail_ok);
                    if (is_null($result)) {
                        return null;
                    }
                    list($records, $schema) = $result;
                }
            }
        }

        // Filter by WHERE
        $pre_filtered_records = array();
        foreach ($records as $record) {
            $test = $this->_execute_expression($where_expr, $record + $bindings, $query, $db, $fail_ok);
            if ($test) {
                $pre_filtered_records[] = $record;
            }
        }
        $records = $pre_filtered_records;

        if (!is_null($group_by)) {
            // GROUP BY
            $record_sets = array();
            foreach ($records as $record) {
                $s = array();
                foreach ($group_by as $v) {
                    $s[] = $record[$v];
                }
                if (!array_key_exists(serialize($s), $record_sets)) {
                    $record_sets[serialize($s)] = array();
                }
                $record_sets[serialize($s)][] = $record;
            }
            $records = array();
            $records_full_set = array();
            foreach ($record_sets as $set) { // Functions have special meaning in GROUP BY, and we need to compute them in the group-aware scope
                $rep = $this->_function_set_scoping($set, $select, $set[0], $query, $db, $fail_ok);
                $records[] = $rep;
                $records_full_set[] = $set;
            }

            // Filter by HAVING
            if ($group_by !== null) {
                if ($having !== null) {
                    $pre_filtered_records = array();
                    foreach ($records as $i => $record) {
                        $test = $this->_execute_expression($having, $record, $query, $db, $fail_ok, $records_full_set[$i]);
                        if ($test) {
                            $pre_filtered_records[] = $record;
                        }
                    }
                    $records = $pre_filtered_records;
                }
            }
        } else {
            // Special handling for DISTINCT
            foreach ($select as $s_term) {
                switch ($s_term[0]) {
                    case 'DISTINCT':
                        $index = array();
                        foreach ($records as $set_item) {
                            $val = array();
                            foreach ($s_term[1] as $di) {
                                $val[] = $set_item[$di];
                            }
                            $index[serialize($val)] = $set_item;
                        }
                        $records = array_values($index);
                        break;
                }
            }

            // Now handle functions (as applied to all records, as no GROUP BY)
            $single_result = false;
            foreach ($select as $s) {
                if (($s[0] == 'MIN') || ($s[0] == 'MAX') || ($s[0] == 'SUM') || ($s[0] == 'X_GROUP_CONCAT') || ($s[0] == 'COUNT') || ($s[0] == 'AVG')) {
                    $single_result = true;
                }
            }
            foreach ($records as $i => $record) {
                $records[$i] = $this->_function_set_scoping($records, $select, $record, $query, $db, $fail_ok);
                if ($single_result) {
                    $records = array($i => $records[$i]);
                    break;
                }
            }
        }

        // Sort by ORDER BY
        if (!is_null($orders)) {
            // Putting in aliases early, as may be used in ORDER BY (not in WHERE though)
            foreach ($records as $h => $record) {
                foreach ($select as $i => $want) {
                    switch ($want[0]) {
                        case 'AS':
                            switch ($want[1][0]) {
                                case 'DISTINCT':
                                case 'MAX':
                                case 'MIN':
                                case 'COUNT':
                                case 'SUM':
                                case 'X_GROUP_CONCAT':
                                case 'AVG':
                                    // Was already specially processed (at *MARKER*), compound function - just copy through
                                    $as = $want[2];
                                    $records[$h][preg_replace('#^.*\.#', '', $as)] = $record[$as];
                                    break 2;
                                default:
                                    $as = $want[2];
                                    $want = $want[1];
                                    $records[$h][preg_replace('#^.*\.#', '', $as)] = $this->_execute_expression($want, $record, $query, $db, $fail_ok);
                                    break;
                            }

                            break;
                    }
                }
            }

            // Do sorting
            sort_maps_by($records, $orders);
        }

        // Cut
        $i = 0;
        $filtered_records = array();
        foreach ($records as $record) {
            if ($i >= $start) {
                if ((!is_null($max)) && ($done >= $max)) {
                    break;
                }
                $filtered_records[] = $record;
                $done++;
            }
            $i++;
        }
        $records = $filtered_records;

        // Selecting correct fields
        $results = array();
        foreach ($records as $record) {
            $_record = array();
            foreach ($select as $i => $want) {
                $as = null;

                switch ($want[0]) { // NB: COUNT, SUM, etc, already have their values rolled out into $record and we do not need to consider it here
                    case 'MAX':
                    case 'MIN':
                    case 'COUNT':
                    case 'SUM':
                    case 'X_GROUP_CONCAT':
                    case 'AVG':
                        // Was already specially process, compound function - just copy through
                        $as = $this->_param_name_for($want[1], $i);
                        $_record[preg_replace('#^.*\.#', '', $as)] = $record[$as];
                        break;

                    case '*':
                        if (array_key_exists(1, $want)) {
                            $filtered_record = array();
                            foreach ($record as $key => $val) {
                                if (substr($key, 0, strlen($want[1] . '.')) == $want[1] . '.') {
                                    $filtered_record[substr($key, strlen($want[1] . '.'))] = $val;
                                }
                            }
                            $_record += $filtered_record;
                        } else {
                            $filtered_record = array();
                            foreach ($record as $key => $val) {
                                if (strpos($key, '.') === false) {
                                    $filtered_record[$key] = $val;
                                }
                            }
                            $_record += $filtered_record;
                        }
                        break;

                    case 'DISTINCT':
                        $val = array();
                        foreach ($want[1] as $param) {
                            if (strpos($param, '.') === false) {
                                $_record[$param] = $record[$param];
                            } else {
                                $_record[preg_replace('#^.*\.#', '', $param)] = $record[$param];
                            }
                        }
                        break;

                    case 'AS':
                        switch ($want[1][0]) {
                            case 'DISTINCT':
                            case 'MAX':
                            case 'MIN':
                            case 'COUNT':
                            case 'SUM':
                            case 'X_GROUP_CONCAT':
                            case 'AVG':
                                // Was already specially process, compound function - just copy through
                                $as = $want[2];
                                $_record[preg_replace('#^.*\.#', '', $as)] = $record[$as];
                                $want = $want[1];
                                break 2;
                            default:
                                $as = $want[2];
                                $want = $want[1];
                                break;
                        }

                    default:
                        if ($as === null) {
                            $as = $this->_param_name_for(((isset($want[1])) && ($want[0] == 'FIELD')) ? $want[1] : ('arb' . strval($i)), $i);
                        }
                        $_record[preg_replace('#^.*\.#', '', $as)] = $this->_execute_expression($want, $record, $query, $db, $fail_ok);
                        break;
                }
            }
            $results[] = $_record;
        }

        // If there are no records, but some functions, we need to add a row
        if ((count($results) == 0) && (is_null($group_by))) {
            $rep = $this->_function_set_scoping(array(), $select, array(), $query, $db, $fail_ok);
            if (count($rep) != 0) {
                foreach ($select as $i => $want) {
                    $as = null;
                    if ($want[0] == 'AS') {
                        $as = $want[2];
                        $want = $want[1];
                    }
                    switch ($want[0]) { // NB: COUNT, SUM, etc, already have their values rolled out into $record and we do not need to consider it here
                        case 'FIELD':
                            $param = $this->_param_name_for($want[1], $i);

                            if ($as === null) {
                                $as = $param;
                            }

                            if (!isset($rep[$param])) {
                                $rep[$param] = null;
                            }

                            if ($param != $as) {
                                $rep[$as] = $rep[$param];
                                unset($rep[$param]);
                            }

                            break;
                    }
                }

                $results[] = $rep;
            }
        }

        // Try and validate some stuff PostgreSQL wouldn't like, so we make XML driver as strict as possible
        //  It's not perfect, only works if we actually have a non-zero result set.
        //  Can't dig into expressions because the XML driver doesn't support ordering by them.
        if (count($results) > 0) {
            if ($orders !== null) {
                $matches = array();
                if (preg_match('#^\!?(\w+)$#', $orders, $matches) != 0) {
                    if (!array_key_exists($matches[1], $records[0])) {
                        warn_exit('Cannot sort by ' . $matches[1] . ', it\'s not selected');
                    }
                }
            }
        }

        // UNION clauses
        foreach ($unions as $union) {
            list($test, $de_dupe) = $union;
            list($union_select, $union_as, $union_joins, $union_where_expr, $union_group_by, $union_having, $union_orders, $union_unions, $union_start, $union_max) = $test;

            $results_b = $this->_execute_query_select($union_select, $union_as, $union_joins, $union_where_expr, $union_group_by, $union_group_by, $union_orders, $union_unions, $query, $db, $union_max, $union_start, $bindings, $fail_ok);
            if ($results_b === null) {
                return null;
            }

            if ($de_dupe) {
                foreach ($results_b as $r) {
                    if (!in_array($r, $results)) {
                        $results[] = $r;
                    }
                }
            } else {
                $results = array_merge($results, $results_b);
            }
        }

        // ---

        return $results;
    }

    /**
     * Extract a save parameter name from an expression.
     *
     * @param  mixed $param Expression
     * @param  integer $i Offset in a field set
     * @return string Parameter name
     */
    protected function _param_name_for($param, $i)
    {
        if (is_array($param) && isset($param[1])) {
            $param = $param[1];
        }
        if (!is_string($param)) {
            $param = 'val' . strval($i);
        }
        return $param;
    }

    /**
     * Run SQL data filter functions over a result set.
     *
     * @param  array $set The set of results we are operating on
     * @param  array $select Parse tree of what we are selecting
     * @param  array $rep Record we are copying the function results into
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return array The result row based on the set
     */
    protected function _function_set_scoping($set, $select, $rep, $query, $db, $fail_ok)
    {
        foreach ($select as $i => $s_term) {
            $as = null;

            if ($s_term[0] == 'AS') {
                $as = $s_term[2];
                $s_term = $s_term[1];
            }

            if (!isset($s_term[1])) {
                continue;
            }

            if ($as === null) {
                $as = $this->_param_name_for($s_term[1], $i);
            }

            switch ($s_term[0]) {
                case 'MAX':
                    $max = mixed();
                    foreach ($set as $set_item) {
                        $val = $this->_execute_expression($s_term[1], $set_item, $query, $db, $fail_ok);
                        if ((is_null($max)) || ($val > $max)) {
                            $max = $val;
                        }
                    }
                    $rep[$as] = $max;
                    break;

                case 'MIN':
                    $min = mixed();
                    foreach ($set as $set_item) {
                        $val = $this->_execute_expression($s_term[1], $set_item, $query, $db, $fail_ok);
                        if ((is_null($min)) || ($val < $min)) {
                            $min = $val;
                        }
                    }
                    $rep[$as] = $min;
                    break;

                case 'COUNT':
                    if ($s_term[1][0] == 'DISTINCT') {
                        $index = array();
                        foreach ($set as $set_item) {
                            $val = array();
                            for ($di = 1; $di < count($s_term[1]); $di++) {
                                $val[] = $set_item[$s_term[1][$di]];
                            }
                            $index[serialize($val)] = true;
                        }
                        $rep[$as] = count($index);
                    } else {
                        $rep[$as] = count($set);
                    }
                    break;

                case 'SUM':
                    $temp = 0;
                    foreach ($set as $set_item) {
                        $val = $this->_execute_expression($s_term[1], $set_item, $query, $db, $fail_ok);
                        $temp += $val;
                    }
                    if (is_integer($temp)) {
                        $rep[$as] = floatval($temp);
                    } else {
                        $rep[$as] = $temp;
                    }
                    break;

                case 'X_GROUP_CONCAT':
                    $temp_str = '';
                    foreach ($set as $set_item) {
                        $val = $this->_execute_expression($s_term[1], $set_item, $query, $db, $fail_ok);
                        if ($temp_str != '') {
                            $temp_str .= ',';
                        }
                        $temp_str .= $val;
                    }
                    $rep[$as] = $temp_str;
                    break;

                case 'AVG':
                    if (count($set) == 0) {
                        $rep[$as] = null;
                    } else {
                        $temp = 0;
                        foreach ($set as $set_item) {
                            $val = $this->_execute_expression($s_term[1], $set_item, $query, $db, $fail_ok);
                            $temp += $val;
                        }
                        if (is_integer($temp)) {
                            $rep[$as] = floatval($temp) / floatval(count($set));
                        } else {
                            $rep[$as] = $temp / floatval(count($set));
                        }
                    }
                    break;
            }
        }
        return $rep;
    }

    /**
     * Read in a table specifier clause for a WHERE query.
     *
     * @param  integer $at Our offset counter
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  array $db Database connection
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  integer $closing_brackets_needed How many closing brackets we expect
     * @return ?array Join condition (null: no join here)
     */
    protected function _read_join(&$at, $tokens, $query, $db, $fail_ok, &$closing_brackets_needed)
    {
        $token = $this->_parsing_read($at, $tokens, $query, true);

        if (($token !== ',') && ($token !== 'JOIN') && ($token !== 'LEFT') && ($token !== 'RIGHT') && ($token !== 'INNER')) {
            if (!is_null($at)) {
                $at--;
            }
            return null;
        }

        if (($token != 'JOIN') && ($token != ',')) {
            if (!$this->_parsing_expects($at, $tokens, 'JOIN', $query)) {
                return null;
            }
        }

        $join_table = $this->_parsing_read($at, $tokens, $query);

        if ($join_table == '(') {
            $closing_brackets_needed++;
            $join_table = $this->_parsing_read($at, $tokens, $query);
        }

        $joined_as_test = $this->_parsing_read($at, $tokens, $query, true);
        if ((!is_null($joined_as_test)) && ($joined_as_test != 'ON') && ($joined_as_test != 'WHERE') && ($joined_as_test != ',') && ($joined_as_test != 'LEFT') && ($joined_as_test != 'RIGHT') && ($joined_as_test != 'INNER') && ($joined_as_test != 'JOIN')) {
            if ($joined_as_test == 'AS') {
                $joined_as_test = $this->_parsing_read($at, $tokens, $query); // 'AS' is optional
            }
            $joined_as = $joined_as_test;
        } else {
            $joined_as = $join_table;
            if (!is_null($joined_as_test)) {
                $at--;
            }
        }

        if ($token == ',') {
            $on_expr = array('LITERAL', true);
        } else {
            if (!$this->_parsing_expects($at, $tokens, 'ON', $query)) {
                return null;
            }
            $on_expr = $this->_parsing_read_expression($at, $tokens, $query, $db, true, true, $fail_ok);
        }

        for ($i = 0; $i < $closing_brackets_needed; $i++) {
            $br = $this->_parsing_read($at, $tokens, $query, true);
            if ($br === ')') {
                $i--;
                $closing_brackets_needed--;
            } else {
                $at--;
                break;
            }
        }

        switch ($token) {
            case ',':
                $join = array('JOIN', $join_table, $joined_as, $on_expr);
                break;
            case 'JOIN':
                $join = array('JOIN', $join_table, $joined_as, $on_expr);
                break;
            case 'LEFT':
                $join = array('LEFT_JOIN', $join_table, $joined_as, $on_expr);
                break;
            case 'RIGHT':
                $join = array('RIGHT_JOIN', $join_table, $joined_as, $on_expr);
                break;
            case 'INNER':
                $join = array('INNER_JOIN', $join_table, $joined_as, $on_expr);
                break;
        }

        return $join;
    }

    /**
     * Optimize a join condition into a join scope set, if possible.
     * This is destructive.
     *
     * @param  array $join_condition Join condition (parsed WHERE-style clause)
     * @param  array $schema Schema so far
     * @param  array $records Records so far
     * @param  string $joined_as The renaming of our table, so we can recognise it in the join condition
     * @return array Altered join condition
     */
    protected function _setify_join_condition_for_optimisation($join_condition, $schema, $records, $joined_as)
    {
        if ($join_condition[0] == 'AND') {
            $join_condition_a = $this->_setify_join_condition_for_optimisation($join_condition[1], $schema, $records, $joined_as);
            $join_condition_b = $this->_setify_join_condition_for_optimisation($join_condition[2], $schema, $records, $joined_as);
            $join_condition = array('AND', $join_condition_a, $join_condition_b);
        } else {
            if ($join_condition[0] == '=') {
                foreach (array(1, 2) as $i) {
                    if (($join_condition[$i][0] == 'FIELD') && ($join_condition[3 - $i][0] == 'FIELD')) { // If this and other-side expression are both FIELD's
                        $var = preg_replace('#^' . $joined_as . '\.#', '', $join_condition[$i][1]); // Find field reference
                        if (array_key_exists($var, $schema)) { // If this side is in the schema
                            $join_condition[$i][1] = array(); // We'll make it a list instead of a field reference
                            foreach ($records as $r) {
                                $join_condition[$i][1][] = $r[$var];
                            }
                            $join_condition[$i][0] = 'LITERAL';
                            break;
                        }
                    }
                }
            }
        }

        return $join_condition;
    }

    /**
     * Get results from a JOIN.
     *
     * @param  array $db Database connection
     * @param  string $joined_as_prior The renaming of our table, so we can recognise it in the join condition
     * @param  array $join Join op-tree
     * @param  string $query Query that was executed
     * @param  array $records Records so far
     * @param  array $schema Schema so far
     * @param  array $where_expr Expression filtering results (used for optimisation, seeing if we can get a quick key match)
     * @param  array $bindings Bindings available in the execution scope
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return ?array A pair: an array of results, an array of the schema for what has been joined (null: error)
     */
    protected function _execute_join($db, $joined_as_prior, $join, $query, $records, $schema, $where_expr, $bindings, $fail_ok = false)
    {
        $joined_as = $join[2];

        $schema_b = $this->_read_schema($db, $join[1], $fail_ok);
        if (is_null($schema_b)) {
            return null;
        }
        $schema_b_plus = $schema_b;
        foreach ($schema_b as $k => $v) {
            $schema_b_plus[$join[2] . '.' . $k] = $v; // Needed so all scoped variables can be put in place as NULL's in a right variable
        }
        $join_condition = $join[3];
        $join_condition = $this->_setify_join_condition_for_optimisation($join_condition, $schema, $records, $joined_as_prior);
        $where_expr = $this->_setify_join_condition_for_optimisation($where_expr, $schema, $records, $joined_as_prior); // Good for implicit joins (,)
        if ($where_expr == array('LITERAL', true)) {
            $where_expr_combined = $join_condition;
        } else {
            $where_expr_combined = array('AND', $where_expr, $join_condition);
        }
        $records_b = $this->_read_all_records($db, $join[1], $joined_as, $schema_b, $where_expr_combined, $bindings, $fail_ok, $query);
        if (is_null($records_b)) {
            return null;
        }

        // Handle the join as condition
        foreach ($records_b as $guid => $record) {
            if (!is_string($guid)) {
                $guid = strval($guid); // As PHP can use type for array keys
            }
            $new_record = $record;
            foreach ($record as $key => $val) {
                $new_record[$joined_as . '.' . $key] = $val;
            }
            $records_b[$guid] = $new_record;
        }

        $records_results = array();
        switch ($join[0]) {
            case 'JOIN':
            case 'INNER_JOIN':
                foreach ($records as $r1) {
                    foreach ($records_b as $r2) {

                        $join_scope = $r1;
                        foreach ($r2 as $key => $val) {
                            if (array_key_exists($key, $join_scope)) { // Don't allow anything ambiguous
                                unset($join_scope[$key]);
                            } else {
                                $join_scope[$key] = $val;
                            }
                        }
                        $test = $this->_execute_expression($join[3], $join_scope + $bindings, $query, $db, $fail_ok);
                        if ($test) {
                            $records_results[] = $r2 + $r1;
                        }
                    }
                }
                break;

            case 'RIGHT_JOIN':
                foreach ($records_b as $r1) {
                    $matched = false;
                    foreach ($records as $r2) {
                        $join_scope = $r1;
                        foreach ($r2 as $key => $val) {
                            if (array_key_exists($key, $join_scope)) { // Don't allow anything ambiguous
                                unset($join_scope[$key]);
                            } else {
                                $join_scope[$key] = $val;
                            }
                        }
                        $test = $this->_execute_expression($join[3], $join_scope + $bindings, $query, $db, $fail_ok);
                        if ($test) {
                            $records_results[] = $r2 + $r1;
                            $matched = true;
                        }
                    }
                    if (!$matched) {
                        $null_padded = $r1;
                        foreach (array_keys($schema) as $field) {
                            $null_padded[$field] = null;
                        }
                        $records_results[] = $null_padded;
                    }
                }
                break;
            case 'LEFT_JOIN':
                foreach ($records as $r1) {
                    $matched = false;
                    foreach ($records_b as $r2) {
                        $join_scope = $r1;
                        foreach ($r2 as $key => $val) {
                            if (array_key_exists($key, $join_scope)) { // Don't allow anything ambiguous
                                unset($join_scope[$key]);
                            } else {
                                $join_scope[$key] = $val;
                            }
                        }
                        $test = $this->_execute_expression($join[3], $join_scope + $bindings, $query, $db, $fail_ok);
                        if ($test) {
                            $records_results[] = $r2 + $r1;
                            $matched = true;
                        }
                    }
                    if (!$matched) {
                        $null_padded = $r1;
                        foreach (array_keys($schema_b_plus) as $field) {
                            $null_padded[$field] = null;
                        }
                        $records_results[] = $null_padded;
                    }
                }
                break;
        }

        foreach ($schema_b_plus as $k => $v) {
            $schema[$k] = $v;
        }

        return array($records_results, $schema);
    }

    /**
     * Reads the next token.
     *
     * @param  integer $at Our offset counter
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  boolean $fail_ok Whether it can return null if we're out of output (otherwise fails)
     * @return ?string Token read (null: error, read too far)
     */
    protected function _parsing_read(&$at, $tokens, $query, $fail_ok = false)
    {
        $at++;

        if (!array_key_exists($at - 1, $tokens)) {
            if ($fail_ok) {
                return null;
            }
            return $this->_bad_query($query, false, 'Unexpected end of query');
        }

        return preg_replace('#^`(.*)`$#', '${1}', $tokens[$at - 1]);
    }

    /**
     * Expect a certain token next.
     *
     * @param  integer $at Our offset counter
     * @param  array $tokens Tokens
     * @param  string $token Token expected
     * @param  string $query Query that was executed
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return boolean Success status
     */
    protected function _parsing_expects(&$at, $tokens, $token, $query, $fail_ok = false)
    {
        $next = $this->_parsing_read($at, $tokens, $query, $fail_ok);
        if ($next !== $token) {
            $this->_bad_query($query, $fail_ok, 'Expected ' . $token . ' but got ' . (($next === null) ? '(no token)' : $next) . ' at token ' . strval($at));
            return false;
        }
        return true;
    }

    /**
     * Check we've consumed all our tokens.
     *
     * @param  integer $at Our offset counter
     * @param  array $tokens Tokens
     * @param  string $query Query that was executed
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @return boolean Success status
     */
    protected function _parsing_check_ended($at, $tokens, $query, $fail_ok = false)
    {
        do {
            $token = $this->_parsing_read($at, $tokens, $query, true);
        } while ($token === ';');
        if (!is_null($token)) {
            $this->_bad_query($query, $fail_ok, 'Extra unexpected tokens in query at token #' . strval($at + 1) . ', "' . $token . '", up to ' . implode(' ', array_slice($tokens, 0, $at)));
            return false;
        }
        return true;
    }

    /**
     * Give out an error message and die, when a query fails.
     *
     * @param  string $query The query that failed
     * @param  boolean $fail_ok Whether to not output an error on some kind of run-time failure (parse errors and clear programming errors are always fatal)
     * @param  ?string $error Error message (null: none)
     * @return ?mixed Always returns null (null: error)
     */
    protected function _bad_query($query, $fail_ok = false, $error = null)
    {
        if (!$fail_ok) {
            $msg = 'Failed on query: ' . $query;
            if (!is_null($error)) {
                $msg .= ' [' . $error . ']';
            }
            fatal_exit($msg);
        }
        return null;
    }

    /**
     * Generate a GUID for a record, preferably from the key, but doesn't have to be.
     *
     * @param  ?array $schema The schema (null: don't have/use)
     * @param  ?array $record The record (null: don't have/use)
     * @return string The GUID
     */
    protected function _guid($schema = null, $record = null)
    {
        if ((!is_null($schema)) && (!is_null($record))) {
            $guid = '';
            ksort($schema);
            $whole_key = true;
            foreach ($schema as $key => $type) {
                if (strpos($type, '*') !== false) {
                    if (array_key_exists($key, $record)) {
                        $val = $record[$key];

                        if ($guid != '') {
                            $guid .= ',';
                        }
                        $new_val = '';
                        if ((is_array($val)) && (count($val) == 1)) {
                            $val = $val[0];
                        }
                        if (is_string($val)) {
                            $new_val = $val;
                        } elseif (is_integer($val)) {
                            $new_val = strval($val);
                        } elseif (is_float($val)) {
                            $new_val = float_to_raw_string($val);
                        }
                        if ($key == 'id') {
                            $guid .= $this->_escape_name($new_val);
                        } else {
                            $guid .= $key . '=' . $this->_escape_name($new_val);
                        }
                    } else {
                        $whole_key = false;
                    }
                }
            }

            if ($whole_key) {
                return $guid;
            }
        }

        $fuzz = strtoupper(md5(uniqid(strval(mt_rand(0, mt_getrandmax())), true)));

        return '{'
               . substr($fuzz, 0, 8) . '-'
               . substr($fuzz, 8, 4) . '-'
               . substr($fuzz, 12, 4) . '-'
               . substr($fuzz, 16, 4) . '-'
               . substr($fuzz, 20, 12)
               . '}';
    }

    /**
     * Escape a value for use in a filesystem path.
     *
     * @param  string $in Value to escape (original value)
     * @return string Escaped value
     */
    protected function _escape_name($in)
    {
        return str_replace(array('=', ':', ',', '/', '|'), array('!equals!', '!colon!', '!comma!', '!slash!', '!pipe!'), $in);
    }

    /**
     * Unescape a value from a filesystem path back to the original.
     *
     * @param  string $in Escaped value
     * @return string Original value
     */
    protected function _unescape_name($in)
    {
        return str_replace(array('!equals!', '!colon!', '!comma!', '!slash!', '!pipe!'), array('=', ':', ',', '/', '|'), $in);
    }
}
