Proyectos de Subversion Moodle

<?php

// This file is part of Moodle - http://moodle.org/

//

// Moodle is free software: you can redistribute it and/or modify

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation, either version 3 of the License, or

// (at your option) any later version.

//

// Moodle is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.

/**

 * Solr engine.

 *

 * @package    search_solr

 * @copyright  2015 Daniel Neis Araujo

 * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later

 */

namespace search_solr;

defined('MOODLE_INTERNAL') || die();

/**

 * Solr engine.

 *

 * @package    search_solr

 * @copyright  2015 Daniel Neis Araujo

 * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later

 */

class engine extends \core_search\engine {

    /**

     * @var string The date format used by solr.

     */

    const DATE_FORMAT = 'Y-m-d\TH:i:s\Z';

    /**

     * @var int Commit documents interval (number of miliseconds).

     */

    const AUTOCOMMIT_WITHIN = 15000;

    /**

     * The maximum number of results to fetch at a time.

     */

    const QUERY_SIZE = 120;

    /**

     * Highlighting fragsize. Slightly larger than output size (500) to allow for ... appending.

     */

    const FRAG_SIZE = 510;

    /**

     * Marker for the start of a highlight.

     */

    const HIGHLIGHT_START = '@@HI_S@@';

    /**

     * Marker for the end of a highlight.

     */

    const HIGHLIGHT_END = '@@HI_E@@';

    /** @var float Boost value for matching course in location-ordered searches */

    const COURSE_BOOST = 1;

    /** @var float Boost value for matching context (in addition to course boost) */

    const CONTEXT_BOOST = 0.5;

    /**

     * @var \SolrClient

     */

    protected $client = null;

    /**

     * @var bool True if we should reuse SolrClients, false if not.

     */

    protected $cacheclient = true;

    /**

     * @var \curl Direct curl object.

     */

    protected $curl = null;

    /**

     * @var array Fields that can be highlighted.

     */

    protected $highlightfields = array('title', 'content', 'description1', 'description2');

    /**

     * @var int Number of total docs reported by Sorl for the last query.

     */

    protected $totalenginedocs = 0;

    /**

     * @var int Number of docs we have processed for the last query.

     */

    protected $processeddocs = 0;

    /**

     * @var int Number of docs that have been skipped while processing the last query.

     */

    protected $skippeddocs = 0;

    /**

     * Solr server major version.

     *

     * @var int

     */

    protected $solrmajorversion = null;

    /**

     * Initialises the search engine configuration.

     *

     * @param bool $alternateconfiguration If true, use alternate configuration settings

     * @return void

     */

    public function __construct(bool $alternateconfiguration = false) {

        parent::__construct($alternateconfiguration);

        $curlversion = curl_version();

        if (isset($curlversion['version']) && stripos($curlversion['version'], '7.35.') === 0) {

            // There is a flaw with curl 7.35.0 that causes problems with client reuse.

            $this->cacheclient = false;

        }

    }

    /**

     * Prepares a Solr query, applies filters and executes it returning its results.

     *

     * @throws \core_search\engine_exception

     * @param  \stdClass $filters Containing query and filters.

     * @param  \stdClass $accessinfo Information about areas user can access.

     * @param  int       $limit The maximum number of results to return.

     * @return \core_search\document[] Results or false if no results

     */

    public function execute_query($filters, $accessinfo, $limit = 0) {

        global $USER;

        if (empty($limit)) {

            $limit = \core_search\manager::MAX_RESULTS;

        }

        // If there is any problem we trigger the exception as soon as possible.

        $client = $this->get_search_client();

        // Create the query object.

        $query = $this->create_user_query($filters, $accessinfo);

        // If the query cannot have results, return none.

        if (!$query) {

            return [];

        }

        // We expect good match rates, so for our first get, we will get a small number of records.

        // This significantly speeds solr response time for first few pages.

        $query->setRows(min($limit * 3, static::QUERY_SIZE));

        $response = $this->get_query_response($query);

        // Get count data out of the response, and reset our counters.

        list($included, $found) = $this->get_response_counts($response);

        $this->totalenginedocs = $found;

        $this->processeddocs = 0;

        $this->skippeddocs = 0;

        if ($included == 0 || $this->totalenginedocs == 0) {

            // No results.

            return array();

        }

        // Get valid documents out of the response.

        $results = $this->process_response($response, $limit);

        // We have processed all the docs in the response at this point.

        $this->processeddocs += $included;

        // If we haven't reached the limit, and there are more docs left in Solr, lets keep trying.

        while (count($results) < $limit && ($this->totalenginedocs - $this->processeddocs) > 0) {

            // Offset the start of the query, and since we are making another call, get more per call.

            $query->setStart($this->processeddocs);

            $query->setRows(static::QUERY_SIZE);

            $response = $this->get_query_response($query);

            list($included, $found) = $this->get_response_counts($response);

            if ($included == 0 || $found == 0) {

                // No new results were found. Found being empty would be weird, so we will just return.

                return $results;

            }

            $this->totalenginedocs = $found;

            // Get the new response docs, limiting to remaining we need, then add it to the end of the results array.

            $newdocs = $this->process_response($response, $limit - count($results));

            $results = array_merge($results, $newdocs);

            // Add to our processed docs count.

            $this->processeddocs += $included;

        }

        return $results;

    }

    /**

     * Takes a query and returns the response in SolrObject format.

     *

     * @param  SolrQuery  $query Solr query object.

     * @return SolrObject|false Response document or false on error.

     */

    protected function get_query_response($query) {

        try {

            return $this->get_search_client()->query($query)->getResponse();

        } catch (\SolrClientException $ex) {

            debugging('Error executing the provided query: ' . $ex->getMessage(), DEBUG_DEVELOPER);

            $this->queryerror = $ex->getMessage();

            return false;

        } catch (\SolrServerException $ex) {

            debugging('Error executing the provided query: ' . $ex->getMessage(), DEBUG_DEVELOPER);

            $this->queryerror = $ex->getMessage();

            return false;

        }

    }

    /**

     * Returns the total number of documents available for the most recently call to execute_query.

     *

     * @return int

     */

    public function get_query_total_count() {

        // Return the total engine count minus the docs we have determined are bad.

        return $this->totalenginedocs - $this->skippeddocs;

    }

    /**

     * Returns count information for a provided response. Will return 0, 0 for invalid or empty responses.

     *

     * @param SolrDocument $response The response document from Solr.

     * @return array A two part array. First how many response docs are in the response.

     *               Second, how many results are vailable in the engine.

     */

    protected function get_response_counts($response) {

        $found = 0;

        $included = 0;

        if (isset($response->grouped->solr_filegroupingid->ngroups)) {

            // Get the number of results for file grouped queries.

            $found = $response->grouped->solr_filegroupingid->ngroups;

            $included = count($response->grouped->solr_filegroupingid->groups);

        } else if (isset($response->response->numFound)) {

            // Get the number of results for standard queries.

            $found = $response->response->numFound;

            if ($found > 0 && is_array($response->response->docs)) {

                $included = count($response->response->docs);

            }

        }

        return array($included, $found);

    }

    /**

     * Prepares a new query object with needed limits, filters, etc.

     *

     * @param \stdClass $filters Containing query and filters.

     * @param \stdClass $accessinfo Information about contexts the user can access

     * @return \SolrDisMaxQuery|null Query object or null if they can't get any results

     */

    protected function create_user_query($filters, $accessinfo) {

        global $USER;

        // Let's keep these changes internal.

        $data = clone $filters;

        $query = new \SolrDisMaxQuery();

        $this->set_query($query, self::replace_underlines($data->q));

        $this->add_fields($query);

        // Search filters applied, we don't cache these filters as we don't want to pollute the cache with tmp filters

        // we are really interested in caching contexts filters instead.

        if (!empty($data->title)) {

            $query->addFilterQuery('{!field cache=false f=title}' . $data->title);

        }

        if (!empty($data->areaids)) {

            // If areaids are specified, we want to get any that match.

            $query->addFilterQuery('{!cache=false}areaid:(' . implode(' OR ', $data->areaids) . ')');

        }

        if (!empty($data->courseids)) {

            $query->addFilterQuery('{!cache=false}courseid:(' . implode(' OR ', $data->courseids) . ')');

        }

        if (!empty($data->groupids)) {

            $query->addFilterQuery('{!cache=false}groupid:(' . implode(' OR ', $data->groupids) . ')');

        }

        if (!empty($data->userids)) {

            $query->addFilterQuery('{!cache=false}userid:(' . implode(' OR ', $data->userids) . ')');

        }

        if (!empty($data->timestart) or !empty($data->timeend)) {

            if (empty($data->timestart)) {

                $data->timestart = '*';

            } else {

                $data->timestart = \search_solr\document::format_time_for_engine($data->timestart);

            }

            if (empty($data->timeend)) {

                $data->timeend = '*';

            } else {

                $data->timeend = \search_solr\document::format_time_for_engine($data->timeend);

            }

            // No cache.

            $query->addFilterQuery('{!cache=false}modified:[' . $data->timestart . ' TO ' . $data->timeend . ']');

        }

        // Restrict to users who are supposed to be able to see a particular result.

        $query->addFilterQuery('owneruserid:(' . \core_search\manager::NO_OWNER_ID . ' OR ' . $USER->id . ')');

        // And finally restrict it to the context where the user can access, we want this one cached.

        // If the user can access all contexts $usercontexts value is just true, we don't need to filter

        // in that case.

        if (!$accessinfo->everything && is_array($accessinfo->usercontexts)) {

            // Join all area contexts into a single array and implode.

            $allcontexts = array();

            foreach ($accessinfo->usercontexts as $areaid => $areacontexts) {

                if (!empty($data->areaids) && !in_array($areaid, $data->areaids)) {

                    // Skip unused areas.

                    continue;

                }

                foreach ($areacontexts as $contextid) {

                    // Ensure they are unique.

                    $allcontexts[$contextid] = $contextid;

                }

            }

            if (empty($allcontexts)) {

                // This means there are no valid contexts for them, so they get no results.

                return null;

            }

            $query->addFilterQuery('contextid:(' . implode(' OR ', $allcontexts) . ')');

        }

        if (!$accessinfo->everything && $accessinfo->separategroupscontexts) {

            // Add another restriction to handle group ids. If there are any contexts using separate

            // groups, then results in that context will not show unless you belong to the group.

            // (Note: Access all groups is taken care of earlier, when computing these arrays.)

            // This special exceptions list allows for particularly pig-headed developers to create

            // multiple search areas within the same module, where one of them uses separate

            // groups and the other uses visible groups. It is a little inefficient, but this should

            // be rare.

            $exceptions = '';

            if ($accessinfo->visiblegroupscontextsareas) {

                foreach ($accessinfo->visiblegroupscontextsareas as $contextid => $areaids) {

                    $exceptions .= ' OR (contextid:' . $contextid . ' AND areaid:(' .

                            implode(' OR ', $areaids) . '))';

                }

            }

            if ($accessinfo->usergroups) {

                // Either the document has no groupid, or the groupid is one that the user

                // belongs to, or the context is not one of the separate groups contexts.

                $query->addFilterQuery('(*:* -groupid:[* TO *]) OR ' .

                        'groupid:(' . implode(' OR ', $accessinfo->usergroups) . ') OR ' .

                        '(*:* -contextid:(' . implode(' OR ', $accessinfo->separategroupscontexts) . '))' .

                        $exceptions);

            } else {

                // Either the document has no groupid, or the context is not a restricted one.

                $query->addFilterQuery('(*:* -groupid:[* TO *]) OR ' .

                        '(*:* -contextid:(' . implode(' OR ', $accessinfo->separategroupscontexts) . '))' .

                        $exceptions);

            }

        }

        if ($this->file_indexing_enabled()) {

            // Now group records by solr_filegroupingid. Limit to 3 results per group.

            $query->setGroup(true);

            $query->setGroupLimit(3);

            $query->setGroupNGroups(true);

            $query->addGroupField('solr_filegroupingid');

        } else {

            // Make sure we only get text files, in case the index has pre-existing files.

            $query->addFilterQuery('type:'.\core_search\manager::TYPE_TEXT);

        }

        // If ordering by location, add in boost for the relevant course or context ids.

        if (!empty($filters->order) && $filters->order === 'location') {

            $coursecontext = $filters->context->get_course_context();

            $query->addBoostQuery('courseid', $coursecontext->instanceid, self::COURSE_BOOST);

            if ($filters->context->contextlevel !== CONTEXT_COURSE) {

                // If it's a block or activity, also add a boost for the specific context id.

                $query->addBoostQuery('contextid', $filters->context->id, self::CONTEXT_BOOST);

            }

        }

        return $query;

    }

    /**

     * Prepares a new query by setting the query, start offset and rows to return.

     *

     * @param SolrQuery $query

     * @param object    $q Containing query and filters.

     */

    protected function set_query($query, $q) {

        // Set hightlighting.

        $query->setHighlight(true);

        foreach ($this->highlightfields as $field) {

            $query->addHighlightField($field);

        }

        $query->setHighlightFragsize(static::FRAG_SIZE);

        $query->setHighlightSimplePre(self::HIGHLIGHT_START);

        $query->setHighlightSimplePost(self::HIGHLIGHT_END);

        $query->setHighlightMergeContiguous(true);

        $query->setQuery($q);

        // A reasonable max.

        $query->setRows(static::QUERY_SIZE);

    }

    /**

     * Sets fields to be returned in the result.

     *

     * @param SolrDisMaxQuery|SolrQuery $query object.

     */

    public function add_fields($query) {

        $documentclass = $this->get_document_classname();

        $fields = $documentclass::get_default_fields_definition();

        $dismax = false;

        if ($query instanceof \SolrDisMaxQuery) {

            $dismax = true;

        }

        foreach ($fields as $key => $field) {

            $query->addField($key);

            if ($dismax && !empty($field['mainquery'])) {

                // Add fields the main query should be run against.

                // Due to a regression in the PECL solr extension, https://bugs.php.net/bug.php?id=72740,

                // a boost value is required, even if it is optional; to avoid boosting one among other fields,

                // the explicit boost value will be the default one, for every field.

                $query->addQueryField($key, 1);

            }

        }

    }

    /**

     * Finds the key common to both highlighing and docs array returned from response.

     * @param object $response containing results.

     */

    public function add_highlight_content($response) {

        if (!isset($response->highlighting)) {

            // There is no highlighting to add.

            return;

        }

        $highlightedobject = $response->highlighting;

        foreach ($response->response->docs as $doc) {

            $x = $doc->id;

            $highlighteddoc = $highlightedobject->$x;

            $this->merge_highlight_field_values($doc, $highlighteddoc);

        }

    }

    /**

     * Adds the highlighting array values to docs array values.

     *

     * @throws \core_search\engine_exception

     * @param object $doc containing the results.

     * @param object $highlighteddoc containing the highlighted results values.

     */

    public function merge_highlight_field_values($doc, $highlighteddoc) {

        foreach ($this->highlightfields as $field) {

            if (!empty($doc->$field)) {

                // Check that the returned value is not an array. No way we can make this work with multivalued solr fields.

                if (is_array($doc->{$field})) {

                    throw new \core_search\engine_exception('multivaluedfield', 'search_solr', '', $field);

                }

                if (!empty($highlighteddoc->$field)) {

                    // Replace by the highlighted result.

                    $doc->$field = reset($highlighteddoc->$field);

                }

            }

        }

    }

    /**

     * Filters the response on Moodle side.

     *

     * @param SolrObject $response Solr object containing the response return from solr server.

     * @param int        $limit The maximum number of results to return. 0 for all.

     * @param bool       $skipaccesscheck Don't use check_access() on results. Only to be used when results have known access.

     * @return array $results containing final results to be displayed.

     */

    protected function process_response($response, $limit = 0, $skipaccesscheck = false) {

        global $USER;

        if (empty($response)) {

            return array();

        }

        if (isset($response->grouped)) {

            return $this->grouped_files_process_response($response, $limit);

        }

        $userid = $USER->id;

        $noownerid = \core_search\manager::NO_OWNER_ID;

        $numgranted = 0;

        if (!$docs = $response->response->docs) {

            return array();

        }

        $out = array();

        if (!empty($response->response->numFound)) {

            $this->add_highlight_content($response);

            // Iterate through the results checking its availability and whether they are available for the user or not.

            foreach ($docs as $key => $docdata) {

                if ($docdata['owneruserid'] != $noownerid && $docdata['owneruserid'] != $userid) {

                    // If owneruserid is set, no other user should be able to access this record.

                    continue;

                }

                if (!$searcharea = $this->get_search_area($docdata->areaid)) {

                    continue;

                }

                $docdata = $this->standarize_solr_obj($docdata);

                if ($skipaccesscheck) {

                    $access = \core_search\manager::ACCESS_GRANTED;

                } else {

                    $access = $searcharea->check_access($docdata['itemid']);

                }

                switch ($access) {

                    case \core_search\manager::ACCESS_DELETED:

                        $this->delete_by_id($docdata['id']);

                        // Remove one from our processed and total counters, since we promptly deleted.

                        $this->processeddocs--;

                        $this->totalenginedocs--;

                        break;

                    case \core_search\manager::ACCESS_DENIED:

                        $this->skippeddocs++;

                        break;

                    case \core_search\manager::ACCESS_GRANTED:

                        $numgranted++;

                        // Add the doc.

                        $out[] = $this->to_document($searcharea, $docdata);

                        break;

                }

                // Stop when we hit our limit.

                if (!empty($limit) && count($out) >= $limit) {

                    break;

                }

            }

        }

        return $out;

    }

    /**

     * Processes grouped file results into documents, with attached matching files.

     *

     * @param SolrObject $response The response returned from solr server

     * @param int        $limit The maximum number of results to return. 0 for all.

     * @return array Final results to be displayed.

     */

    protected function grouped_files_process_response($response, $limit = 0) {

        // If we can't find the grouping, or there are no matches in the grouping, return empty.

        if (!isset($response->grouped->solr_filegroupingid) || empty($response->grouped->solr_filegroupingid->matches)) {

            return array();

        }

        $numgranted = 0;

        $orderedids = array();

        $completedocs = array();

        $incompletedocs = array();

        $highlightingobj = $response->highlighting;

        // Each group represents a "master document".

        $groups = $response->grouped->solr_filegroupingid->groups;

        foreach ($groups as $group) {

            $groupid = $group->groupValue;

            $groupdocs = $group->doclist->docs;

            $firstdoc = reset($groupdocs);

            if (!$searcharea = $this->get_search_area($firstdoc->areaid)) {

                // Well, this is a problem.

                continue;

            }

            // Check for access.

            $access = $searcharea->check_access($firstdoc->itemid);

            switch ($access) {

                case \core_search\manager::ACCESS_DELETED:

                    // If deleted from Moodle, delete from index and then continue.

                    $this->delete_by_id($firstdoc->id);

                    // Remove one from our processed and total counters, since we promptly deleted.

                    $this->processeddocs--;

                    $this->totalenginedocs--;

                    continue 2;

                    break;

                case \core_search\manager::ACCESS_DENIED:

                    // This means we should just skip for the current user.

                    $this->skippeddocs++;

                    continue 2;

                    break;

            }

            $numgranted++;

            $maindoc = false;

            $fileids = array();

            // Seperate the main document and any files returned.

            foreach ($groupdocs as $groupdoc) {

                if ($groupdoc->id == $groupid) {

                    $maindoc = $groupdoc;

                } else if (isset($groupdoc->solr_fileid)) {

                    $fileids[] = $groupdoc->solr_fileid;

                }

            }

            // Store the id of this group, in order, for later merging.

            $orderedids[] = $groupid;

            if (!$maindoc) {

                // We don't have the main doc, store what we know for later building.

                $incompletedocs[$groupid] = $fileids;

            } else {

                if (isset($highlightingobj->$groupid)) {

                    // Merge the highlighting for this doc.

                    $this->merge_highlight_field_values($maindoc, $highlightingobj->$groupid);

                }

                $docdata = $this->standarize_solr_obj($maindoc);

                $doc = $this->to_document($searcharea, $docdata);

                // Now we need to attach the result files to the doc.

                foreach ($fileids as $fileid) {

                    $doc->add_stored_file($fileid);

                }

                $completedocs[$groupid] = $doc;

            }

            if (!empty($limit) && $numgranted >= $limit) {

                // We have hit the max results, we will just ignore the rest.

                break;

            }

        }

        $incompletedocs = $this->get_missing_docs($incompletedocs);

        $out = array();

        // Now merge the complete and incomplete documents, in results order.

        foreach ($orderedids as $docid) {

            if (isset($completedocs[$docid])) {

                $out[] = $completedocs[$docid];

            } else if (isset($incompletedocs[$docid])) {

                $out[] = $incompletedocs[$docid];

            }

        }

        return $out;

    }

    /**

     * Retreive any missing main documents and attach provided files.

     *

     * The missingdocs array should be an array, indexed by document id, of main documents we need to retrieve. The value

     * associated to the key should be an array of stored_files or stored file ids to attach to the result document.

     *

     * Return array also indexed by document id.

     *

     * @param array() $missingdocs An array, indexed by document id, with arrays of files/ids to attach.

     * @return document[]

     */

    protected function get_missing_docs($missingdocs) {

        if (empty($missingdocs)) {

            return array();

        }

        $docids = array_keys($missingdocs);

        // Build a custom query that will get all the missing documents.

        $query = new \SolrQuery();

        $this->set_query($query, '*');

        $this->add_fields($query);

        $query->setRows(count($docids));

        $query->addFilterQuery('{!cache=false}id:(' . implode(' OR ', $docids) . ')');

        $response = $this->get_query_response($query);

        // We know the missing docs have already been checked for access, so don't recheck.

        $results = $this->process_response($response, 0, true);

        $out = array();

        foreach ($results as $result) {

            $resultid = $result->get('id');

            if (!isset($missingdocs[$resultid])) {

                // We got a result we didn't expect. Skip it.

                continue;

            }

            // Attach the files.

            foreach ($missingdocs[$resultid] as $filedoc) {

                $result->add_stored_file($filedoc);

            }

            $out[$resultid] = $result;

        }

        return $out;

    }

    /**

     * Returns a standard php array from a \SolrObject instance.

     *

     * @param \SolrObject $obj

     * @return array The returned document as an array.

     */

    public function standarize_solr_obj(\SolrObject $obj) {

        $properties = $obj->getPropertyNames();

        $docdata = array();

        foreach($properties as $name) {

            // http://php.net/manual/en/solrobject.getpropertynames.php#98018.

            $name = trim($name);

            $docdata[$name] = $obj->offsetGet($name);

        }

        return $docdata;

    }

    /**

     * Adds a document to the search engine.

     *

     * This does not commit to the search engine.

     *

     * @param document $document

     * @param bool     $fileindexing True if file indexing is to be used

     * @return bool

     */

    public function add_document($document, $fileindexing = false) {

        $docdata = $document->export_for_engine();

        if (!$this->add_solr_document($docdata)) {

            return false;

        }

        if ($fileindexing) {

            // This will take care of updating all attached files in the index.

            $this->process_document_files($document);

        }

        return true;

    }

    /**

     * Adds a batch of documents to the engine at once.

     *

     * @param \core_search\document[] $documents Documents to add

     * @param bool $fileindexing If true, indexes files (these are done one at a time)

     * @return int[] Array of three elements: successfully processed, failed processed, batch count

     */

    public function add_document_batch(array $documents, bool $fileindexing = false): array {

        $docdatabatch = [];

        foreach ($documents as $document) {

            $docdatabatch[] = $document->export_for_engine();

        }

        $resultcounts = $this->add_solr_documents($docdatabatch);

        // Files are processed one document at a time (if there are files it's slow anyway).

        if ($fileindexing) {

            foreach ($documents as $document) {

                // This will take care of updating all attached files in the index.

                $this->process_document_files($document);

            }

        }

        return $resultcounts;

    }

    /**

     * Replaces underlines at edges of words in the content with spaces.

     *

     * For example '_frogs_' will become 'frogs', '_frogs and toads_' will become 'frogs and toads',

     * and 'frogs_and_toads' will be left as 'frogs_and_toads'.

     *

     * The reason for this is that for italic content_to_text puts _italic_ underlines at the start

     * and end of the italicised phrase (not between words). Solr treats underlines as part of the

     * word, which means that if you search for a word in italic then you can't find it.

     *

     * @param string $str String to replace

     * @return string Replaced string

     */

    protected static function replace_underlines(string $str): string {

        return preg_replace('~\b_|_\b~', '', $str);

    }

    /**

     * Creates a Solr document object.

     *

     * @param array $doc Array of document fields

     * @return \SolrInputDocument Created document

     */

    protected function create_solr_document(array $doc): \SolrInputDocument {

        $solrdoc = new \SolrInputDocument();

        // Replace underlines in the content with spaces. The reason for this is that for italic

        // text, content_to_text puts _italic_ underlines. Solr treats underlines as part of the

        // word, which means that if you search for a word in italic then you can't find it.

        if (array_key_exists('content', $doc)) {

            $doc['content'] = self::replace_underlines($doc['content']);

        }

        // Set all the fields.

        foreach ($doc as $field => $value) {

            $solrdoc->addField($field, $value);

        }

        return $solrdoc;

    }

    /**

     * Adds a text document to the search engine.

     *

     * @param array $doc

     * @return bool

     */

    protected function add_solr_document($doc) {

        $solrdoc = $this->create_solr_document($doc);

        try {

            $result = $this->get_search_client()->addDocument($solrdoc, true, static::AUTOCOMMIT_WITHIN);

            return true;

        } catch (\SolrClientException $e) {

            debugging('Solr client error adding document with id ' . $doc['id'] . ': ' . $e->getMessage(), DEBUG_DEVELOPER);

        } catch (\SolrServerException $e) {

            // We only use the first line of the message, as it's a fully java stacktrace behind it.

            $msg = strtok($e->getMessage(), "\n");

            debugging('Solr server error adding document with id ' . $doc['id'] . ': ' . $msg, DEBUG_DEVELOPER);

        }

        return false;

    }

    /**

     * Adds multiple text documents to the search engine.

     *

     * @param array $docs Array of documents (each an array of fields) to add

     * @return int[] Array of success, failure, batch count

     * @throws \core_search\engine_exception

     */

    protected function add_solr_documents(array $docs): array {

        $solrdocs = [];

        foreach ($docs as $doc) {

            $solrdocs[] = $this->create_solr_document($doc);

        }

        try {

            // Add documents in a batch and report that they all succeeded.

            $this->get_search_client()->addDocuments($solrdocs, true, static::AUTOCOMMIT_WITHIN);

            return [count($solrdocs), 0, 1];

        } catch (\SolrClientException $e) {

            // If there is an exception, fall through...

            $donothing = true;

        } catch (\SolrServerException $e) {

            // If there is an exception, fall through...

            $donothing = true;

        }

        // When there is an error, we fall back to adding them individually so that we can report

        // which document(s) failed. Since it overwrites, adding the successful ones multiple

        // times won't hurt.

        $success = 0;

        $failure = 0;

        $batches = 0;

        foreach ($docs as $doc) {

            $result = $this->add_solr_document($doc);

            $batches++;

            if ($result) {

                $success++;

            } else {

                $failure++;

            }

        }

        return [$success, $failure, $batches];

    }

    /**

     * Index files attached to the docuemnt, ensuring the index matches the current document files.

     *

     * For documents that aren't known to be new, we check the index for existing files.

     * - New files we will add.

     * - Existing and unchanged files we will skip.

     * - File that are in the index but not on the document will be deleted from the index.

     * - Files that have changed will be re-indexed.

     *

     * @param document $document

     */

    protected function process_document_files($document) {

        if (!$this->file_indexing_enabled()) {

            return;

        }

        // Maximum rows to process at a time.

        $rows = 500;

        // Get the attached files.

        $files = $document->get_files();

        // If this isn't a new document, we need to check the exiting indexed files.

        if (!$document->get_is_new()) {

            // We do this progressively, so we can handle lots of files cleanly.

            list($numfound, $indexedfiles) = $this->get_indexed_files($document, 0, $rows);

            $count = 0;

            $idstodelete = array();

            do {

                // Go through each indexed file. We want to not index any stored and unchanged ones, delete any missing ones.

                foreach ($indexedfiles as $indexedfile) {

                    $fileid = $indexedfile->solr_fileid;

                    if (isset($files[$fileid])) {

                        // Check for changes that would mean we need to re-index the file. If so, just leave in $files.

                        // Filelib does not guarantee time modified is updated, so we will check important values.

                        if ($indexedfile->modified != $files[$fileid]->get_timemodified()) {

                            continue;

                        }

                        if (strcmp($indexedfile->title, $files[$fileid]->get_filename()) !== 0) {

                            continue;

                        }

                        if ($indexedfile->solr_filecontenthash != $files[$fileid]->get_contenthash()) {

                            continue;

                        }

                        if ($indexedfile->solr_fileindexstatus == document::INDEXED_FILE_FALSE &&

                                $this->file_is_indexable($files[$fileid])) {

                            // This means that the last time we indexed this file, filtering blocked it.

                            // Current settings say it is indexable, so we will allow it to be indexed.

                            continue;

                        }

                        // If the file is already indexed, we can just remove it from the files array and skip it.

                        unset($files[$fileid]);

                    } else {

                        // This means we have found a file that is no longer attached, so we need to delete from the index.

                        // We do it later, since this is progressive, and it could reorder results.

                        $idstodelete[] = $indexedfile->id;

                    }

                }

                $count += $rows;

                if ($count < $numfound) {

                    // If we haven't hit the total count yet, fetch the next batch.

                    list($numfound, $indexedfiles) = $this->get_indexed_files($document, $count, $rows);

                }

            } while ($count < $numfound);

            // Delete files that are no longer attached.

            foreach ($idstodelete as $id) {

                // We directly delete the item using the client, as the engine delete_by_id won't work on file docs.

                $this->get_search_client()->deleteById($id);

            }

        }

        // Now we can actually index all the remaining files.

        foreach ($files as $file) {

            $this->add_stored_file($document, $file);

        }

    }

    /**

     * Get the currently indexed files for a particular document, returns the total count, and a subset of files.

     *

     * @param document $document

     * @param int      $start The row to start the results on. Zero indexed.

     * @param int      $rows The number of rows to fetch

     * @return array   A two element array, the first is the total number of availble results, the second is an array

     *                 of documents for the current request.

     */

    protected function get_indexed_files($document, $start = 0, $rows = 500) {

        // Build a custom query that will get any document files that are in our solr_filegroupingid.

        $query = new \SolrQuery();

        // We want to get all file records tied to a document.

        // For efficiency, we are building our own, stripped down, query.

        $query->setQuery('*');

        $query->setRows($rows);

        $query->setStart($start);

        // We want a consistent sorting.

        $query->addSortField('id');

        // We only want the bare minimum of fields.

        $query->addField('id');

        $query->addField('modified');

        $query->addField('title');

        $query->addField('solr_fileid');

        $query->addField('solr_filecontenthash');

        $query->addField('solr_fileindexstatus');

        $query->addFilterQuery('{!cache=false}solr_filegroupingid:(' . $document->get('id') . ')');

        $query->addFilterQuery('type:' . \core_search\manager::TYPE_FILE);