Proyectos de Subversion Moodle

<?php

// This file is part of Moodle - http://moodle.org/

//

// Moodle is free software: you can redistribute it and/or modify

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation, either version 3 of the License, or

// (at your option) any later version.

//

// Moodle is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.

/**

 * Datasets manager.

 *

 * @package   core_analytics

 * @copyright 2016 David Monllao {@link http://www.davidmonllao.com}

 * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later

 */

namespace core_analytics;

defined('MOODLE_INTERNAL') || die();

/**

 * Datasets manager.

 *

 * @package   core_analytics

 * @copyright 2016 David Monllao {@link http://www.davidmonllao.com}

 * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later

 */

class dataset_manager {

    /**

     * File area for labelled datasets.

     */

    const LABELLED_FILEAREA = 'labelled';

    /**

     * File area for unlabelled datasets.

     */

    const UNLABELLED_FILEAREA = 'unlabelled';

    /**

     * File area for exported datasets.

     */

    const EXPORT_FILEAREA = 'export';

    /**

     * Evaluation file file name.

     */

    const EVALUATION_FILENAME = 'evaluation.csv';

    /**

     * The model id.

     *

     * @var int

     */

    protected $modelid;

    /**

     * Range processor in use.

     *

     * @var string

     */

    protected $timesplittingid;

    /**

     * @var int

     */

    protected $analysableid;

    /**

     * Whether this is a dataset for evaluation or not.

     *

     * @var bool

     */

    protected $evaluation;

    /**

     * The dataset filearea. Must be one of the self::*_FILEAREA options.

     *

     * @var string

     */

    protected $filearea;

    /**

     * Constructor method.

     *

     * @throws \coding_exception

     * @param int $modelid

     * @param int $analysableid

     * @param string $timesplittingid

     * @param string $filearea

     * @param bool $evaluation

     * @return void

     */

    public function __construct($modelid, $analysableid, $timesplittingid, $filearea, $evaluation = false) {

        if ($filearea !== self::EXPORT_FILEAREA && $filearea !== self::LABELLED_FILEAREA &&

                $filearea !== self::UNLABELLED_FILEAREA) {

            throw new \coding_exception('Invalid provided filearea');

        }

        $this->modelid = $modelid;

        $this->analysableid = $analysableid;

        $this->timesplittingid = $timesplittingid;

        $this->filearea = $filearea;

        $this->evaluation = $evaluation;

    }

    /**

     * Store the dataset in the internal file system.

     *

     * @param array $data

     * @return \stored_file

     */

    public function store($data) {

        // Delete previous file if it exists.

        $fs = get_file_storage();

        $filerecord = [

            'component' => 'analytics',

            'filearea' => $this->filearea,

            'itemid' => $this->modelid,

            'contextid' => \context_system::instance()->id,

            'filepath' => '/analysable/' . $this->analysableid . '/' .

                \core_analytics\analysis::clean_time_splitting_id($this->timesplittingid) . '/',

            'filename' => self::get_filename($this->evaluation)

        ];

        // Delete previous and old (we already checked that previous copies are not recent) evaluation files for this analysable.

        if ($this->evaluation) {

            $select = " = {$filerecord['itemid']} AND filepath = :filepath";

            $fs->delete_area_files_select($filerecord['contextid'], $filerecord['component'], $filerecord['filearea'],

                $select, array('filepath' => $filerecord['filepath']));

        }

        // Write all this stuff to a tmp file.

        $filepath = make_request_directory() . DIRECTORY_SEPARATOR . $filerecord['filename'];

        $fh = fopen($filepath, 'w+');

        if (!$fh) {

            return false;

        }

        foreach ($data as $line) {

            fputcsv($fh, $line);

        }

        fclose($fh);

        return $fs->create_file_from_pathname($filerecord, $filepath);

    }

    /**

     * Returns the previous evaluation file.

     *

     * Important to note that this is per modelid + timesplittingid, when dealing with multiple

     * analysables this is the merged file. Do not confuse with self::get_evaluation_analysable_file

     *

     * @param int $modelid

     * @param string $timesplittingid

     * @return \stored_file

     */

    public static function get_previous_evaluation_file($modelid, $timesplittingid) {

        $fs = get_file_storage();

        // Evaluation data is always labelled.

        $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';

        return $fs->get_file(\context_system::instance()->id, 'analytics', self::LABELLED_FILEAREA, $modelid,

            $filepath, self::EVALUATION_FILENAME);

    }

    /**

     * Gets the list of files that couldn't be previously used for training and prediction.

     *

     * @param int $modelid

     * @param bool $includetarget

     * @param string[] $timesplittingids

     * @return null

     */

    public static function get_pending_files($modelid, $includetarget, $timesplittingids) {

        global $DB;

        $fs = get_file_storage();

        if ($includetarget) {

            $filearea = self::LABELLED_FILEAREA;

            $usedfileaction = 'trained';

        } else {

            $filearea = self::UNLABELLED_FILEAREA;

            $usedfileaction = 'predicted';

        }

        $select = 'modelid = :modelid AND action = :action';

        $params = array('modelid' => $modelid, 'action' => $usedfileaction);

        $usedfileids = $DB->get_fieldset_select('analytics_used_files', 'fileid', $select, $params);

        // Very likely that we will only have 1 time splitting method here.

        $filesbytimesplitting = array();

        foreach ($timesplittingids as $timesplittingid) {

            $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';

            $files = $fs->get_directory_files(\context_system::instance()->id, 'analytics', $filearea, $modelid, $filepath);

            foreach ($files as $file) {

                // Discard evaluation files.

                if ($file->get_filename() === self::EVALUATION_FILENAME) {

                    continue;

                }

                // No dirs.

                if ($file->is_directory()) {

                    continue;

                }

                // Already used for training.

                if (in_array($file->get_id(), $usedfileids)) {

                    continue;

                }

                $filesbytimesplitting[$timesplittingid][] = $file;

            }

        }

        return $filesbytimesplitting;

    }

    /**

     * Deletes previous evaluation files of this model.

     *

     * @param int $modelid

     * @param string $timesplittingid

     * @return bool

     */

    public static function delete_previous_evaluation_file($modelid, $timesplittingid) {

        if ($file = self::get_previous_evaluation_file($modelid, $timesplittingid)) {

            $file->delete();

            return true;

        }

        return false;

    }

    /**

     * Returns this (model + analysable + time splitting) file.

     *

     * @param int $modelid

     * @param int $analysableid

     * @param string $timesplittingid

     * @return \stored_file

     */

    public static function get_evaluation_analysable_file($modelid, $analysableid, $timesplittingid) {

        // Delete previous file if it exists.

        $fs = get_file_storage();

        // Always evaluation.csv and labelled as it is an evaluation file.

        $filearea = self::LABELLED_FILEAREA;

        $filename = self::get_filename(true);

        $filepath = '/analysable/' . $analysableid . '/' .

            \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';

        return $fs->get_file(\context_system::instance()->id, 'analytics', $filearea, $modelid, $filepath, $filename);

    }

    /**

     * Merge multiple files into one.

     *

     * Important! It is the caller responsability to ensure that the datasets are compatible.

     *

     * @param array  $files

     * @param int    $modelid

     * @param string $timesplittingid

     * @param string $filearea

     * @param bool   $evaluation

     * @return \stored_file

     */

    public static function merge_datasets(array $files, $modelid, $timesplittingid, $filearea, $evaluation = false) {

        $tmpfilepath = make_request_directory() . DIRECTORY_SEPARATOR . 'tmpfile.csv';

        // Add headers.

        // We could also do this with a single iteration gathering all files headers and appending them to the beginning of the file

        // once all file contents are merged.

        $varnames = '';

        $analysablesvalues = array();

        foreach ($files as $file) {

            $rh = $file->get_content_file_handle();

            // Copy the var names as they are, all files should have the same var names.

            $varnames = fgetcsv($rh);

            $analysablesvalues[] = fgetcsv($rh);

            // Copy the columns as they are, all files should have the same columns.

            $columns = fgetcsv($rh);

        }

        // Merge analysable values skipping the ones that are the same in all analysables.

        $values = array();

        foreach ($analysablesvalues as $analysablevalues) {

            foreach ($analysablevalues as $varkey => $value) {

                // Sha1 to make it unique.

                $values[$varkey][sha1($value)] = $value;

            }

        }

        foreach ($values as $varkey => $varvalues) {

            $values[$varkey] = implode('|', $varvalues);

        }

        // Start writing to the merge file.

        $wh = fopen($tmpfilepath, 'w');

        if (!$wh) {

            throw new \moodle_exception('errorcannotwritedataset', 'analytics', '', $tmpfilepath);

        }

        fputcsv($wh, $varnames);

        fputcsv($wh, $values);

        fputcsv($wh, $columns);

        // Iterate through all files and add them to the tmp one. We don't want file contents in memory.

        foreach ($files as $file) {

            $rh = $file->get_content_file_handle();

            // Skip headers.

            fgets($rh);

            fgets($rh);

            fgets($rh);

            // Copy all the following lines.

            while ($line = fgets($rh)) {

                fwrite($wh, $line);

            }

            fclose($rh);

        }

        fclose($wh);

        $filerecord = [

            'component' => 'analytics',

            'filearea' => $filearea,

            'itemid' => $modelid,

            'contextid' => \context_system::instance()->id,

            'filepath' => '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/',

            'filename' => self::get_filename($evaluation)

        ];

        $fs = get_file_storage();

        return $fs->create_file_from_pathname($filerecord, $tmpfilepath);

    }

    /**

     * Exports the model training data.

     *

     * @param int $modelid

     * @param string $timesplittingid

     * @return \stored_file|false

     */

    public static function export_training_data($modelid, $timesplittingid) {

        $fs = get_file_storage();

        $contextid = \context_system::instance()->id;

        $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';

        $files = $fs->get_directory_files($contextid, 'analytics', self::LABELLED_FILEAREA, $modelid,

            $filepath, true, false);

        // Discard evaluation files.

        foreach ($files as $key => $file) {

            if ($file->get_filename() === self::EVALUATION_FILENAME) {

                unset($files[$key]);

            }

        }

        if (empty($files)) {

            return false;

        }

        return self::merge_datasets($files, $modelid, $timesplittingid, self::EXPORT_FILEAREA);

    }

    /**

     * Returns the dataset file data structured by sampleids using the indicators and target column names.

     *

     * @param \stored_file $dataset

     * @return array

     */

    public static function get_structured_data(\stored_file $dataset) {

        if ($dataset->get_filearea() !== 'unlabelled') {

            throw new \coding_exception('Sorry, only support for unlabelled data');

        }

        $rh = $dataset->get_content_file_handle();

        // Skip dataset info.

        fgets($rh);

        fgets($rh);

        $calculations = array();

        $headers = fgetcsv($rh);

        // Get rid of the sampleid column name.

        array_shift($headers);

        while ($columns = fgetcsv($rh)) {

            $uniquesampleid = array_shift($columns);

            // Unfortunately fgetcsv does not respect line's var types.

            $calculations[$uniquesampleid] = array_map(function($value) {

                if ($value === '') {

                    // We really want them as null because converted to float become 0

                    // and we need to treat the values separately.

                    return null;

                } else if (is_numeric($value)) {

                    return floatval($value);

                }

                return $value;

            }, array_combine($headers, $columns));

        }

        return $calculations;

    }

    /**

     * Delete all files of a model.

     *

     * @param int $modelid

     * @return bool

     */

    public static function clear_model_files($modelid) {

        $fs = get_file_storage();

        return $fs->delete_area_files(\context_system::instance()->id, 'analytics', false, $modelid);

    }

    /**

     * Returns the file name to be used.

     *

     * @param strinbool $evaluation

     * @return string

     */

    protected static function get_filename($evaluation) {

        if ($evaluation === true) {

            $filename = self::EVALUATION_FILENAME;

        } else {

            // Incremental time, the lock will make sure we don't have concurrency problems.

            $filename = microtime(true) . '.csv';

        }

        return $filename;

    }

}