public function test_fix_utf8()
 {
     // Make sure valid data including other types is not changed.
     $this->assertSame(null, fix_utf8(null));
     $this->assertSame(1, fix_utf8(1));
     $this->assertSame(1.1, fix_utf8(1.1));
     $this->assertSame(true, fix_utf8(true));
     $this->assertSame('', fix_utf8(''));
     $this->assertSame('abc', fix_utf8('abc'));
     $array = array('do', 're', 'mi');
     $this->assertSame($array, fix_utf8($array));
     $object = new stdClass();
     $object->a = 'aa';
     $object->b = 'bb';
     $this->assertEquals($object, fix_utf8($object));
     // valid utf8 string
     $this->assertSame("žlutý koníček přeskočil potůček \n\t\r", fix_utf8("žlutý koníček přeskočil potůček \n\t\r"));
     // Invalid utf8 string.
     $this->assertSame('aš', fix_utf8('a' . chr(130) . 'š'), 'This fails with buggy iconv() when mbstring extenstion is not available as fallback.');
 }

/**
 * Makes sure the data is using valid utf8, invalid characters are discarded.
 *
 * Note: this function is not intended for full objects with methods and private properties.
 *
 * @param mixed $value
 * @return mixed with proper utf-8 encoding
 */
function fix_utf8($value)
{
    if (is_null($value) or $value === '') {
        return $value;
    } else {
        if (is_string($value)) {
            if ((string) (int) $value === $value) {
                // Shortcut.
                return $value;
            }
            // No null bytes expected in our data, so let's remove it.
            $value = str_replace("", '', $value);
            // Note: this duplicates min_fix_utf8() intentionally.
            static $buggyiconv = null;
            if ($buggyiconv === null) {
                $buggyiconv = (!function_exists('iconv') or @iconv('UTF-8', 'UTF-8//IGNORE', '100' . chr(130) . '€') !== '100€');
            }
            if ($buggyiconv) {
                if (function_exists('mb_convert_encoding')) {
                    $subst = mb_substitute_character();
                    mb_substitute_character('');
                    $result = mb_convert_encoding($value, 'utf-8', 'utf-8');
                    mb_substitute_character($subst);
                } else {
                    // Warn admins on admin/index.php page.
                    $result = $value;
                }
            } else {
                $result = @iconv('UTF-8', 'UTF-8//IGNORE', $value);
            }
            return $result;
        } else {
            if (is_array($value)) {
                foreach ($value as $k => $v) {
                    $value[$k] = fix_utf8($v);
                }
                return $value;
            } else {
                if (is_object($value)) {
                    // Do not modify original.
                    $value = clone $value;
                    foreach ($value as $k => $v) {
                        $value->{$k} = fix_utf8($v);
                    }
                    return $value;
                } else {
                    // This is some other type, no utf-8 here.
                    return $value;
                }
            }
        }
    }
}

function FetchSaveMovie($id, $lookup)
{
    $debug = 0;
    $video = runSQL('SELECT * FROM ' . TBL_DATA . ' WHERE id = ' . $id);
    // get fields (according to list) from db to be saved later
    if ($debug) {
        echo "\n=================== Video DB Data ============================\n";
        print_r($video[0]);
        echo "\n=================== Video DB Data ============================\n";
    }
    $imdbID = $video[0]['imdbID'];
    echo "Movie/imdb -- " . $video[0]['title'] . "/" . $video[0]['imdbID'] . "\n";
    if (empty($imdbID)) {
        echo "No imdbID\n";
        return;
    }
    if (empty($engine)) {
        $engine = engineGetEngine($imdbID);
    }
    if ($debug) {
        echo "IMDBID = {$imdbID}, engine = {$engine}\n";
    }
    $imdbdata = engineGetData($imdbID, $engine);
    # removed due to performance issues of is_utf8
    // fix erroneous IMDB encoding issues
    if (!is_utf8($imdbdata)) {
        echo "Applying encoding fix\n";
        $imdbdata = fix_utf8($imdbdata);
    }
    if (empty($imdbdata[title])) {
        echo "Fetch failed , try again...\n";
        $imdbdata = engineGetData($imdbID, $engine);
    }
    if (empty($imdbdata[title])) {
        echo "Fetch failed again , next movie";
        return;
    }
    if ($debug) {
        echo "\n===================  IMDB Data ============================\n";
        print_r($imdbdata);
        echo "\n===================  IMDB Data ============================\n";
    }
    if (!empty($imdbdata[title])) {
        //
        // NOTE: comment out any of the following lines if you do not want them updated
        //
        $video[0][title] = $imdbdata[title];
        $video[0][subtitle] = $imdbdata[subtitle];
        $video[0][year] = $imdbdata[year];
        $video[0][imgurl] = $imdbdata[coverurl];
        $video[0][runtime] = $imdbdata[runtime];
        $video[0][director] = $imdbdata[director];
        $video[0][rating] = $imdbdata[rating];
        $video[0][country] = $imdbdata[country];
        $video[0][language] = $imdbdata[language];
        $video[0][actors] = $imdbdata[cast];
        $video[0][plot] = $imdbdata[plot];
    }
    if (count($genres) == 0 || $lookup > 1) {
        $genres = array();
        $gnames = $imdbdata['genres'];
        if (isset($gnames)) {
            foreach ($gnames as $gname) {
                // check if genre is found- otherwise fail silently
                if (is_numeric($genre = getGenreId($gname))) {
                    $genres[] = $genre;
                } else {
                    echo "MISSING GENRE {$gname}\n";
                }
            }
        }
    }
    // custom filds , not working for now
    for ($i = 1; $i <= 4; $i++) {
        $custom = 'custom' . $i;
        $type = $config[$custom . 'type'];
        if (!empty($type)) {
            // copy imdb data into corresponding custom field
            $video[0][$custom] = $imdbdata[$type];
            echo "CUSTOM {$custom} {$type} = {$imdbdata[$type]}\n";
        }
    }
    //  -------- SAVE
    $SETS = prepareSQL($video[0]);
    if ($debug) {
        echo "\n===================  Final Data ============================\n";
        echo "SETS = {$SETS} \n";
        echo "\n===================  Final Data ============================\n";
    }
    $id = updateDB($SETS, $id);
    // save genres
    setItemGenres($id, $genres);
    // set seen for currently logged in user
    set_userseen($id, $seen);
}

 /**
  * Add unicode flag to all files in archive.
  *
  * NOTE: single disk archives only, no ZIP64 support.
  *
  * @return bool success, modifies the file contents
  */
 protected function fix_utf8_flags()
 {
     if ($this->emptyziphack) {
         return true;
     }
     if (!file_exists($this->archivepathname)) {
         return true;
     }
     // Note: the ZIP structure is described at http://www.pkware.com/documents/casestudies/APPNOTE.TXT
     if (!($fp = fopen($this->archivepathname, 'rb+'))) {
         return false;
     }
     if (!($filesize = filesize($this->archivepathname))) {
         return false;
     }
     $centralend = self::zip_get_central_end($fp, $filesize);
     if ($centralend === false or $centralend['disk'] !== 0 or $centralend['disk_start'] !== 0 or $centralend['offset'] === 0xffffffff) {
         // Single disk archives only and o support for ZIP64, sorry.
         fclose($fp);
         return false;
     }
     fseek($fp, $centralend['offset']);
     $data = fread($fp, $centralend['size']);
     $pos = 0;
     $files = array();
     for ($i = 0; $i < $centralend['entries']; $i++) {
         $file = self::zip_parse_file_header($data, $centralend, $pos);
         if ($file === false) {
             // Wrong header, sorry.
             fclose($fp);
             return false;
         }
         $newgeneral = $file['general'] | pow(2, 11);
         if ($newgeneral === $file['general']) {
             // Nothing to do with this file.
             continue;
         }
         if (preg_match('/^[a-zA-Z0-9_\\-\\.]*$/', $file['name'])) {
             // ASCII file names are always ok.
             continue;
         }
         if ($file['extra']) {
             // Most probably not created by php zip ext, better to skip it.
             continue;
         }
         if (fix_utf8($file['name']) !== $file['name']) {
             // Does not look like a valid utf-8 encoded file name, skip it.
             continue;
         }
         // Read local file header.
         fseek($fp, $file['local_offset']);
         $localfile = unpack('Vsig/vversion_req/vgeneral/vmethod/vmtime/vmdate/Vcrc/Vsize_compressed/Vsize/vname_length/vextra_length', fread($fp, 30));
         if ($localfile['sig'] !== 0x4034b50) {
             // Borked file!
             fclose($fp);
             return false;
         }
         $file['local'] = $localfile;
         $files[] = $file;
     }
     foreach ($files as $file) {
         $localfile = $file['local'];
         // Add the unicode flag in central file header.
         fseek($fp, $file['central_offset'] + 8);
         if (ftell($fp) === $file['central_offset'] + 8) {
             $newgeneral = $file['general'] | pow(2, 11);
             fwrite($fp, pack('v', $newgeneral));
         }
         // Modify local file header too.
         fseek($fp, $file['local_offset'] + 6);
         if (ftell($fp) === $file['local_offset'] + 6) {
             $newgeneral = $localfile['general'] | pow(2, 11);
             fwrite($fp, pack('v', $newgeneral));
         }
     }
     fclose($fp);
     return true;
 }

/**
 * Determine if there is data waiting to be processed from a form
 *
 * Used on most forms in Moodle to check for data
 * Returns the data as an object, if it's found.
 * This object can be used in foreach loops without
 * casting because it's cast to (array) automatically
 *
 * Checks that submitted POST data exists and returns it as object.
 *
 * @return mixed false or object
 */
function data_submitted()
{
    if (empty($_POST)) {
        return false;
    } else {
        return (object) fix_utf8($_POST);
    }
}

/**
 * Makes sure the data is using valid utf8, invalid characters are discarded.
 *
 * Note: this function is not intended for full objects with methods and private properties.
 *
 * @param mixed $value
 * @return mixed with proper utf-8 encoding
 */
function fix_utf8($value)
{
    if (is_null($value) or $value === '') {
        return $value;
    } else {
        if (is_string($value)) {
            if ((string) (int) $value === $value) {
                // shortcut
                return $value;
            }
            return iconv('UTF-8', 'UTF-8//IGNORE', $value);
        } else {
            if (is_array($value)) {
                foreach ($value as $k => $v) {
                    $value[$k] = fix_utf8($v);
                }
                return $value;
            } else {
                if (is_object($value)) {
                    $value = clone $value;
                    // do not modify original
                    foreach ($value as $k => $v) {
                        $value->{$k} = fix_utf8($v);
                    }
                    return $value;
                } else {
                    // this is some other type, no utf-8 here
                    return $value;
                }
            }
        }
    }
}

/**
 * Makes sure the data is using valid utf8, invalid characters are discarded.
 *
 * Note: this function is not intended for full objects with methods and private properties.
 *
 * @param mixed $value
 * @return mixed with proper utf-8 encoding
 */
function fix_utf8($value)
{
    if (is_null($value) or $value === '') {
        return $value;
    } else {
        if (is_string($value)) {
            if ((string) (int) $value === $value) {
                // shortcut
                return $value;
            }
            // lower error reporting because glibc throws bogus notices
            $olderror = error_reporting();
            if ($olderror & E_NOTICE) {
                error_reporting($olderror ^ E_NOTICE);
            }
            $result = iconv('UTF-8', 'UTF-8//IGNORE', $value);
            if ($olderror & E_NOTICE) {
                error_reporting($olderror);
            }
            return $result;
        } else {
            if (is_array($value)) {
                foreach ($value as $k => $v) {
                    $value[$k] = fix_utf8($v);
                }
                return $value;
            } else {
                if (is_object($value)) {
                    $value = clone $value;
                    // do not modify original
                    foreach ($value as $k => $v) {
                        $value->{$k} = fix_utf8($v);
                    }
                    return $value;
                } else {
                    // this is some other type, no utf-8 here
                    return $value;
                }
            }
        }
    }
}

 function test_fix_utf8()
 {
     // make sure valid data including other types is not changed
     $this->assertidentical(null, fix_utf8(null));
     $this->assertidentical(1, fix_utf8(1));
     $this->assertidentical(1.1, fix_utf8(1.1));
     $this->assertidentical(true, fix_utf8(true));
     $this->assertidentical('', fix_utf8(''));
     $array = array('do', 're', 'mi');
     $this->assertidentical($array, fix_utf8($array));
     $object = new stdClass();
     $object->a = 'aa';
     $object->b = 'bb';
     $this->assertidentical($object, fix_utf8($object));
     // valid utf8 string
     $this->assertidentical("žlutý koníček přeskočil potůček \n\t\r", fix_utf8("žlutý koníček přeskočil potůček \n\t\r"));
     // invalid utf8 string
     $this->assertidentical('aaabbb', fix_utf8('aaa' . chr(130) . 'bbb'));
 }

/**
 * Makes sure the data is using valid utf8, invalid characters are discarded.
 *
 * Note: this function is not intended for full objects with methods and private properties.
 *
 * @param mixed $value
 * @return mixed with proper utf-8 encoding
 */
function fix_utf8($value)
{
    if (is_null($value) or $value === '') {
        return $value;
    } else {
        if (is_string($value)) {
            if ((string) (int) $value === $value) {
                // shortcut
                return $value;
            }
            // Lower error reporting because glibc throws bogus notices.
            $olderror = error_reporting();
            if ($olderror & E_NOTICE) {
                error_reporting($olderror ^ E_NOTICE);
            }
            // Note: this duplicates min_fix_utf8() intentionally.
            static $buggyiconv = null;
            if ($buggyiconv === null) {
                $buggyiconv = (!function_exists('iconv') or iconv('UTF-8', 'UTF-8//IGNORE', '100' . chr(130) . '\\80') !== '100\\80');
            }
            if ($buggyiconv) {
                if (function_exists('mb_convert_encoding')) {
                    $subst = mb_substitute_character();
                    mb_substitute_character('');
                    $result = mb_convert_encoding($value, 'utf-8', 'utf-8');
                    mb_substitute_character($subst);
                } else {
                    // Warn admins on admin/index.php page.
                    $result = $value;
                }
            } else {
                $result = iconv('UTF-8', 'UTF-8//IGNORE', $value);
            }
            if ($olderror & E_NOTICE) {
                error_reporting($olderror);
            }
            return $result;
        } else {
            if (is_array($value)) {
                foreach ($value as $k => $v) {
                    $value[$k] = fix_utf8($v);
                }
                return $value;
            } else {
                if (is_object($value)) {
                    $value = clone $value;
                    // do not modify original
                    foreach ($value as $k => $v) {
                        $value->{$k} = fix_utf8($v);
                    }
                    return $value;
                } else {
                    // this is some other type, no utf-8 here
                    return $value;
                }
            }
        }
    }
}

 /**
  * Upload program objectives.
  * @param object $mform	form definition
  * @return void
  */
 private function upload_program_obj($mform)
 {
     global $DB, $CFG, $USER;
     global $categoryId;
     $obj_was_uploaded = $mform->getSubmitValue('upload_program_obj');
     if ($obj_was_uploaded) {
         $files = $this->get_draft_files('temp_program_obj');
         if (!empty($files)) {
             $file = reset($files);
             $content = $file->get_content();
             $all_rows = explode("\n", $content);
             $groups = array();
             $titles = array();
             $type = new stdClass();
             $type->typename = $mform->getSubmitValue('new_group');
             $type->category = $categoryId;
             $masterid = $DB->insert_record('objectivetypes', $type, true);
             foreach ($all_rows as $row) {
                 $parsed = str_getcsv($row);
                 if (!is_null($parsed[0])) {
                     if ($parsed[0] != '' && $parsed[1] != '') {
                         $parsed[0] = fix_utf8($parsed[0]);
                         $parsed[1] = fix_utf8($parsed[1]);
                         if (!array_key_exists($parsed[0], $groups)) {
                             $group = new stdClass();
                             $group->groupname = $parsed[0];
                             $group->parent = $masterid;
                             $groupid = $DB->insert_record('objectivegroups', $group, true);
                             $groups[$parsed[0]] = $groupid;
                             // set to ID
                         }
                         if (!array_key_exists($parsed[1], $titles)) {
                             $title = new stdClass();
                             $title->objectivename = $parsed[1];
                             $title->parent = null;
                             $title->objectivegroup = $groups[$parsed[0]];
                             //echo "writing title\n";
                             //echo $parsed[1];
                             $titleid = $DB->insert_record('programobjectives', $title, true);
                             //echo "wrote title\n";
                             //$titleid = 1;
                             $titles[$parsed[1]] = $titleid;
                             // set to ID
                         }
                         if ($parsed[2] != '') {
                             $parsed[2] = fix_utf8($parsed[2]);
                             $title = new stdClass();
                             $title->objectivename = $parsed[2];
                             $title->parent = $titles[$parsed[1]];
                             $title->objectivegroup = $groups[$parsed[0]];
                             $DB->insert_record('programobjectives', $title, false);
                         }
                         //$this->insert_program_objective($groups[$parsed[0]], $groups[$parsed[1]], $parsed[2]);
                     }
                 }
             }
         }
     }
 }

 /**
  * Converts the text between different encodings. It uses iconv extension with //TRANSLIT parameter,
  * falls back to typo3. If both source and target are utf-8 it tries to fix invalid characters only.
  *
  * @param string $text
  * @param string $fromCS source encoding
  * @param string $toCS result encoding
  * @return string|bool converted string or false on error
  */
 public static function convert($text, $fromCS, $toCS = 'utf-8')
 {
     $fromCS = self::parse_charset($fromCS);
     $toCS = self::parse_charset($toCS);
     $text = (string) $text;
     // we can work only with strings
     if ($text === '') {
         return '';
     }
     if ($fromCS === 'utf-8') {
         $text = fix_utf8($text);
         if ($toCS === 'utf-8') {
             return $text;
         }
     }
     if ($toCS === 'ascii') {
         // Try to normalize the conversion a bit.
         $text = self::specialtoascii($text, $fromCS);
     }
     // Prevent any error notices, do not use //IGNORE so that we get
     // consistent result from Typo3 if iconv fails.
     $result = @iconv($fromCS, $toCS . '//TRANSLIT', $text);
     if ($result === false or $result === '') {
         // note: iconv is prone to return empty string when invalid char encountered, or false if encoding unsupported
         $oldlevel = error_reporting(E_PARSE);
         $result = self::typo3()->conv((string) $text, $fromCS, $toCS);
         error_reporting($oldlevel);
     }
     return $result;
 }

// PayPal does not like when we return error messages here,
// the custom handler just logs exceptions and stops.
set_exception_handler('enrol_paypal_ipn_exception_handler');
/// Keep out casual intruders
if (empty($_POST) or !empty($_GET)) {
    print_error("Sorry, you can not use the script that way.");
}
/// Read all the data from PayPal and get it ready for later;
/// we expect only valid UTF-8 encoding, it is the responsibility
/// of user to set it up properly in PayPal business account,
/// it is documented in docs wiki.
$req = 'cmd=_notify-validate';
$data = new stdClass();
foreach ($_POST as $key => $value) {
    $req .= "&{$key}=" . urlencode($value);
    $data->{$key} = fix_utf8($value);
}
$custom = explode('-', $data->custom);
$data->userid = (int) $custom[0];
$data->courseid = (int) $custom[1];
$data->instanceid = (int) $custom[2];
$data->payment_gross = $data->mc_gross;
$data->payment_currency = $data->mc_currency;
$data->timeupdated = time();
/// get the user and course records
if (!($user = $DB->get_record("user", array("id" => $data->userid)))) {
    message_paypal_error_to_admin("Not a valid user id", $data);
    die;
}
if (!($course = $DB->get_record("course", array("id" => $data->courseid)))) {
    message_paypal_error_to_admin("Not a valid course id", $data);

/**
 * Registers the commit info for all new commits on the given branch
 *
 * @param PHPGit_Repository $repo repository to parse
 * @param string $gitbranch the real name of the branch to analyze (eg 'master')
 * @param string $branch the future name of the same branch (eg 'MOODLE_28_STABLE')
 * @param string $mergemode either 'merges' or 'no-merges'
 * @param bool $showprogress
 * @internal
 */
function dev_git_record_commits(PHPGit_Repository $repo, $gitbranch, $branch, $mergemode, $showprogress = false)
{
    global $DB;
    $startpoints = get_config('local_dev', 'gitstartpoints');
    if ($startpoints === false) {
        set_config('gitstartpoints', json_encode(array()), 'local_dev');
        $startpoints = get_config('local_dev', 'gitstartpoints');
    }
    $startpoints = json_decode($startpoints, true);
    $reponame = basename($repo->getDir());
    $exclude = empty($startpoints[$branch][$mergemode]) ? '' : $startpoints[$branch][$mergemode];
    if ($mergemode === 'merges') {
        fputs(STDOUT, "Searching merges on {$gitbranch} ({$branch})" . ($exclude ? " from {$exclude}" : "") . PHP_EOL);
        $mergeflag = 1;
    } else {
        if ($mergemode === 'no-merges') {
            fputs(STDOUT, "Searching non-merges on {$gitbranch} ({$branch})" . ($exclude ? " from {$exclude}" : "") . PHP_EOL);
            $mergeflag = 0;
        }
    }
    $exclude = empty($exclude) ? '' : '^' . $exclude;
    $commits = explode(PHP_EOL, $repo->git("rev-list --reverse --{$mergemode} --format='tformat:COMMIT:%H TIMESTAMP:%at AUTHORNAME:%an AUTHOREMAIL:%ae SUBJECT:%s' {$gitbranch} {$exclude}"));
    $total = floor(count($commits) / 2);
    $counter = 0;
    if ($showprogress and $total == 0) {
        fputs(STDOUT, 'no commits found');
    }
    foreach ($commits as $commit) {
        $pattern = '/^COMMIT:([0-9a-f]{40}) TIMESTAMP:([0-9]+) AUTHORNAME:(.+) AUTHOREMAIL:(.+) SUBJECT:(.*)$/';
        if (!preg_match($pattern, $commit, $matches)) {
            continue;
        }
        $record = new stdClass();
        $record->repository = $reponame;
        $record->commithash = $matches[1];
        $record->authordate = $matches[2];
        $record->authorname = $matches[3];
        $record->authoremail = $matches[4];
        $record->subject = $matches[5];
        $record->merge = $mergeflag;
        $record = @fix_utf8($record);
        // register the commit info record if it does not exist yet
        $existing = $DB->get_record('dev_git_commits', array('repository' => $reponame, 'commithash' => $record->commithash), 'id', IGNORE_MISSING);
        if ($existing === false) {
            $commitid = $DB->insert_record('dev_git_commits', $record, true, true);
        } else {
            $commitid = $existing->id;
        }
        // register the branch containing the current commit
        if (!$DB->record_exists('dev_git_commit_branches', array('branch' => $branch, 'commitid' => $commitid))) {
            $branchinfo = new stdClass();
            $branchinfo->commitid = $commitid;
            $branchinfo->branch = $branch;
            $DB->insert_record('dev_git_commit_branches', $branchinfo, false, true);
        }
        if ($showprogress) {
            fputs(STDOUT, ++$counter . '/' . $total . "\r");
        }
        $startpoints[$branch][$mergemode] = $record->commithash;
        if ($counter % 1000 == 0) {
            set_config('gitstartpoints', json_encode($startpoints), 'local_dev');
        }
    }
    set_config('gitstartpoints', json_encode($startpoints), 'local_dev');
    if ($showprogress) {
        fputs(STDOUT, PHP_EOL);
    }
}

    /**
     * Converts the text between different encodings. It uses iconv extension with //TRANSLIT parameter,
     * falls back to typo3. If both source and target are utf-8 it tries to fix invalid characters only.
     *
     * @param string $text
     * @param string $fromCS source encoding
     * @param string $toCS result encoding
     * @return string|bool converted string or false on error
     */
    public static function convert($text, $fromCS, $toCS='utf-8') {
        $fromCS = self::parse_charset($fromCS);
        $toCS   = self::parse_charset($toCS);

        $text = (string)$text; // we can work only with strings

        if ($text === '') {
            return '';
        }

        if ($toCS === 'utf-8' and $fromCS === 'utf-8') {
            return fix_utf8($text);
        }

        $result = iconv($fromCS, $toCS.'//TRANSLIT', $text);

        if ($result === false or $result === '') {
            // note: iconv is prone to return empty string when invalid char encountered, or false if encoding unsupported
            $oldlevel = error_reporting(E_PARSE);
            $result = self::typo3()->conv((string)$text, $fromCS, $toCS);
            error_reporting($oldlevel);
        }

        return $result;
    }