Skip to content

[RFC] First draft of deleted file check #25559

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Jul 13, 2021
Merged

[RFC] First draft of deleted file check #25559

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
0ffc05e
First draft of deleted file check
wilsonge Jul 15, 2019
dc1ac61
Dynamic directories and add back feedback
wilsonge Jul 15, 2019
fbcf783
Fix spaces
wilsonge Jul 15, 2019
af95f25
Add more spaces
wilsonge Jul 15, 2019
58c83ae
Ignore search and installation directories
wilsonge Nov 11, 2019
1ba7854
Reverse order the folders so we delete nested folders before top leve…
wilsonge Nov 11, 2019
cea3c61
Remove some language files for com_search which is sticking around.
wilsonge Jan 26, 2020
7ea8473
Remove repeatable fields
wilsonge Mar 22, 2020
a8ab3b7
Ignore some extra weblink related files
wilsonge Oct 26, 2020
ef26847
ignore mod_search
wilsonge Jan 3, 2021
3b87342
Don't remove previous release's sample data images
richard67 Apr 7, 2021
4222961
Handle files which have been renamed only
richard67 Apr 7, 2021
93f13c9
Remove comma from file to be renamed
richard67 Apr 7, 2021
512658a
Add a comment for better code reading
richard67 Apr 7, 2021
c61e028
Alpha order $previousReleaseExclude and $filesToKeep (#68)
richard67 Jun 27, 2021
eee17c8
[CMS PR 25559] Add list of folders to keep on update (#69)
richard67 Jul 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions build/.gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,5 @@

# Ignore the deleted file script output
/deleted_files.txt
/deleted_folders.txt
/renamed_files.txt
238 changes: 164 additions & 74 deletions build/deleted_file_check.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,59 +18,11 @@
* @license GNU General Public License version 2 or later; see LICENSE.txt
*/

// Detect the native operating system type.
$os = strtoupper(substr(PHP_OS, 0, 3));

if ($os === 'WIN')
{
echo 'Sorry, this script is not supported on Windows.' . PHP_EOL;

exit(1);
}

/*
* Constants
*/

const PHP_TAB = "\t";

/*
* Globals
*/

global $currentDir;
$currentDir = getcwd();

global $changedFiles;
$changedFiles = array();

global $deletedFiles;
$deletedFiles = array();

global $gitBinary;
ob_start();
passthru('which git', $gitBinary);
$gitBinary = trim(ob_get_clean());

/*
* Functions
*/

function run_git_command($command)
{
global $currentDir;

chdir(dirname(__DIR__));

ob_start();
passthru($command);
$return = trim(ob_get_clean());

chdir($currentDir);

return $return;
}

function usage($command)
{
echo PHP_EOL;
Expand All @@ -90,7 +42,7 @@ function usage($command)
if (empty($options['from']))
{
echo PHP_EOL;
echo 'Missing starting commit reference' . PHP_EOL;
echo 'Missing starting directory' . PHP_EOL;

usage($argv[0]);

Expand All @@ -100,46 +52,184 @@ function usage($command)
// Missing the to reference? No problem, grab the current HEAD
if (empty($options['to']))
{
$options['to'] = run_git_command("$gitBinary rev-parse HEAD");
echo PHP_EOL;
echo 'Missing ending directory' . PHP_EOL;

usage($argv[0]);

exit(1);
}

// Parse the git diff to know what files have been added, removed, or renamed
$fileDiff = explode("\n", run_git_command("$gitBinary diff --name-status {$options['from']} {$options['to']}"));
// Directories to skip for the check (needs to include anything from J3 we want to keep)
$previousReleaseExclude = [
$options['from'] . '/administrator/components/com_search',
$options['from'] . '/components/com_search',
$options['from'] . '/images/sampledata',
$options['from'] . '/installation',
$options['from'] . '/modules/mod_search',
$options['from'] . '/plugins/fields/repeatable',
$options['from'] . '/plugins/search',
];

$deletedFiles = array();
/**
* @param SplFileInfo $file The file being checked
* @param mixed $key ?
* @param RecursiveCallbackFilterIterator $iterator The iterator being processed
*
* @return bool True if you need to recurse or if the item is acceptable
*/
$previousReleaseFilter = function ($file, $key, $iterator) use ($previousReleaseExclude) {
if ($iterator->hasChildren() && !in_array($file->getPathname(), $previousReleaseExclude))
{
return true;
}

foreach ($fileDiff as $file)
{
$fileName = substr($file, 2);
return $file->isFile();
};

// Directories to skip for the check
$newReleaseExclude = [
$options['to'] . '/installation'
];

// Act on the file based on the action
switch (substr($file, 0, 1))
/**
* @param SplFileInfo $file The file being checked
* @param mixed $key ?
* @param RecursiveCallbackFilterIterator $iterator The iterator being processed
*
* @return bool True if you need to recurse or if the item is acceptable
*/
$newReleaseFilter = function ($file, $key, $iterator) use ($newReleaseExclude) {
if ($iterator->hasChildren() && !in_array($file->getPathname(), $newReleaseExclude))
{
// This is a new case with git 2.9 to handle renamed files
case 'R':
// Explode the file on the tab character; key 0 is the action (rename), key 1 is the old filename, and key 2 is the new filename
$renamedFileData = explode("\t", $file);
return true;
}

// And flag the old file as deleted
$deletedFiles[] = "'/{$renamedFileData[1]}',";
return $file->isFile();
};

break;
$previousReleaseDirIterator = new RecursiveDirectoryIterator($options['from'], RecursiveDirectoryIterator::SKIP_DOTS);
$previousReleaseIterator = new RecursiveIteratorIterator(
new RecursiveCallbackFilterIterator($previousReleaseDirIterator, $previousReleaseFilter),
RecursiveIteratorIterator::SELF_FIRST
);
$previousReleaseFiles = [];
$previousReleaseFolders = [];

case 'D':
$deletedFiles[] = "'/$fileName',";
foreach ($previousReleaseIterator as $info)
{
if ($info->isDir())
{
$previousReleaseFolders[] = "'" . str_replace($options['from'], '', $info->getPathname()) . "',";
continue;
}

$previousReleaseFiles[] = "'" . str_replace($options['from'], '', $info->getPathname()) . "',";
}

break;
$newReleaseDirIterator = new RecursiveDirectoryIterator($options['to'], RecursiveDirectoryIterator::SKIP_DOTS);
$newReleaseIterator = new RecursiveIteratorIterator(
new RecursiveCallbackFilterIterator($newReleaseDirIterator, $newReleaseFilter),
RecursiveIteratorIterator::SELF_FIRST
);
$newReleaseFiles = [];
$newReleaseFolders = [];

default:
// Ignore file additions and modifications
break;
foreach ($newReleaseIterator as $info)
{
if ($info->isDir())
{
$newReleaseFolders[] = "'" . str_replace($options['to'], '', $info->getPathname()) . "',";
continue;
}

$newReleaseFiles[] = "'" . str_replace($options['to'], '', $info->getPathname()) . "',";
}

asort($deletedFiles);
$filesDifference = array_diff($previousReleaseFiles, $newReleaseFiles);

$foldersDifference = array_diff($previousReleaseFolders, $newReleaseFolders);

// Specific files (e.g. language files) that we want to keep on upgrade
$filesToKeep = [
"'/administrator/language/en-GB/en-GB.com_search.ini',",
"'/administrator/language/en-GB/en-GB.com_search.sys.ini',",
"'/administrator/language/en-GB/en-GB.plg_editors-xtd_weblink.ini',",
"'/administrator/language/en-GB/en-GB.plg_editors-xtd_weblink.sys.ini',",
"'/administrator/language/en-GB/en-GB.plg_fields_repeatable.ini',",
"'/administrator/language/en-GB/en-GB.plg_fields_repeatable.sys.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_categories.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_categories.sys.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_contacts.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_contacts.sys.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_content.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_content.sys.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_newsfeeds.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_newsfeeds.sys.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_tags.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_tags.sys.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_weblinks.ini',",
"'/administrator/language/en-GB/en-GB.plg_search_weblinks.sys.ini',",
"'/administrator/language/en-GB/en-GB.plg_system_weblinks.ini',",
"'/administrator/language/en-GB/en-GB.plg_system_weblinks.sys.ini',",
"'/language/en-GB/en-GB.com_search.ini',",
"'/language/en-GB/en-GB.mod_search.ini',",
"'/language/en-GB/en-GB.mod_search.sys.ini',",
];

// Specific folders that we want to keep on upgrade
$foldersToKeep = [
"'/bin',",
];

// Remove folders from the results which we want to keep on upgrade
foreach ($foldersToKeep as $folder)
{
if (($key = array_search($folder, $foldersDifference)) !== false) {
unset($foldersDifference[$key]);
}
}

asort($filesDifference);
rsort($foldersDifference);

$deletedFiles = [];
$renamedFiles = [];

foreach ($filesDifference as $file)
{
// Don't remove any specific files (e.g. language files) that we want to keep on upgrade
if (array_search($file, $filesToKeep) !== false)
{
continue;
}

// Check for files which might have been renamed only
$matches = preg_grep('/^' . preg_quote($file, '/') . '$/i', $newReleaseFiles);

if ($matches !== false)
{
foreach ($matches as $match)
{
if (dirname($match) === dirname($file) && strtolower(basename($match)) === strtolower(basename($file)))
{
// File has been renamed only: Add to renamed files list
$renamedFiles[] = substr($file, 0, -1) . ' => ' . $match;

// Go on with the next file in $filesDifference
continue 2;
}
}
}

// File has been really deleted and not just renamed
$deletedFiles[] = $file;
}

// Write the deleted files list to a file for later reference
// Write the lists to files for later reference
file_put_contents(__DIR__ . '/deleted_files.txt', implode("\n", $deletedFiles));
file_put_contents(__DIR__ . '/deleted_folders.txt', implode("\n", $foldersDifference));
file_put_contents(__DIR__ . '/renamed_files.txt', implode("\n", $renamedFiles));

echo PHP_EOL;
echo 'There are ' . count($deletedFiles) . ' deleted files in comparison to "' . $options['from'] . '"' . PHP_EOL;
echo 'There are ' . count($deletedFiles) . ' deleted files, ' . count($foldersDifference) . ' deleted folders and ' . count($renamedFiles) . ' renamed files in comparison to "' . $options['from'] . '"' . PHP_EOL;