ENH Update reference to supported modules data (#45)

silverstripe · May 9, 2024 · c42e9b4 · c42e9b4
1 parent 3029172
commit c42e9b4
Show file tree

Hide file tree

Showing 8 changed files with 81 additions and 170 deletions.
diff --git a/README.md b/README.md
@@ -102,7 +102,3 @@ MS_GITHUB_TOKEN=abc123 php run.php labels --dry-run --only=silverstripe-config,s
 | --exclude=[modules] | Exclude the specified modules (without account prefix) separated by commas e.g. `silverstripe-mfa,silverstripe-totp` |
 | --dry-run | Do not update labels in GitHub, output to terminal only |
 | --no-delete | Do not delete `_data` directory before running |
-
-## Updating the tool when a new major version of CMS is updated
-
-Update the `CURRENT_CMS_MAJOR` constant in `run.php`
diff --git a/composer.json b/composer.json
@@ -1,6 +1,7 @@
 {
     "require": {
         "php": ">=8.1",
+        "silverstripe/supported-modules": "dev-main",
         "symfony/console": "^6.3",
         "symfony/process": "^6.3",
         "panlatent/cron-expression-descriptor": "^1"

diff --git a/funcs_scripts.php b/funcs_scripts.php
@@ -1,6 +1,7 @@
 <?php
 
 use Panlatent\CronExpressionDescriptor\ExpressionDescriptor;
+use SilverStripe\SupportedModules\MetaData;
 
 // These functions in scripts can be used in scripts
 
@@ -243,8 +244,14 @@ function is_docs()
  */
 function is_gha_repository()
 {
-    global $MODULE_DIR;
-    return !is_module() && strpos($MODULE_DIR, '/gha-') !== false;
+    global $GITHUB_REF;
+    return in_array(
+        $GITHUB_REF,
+        array_column(
+            MetaData::getAllRepositoryMetaData()[MetaData::CATEGORY_WORKFLOW],
+            'github'
+        )
+    );
 }
 
 /**
@@ -257,8 +264,8 @@ function is_gha_repository()
  */
 function module_name()
 {
-    global $MODULE_DIR;
-    $parts = explode('/', $MODULE_DIR);
+    global $GITHUB_REF;
+    $parts = explode('/', $GITHUB_REF);
     return end($parts);
 }
 
@@ -270,15 +277,15 @@ function module_name()
  */
 function module_is_one_of($repos)
 {
-    global $MODULE_DIR;
+    global $GITHUB_REF;
     if (!is_array($repos)) {
         error('repos is not an array');
     }
     foreach ($repos as $repo) {
         if (!is_string($repo)) {
             error('repo is not a string');
         }
-        if (strpos($MODULE_DIR, "/$repo") !== false) {
+        if (strpos($GITHUB_REF, "/$repo") !== false) {
             return true;
         }
     }
@@ -293,11 +300,8 @@ function module_is_one_of($repos)
  */
 function module_account()
 {
-    $s = read_file('.git/config');
-    if (!preg_match('#github.com:([^/]+)/#', $s, $matches)) {
-        error('Could not determine github account');
-    }
-    return $matches[1];
+    global $GITHUB_REF;
+    return explode('/', $GITHUB_REF)[0];
 }
 
 /**

diff --git a/funcs_utils.php b/funcs_utils.php
@@ -1,5 +1,7 @@
 <?php
 
+use SilverStripe\SupportedModules\BranchLogic;
+use SilverStripe\SupportedModules\MetaData;
 use Symfony\Component\Console\Output\NullOutput;
 use Symfony\Component\Console\Input\ArgvInput;
 use Symfony\Component\Console\Style\SymfonyStyle;
@@ -46,85 +48,6 @@ function write_file($path, $contents)
     info("Wrote to $path");
 }
 
-/**
- * Returns all the supported modules for a particular cms major version
- * Will download the list if it doesn't exist
- */
-function supported_modules($cmsMajor)
-{
-    $filename = "_data/modules-cms$cmsMajor.json";
-    if (!file_exists($filename)) {
-        $url = "https://raw.githubusercontent.com/silverstripe/supported-modules/$cmsMajor/modules.json";
-        info("Downloading $url to $filename");
-        $contents = file_get_contents($url);
-        file_put_contents($filename, $contents);
-    }
-    $json = json_decode(file_get_contents($filename), true);
-    if (is_null($json)) {
-        $lastError = json_last_error();
-        error("Could not parse from $filename - last error was $lastError");
-    }
-    $modules = [];
-    foreach ($json as $module) {
-        $ghrepo = $module['github'];
-        $modules[] = [
-            'ghrepo' => $ghrepo,
-            'account' => explode('/', $ghrepo)[0],
-            'repo' => explode('/', $ghrepo)[1],
-            'cloneUrl' => "git@github.com:$ghrepo.git",
-        ];
-    }
-    return $modules;
-}
-
-/**
- * Hardcoded list of non-supported, additional repositories to standardise (e.g. silverstripe/gha-*)
- *
- * Repositories in this list should only have a single supported major version
- * This will only be included if the $cmsMajor is the CURRENT_CMS_MAJOR
- */
-function extra_repositories()
-{
-    $importantRepos = [
-        'silverstripe/markdown-php-codesniffer',
-        'silverstripe/silverstripe-standards',
-        'silverstripe/documentation-lint',
-        'silverstripe/.github',
-    ];
-    $modules = [];
-    // iterating to page 10 will be enough to get all the repos well into the future
-    for ($i = 0; $i < 10; $i++) {
-        $path = "_data/extra_repositories-$i.json";
-        if (file_exists($path)) {
-            info("Reading local data from $path");
-            $json = json_decode(file_get_contents($path), true);
-        } else {
-            $json = github_api("https://api.github.com/orgs/silverstripe/repos?per_page=100&page=$i");
-            file_put_contents($path, json_encode($json, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
-        }
-        if (empty($json)) {
-            break;
-        }
-        foreach ($json as $repo) {
-            if ($repo['archived']) {
-                continue;
-            }
-            $ghrepo = $repo['full_name'];
-            // Only include repos we care about
-            if (!in_array($ghrepo, $importantRepos) && strpos($ghrepo, '/gha-') === false) {
-                continue;
-            }
-            $modules[] = [
-                'ghrepo' => $ghrepo,
-                'account' => explode('/', $ghrepo)[0],
-                'repo' => explode('/', $ghrepo)[1],
-                'cloneUrl' => "git@github.com:$ghrepo.git",
-            ];
-        }
-    }
-    return $modules;
-}
-
 /**
  * Returns a list of all scripts files to run against a particular cms major version
  */
@@ -349,85 +272,45 @@ function branch_to_checkout($branches, $defaultBranch, $currentBranch, $currentB
 }
 
 /**
- * Uses composer.json to workout the current branch cms major version
- *
- * If composer.json does not exist then it's assumed to be CURRENT_CMS_MAJOR
+ * Works out the current branch cms major version
  */
 function current_branch_cms_major(
     // this param is only used for unit testing
     string $composerJson = ''
 ) {
-    global $MODULE_DIR;
+    global $MODULE_DIR, $GITHUB_REF;
 
-    // Some repositories don't have a valid matching CMS major
-    $ignoreCMSMajor = [
-        '/silverstripe-simple',
-        '/markdown-php-codesniffer',
-    ];
-    foreach ($ignoreCMSMajor as $ignore) {
-        if (strpos($MODULE_DIR, $ignore) !== false) {
-            return CURRENT_CMS_MAJOR;
-        }
+    // This repo matches every major and matches start at the lowest - but we only want the highest stable.
+    if ($GITHUB_REF === 'silverstripe/silverstripe-simple') {
+        return MetaData::HIGHEST_STABLE_CMS_MAJOR;
     }
 
     if ($composerJson) {
         $contents = $composerJson;
     } elseif (check_file_exists('composer.json')) {
         $contents = read_file('composer.json');
-    } else {
-        return CURRENT_CMS_MAJOR;
     }
-
-    // special logic for developer-docs
-    if (strpos($MODULE_DIR, '/developer-docs') !== false) {
-        $currentBranch = cmd('git rev-parse --abbrev-ref HEAD', $MODULE_DIR);
-        if (!preg_match('#^(pulls/)?([0-9]+)(\.[0-9]+)?(/|$)#', $currentBranch, $matches)) {
-            error("Could not work out current major for developer-docs from branch $currentBranch");
-        }
-        return $matches[2];
-    }
-
-    $json = json_decode($contents);
-    if (is_null($json)) {
+    $composerJson = json_decode($contents);
+    if (is_null($composerJson)) {
         $lastError = json_last_error();
         error("Could not parse from composer.json - last error was $lastError");
     }
-    $matchedOnBranchThreeLess = false;
-    $version = preg_replace('#[^0-9\.]#', '', $json->require->{'silverstripe/framework'} ?? '');
-    if (!$version) {
-        $version = preg_replace('#[^0-9\.]#', '', $json->require->{'silverstripe/cms'} ?? '');
-    }
-    if (!$version) {
-        $version = preg_replace('#[^0-9\.]#', '', $json->require->{'silverstripe/mfa'} ?? '');
-    }
-    if (!$version) {
-        $version = preg_replace('#[^0-9\.]#', '', $json->require->{'silverstripe/assets'} ?? '');
-        if ($version) {
-            $matchedOnBranchThreeLess = true;
-        }
-    }
-    if (!$version) {
-        $version = preg_replace('#[^0-9\.]#', '', $json->require->{'cwp/starter-theme'} ?? '');
-        if ($version) {
-            $version += 1;
-        }
-    }
-    $cmsMajor = '';
-    if (preg_match('#^([0-9]+)+\.?[0-9]*$#', $version, $matches)) {
-        $cmsMajor = $matches[1];
-        if ($matchedOnBranchThreeLess) {
-            $cmsMajor += 3;
-        }
-    } else {
-        $phpVersion = $json->require->{'php'} ?? '';
-        if (substr($phpVersion,0, 4) === '^7.4') {
-            $cmsMajor = 4;
-        } elseif (substr($phpVersion,0, 4) === '^8.1') {
-            $cmsMajor = 5;
+
+    $repoData = MetaData::getMetaDataForRepository($GITHUB_REF);
+    $branchMajor = '';
+    // If we're running unit tests, $MODULE_DIR will be some fake value with causes errors here
+    if (!running_unit_tests()) {
+        $currentBranch = cmd('git rev-parse --abbrev-ref HEAD', $MODULE_DIR);
+        if (preg_match('#^(pulls/)?([0-9]+)(\.[0-9]+)?(/|$)#', $currentBranch, $matches)) {
+            $branchMajor = $matches[2];
         }
     }
+    $cmsMajor = BranchLogic::getCmsMajor($repoData, $branchMajor, $composerJson, true);
+
     if ($cmsMajor === '') {
-        error('Could not work out what the current CMS major version is');
+        // The supported modules metadata has a bunch of repos with no specific major version mapping.
+        // Just assume they're on the highest major in that case.
+        return MetaData::HIGHEST_STABLE_CMS_MAJOR;
     }
     return (string) $cmsMajor;
 }
@@ -445,13 +328,18 @@ function setup_directories($input, $dirs = [DATA_DIR, MODULES_DIR]) {
     }
 }
 
-function filtered_modules($cmsMajor, $input) {
-    $modules = supported_modules($cmsMajor);
-    if ($cmsMajor === CURRENT_CMS_MAJOR) {
-        // only include extra_repositories() when using the current CMS major version because the extra rexpositories
-        // don't have multi majors branches supported e.g. gha-generate-matrix
-        $modules = array_merge($modules, extra_repositories());
-    }
+function filtered_modules($cmsMajor, $input)
+{
+    $repos = MetaData::removeReposNotInCmsMajor(
+        MetaData::getAllRepositoryMetaData(false),
+        $cmsMajor,
+        // For repositories that only have a single support branch such as gha-generate-matrix, only include
+        // them when updating the currently supported CMS major.
+        $cmsMajor === MetaData::HIGHEST_STABLE_CMS_MAJOR
+    );
+
+    $modules = convert_repos_data_to_modules($repos);
+
     if ($input->getOption('only')) {
         $only = explode(',', $input->getOption('only'));
         $modules = array_filter($modules, function ($module) use ($only) {
@@ -466,3 +354,18 @@ function filtered_modules($cmsMajor, $input) {
     }
     return $modules;
 }
+
+function convert_repos_data_to_modules(array $repos)
+{
+    $modules = [];
+    foreach ($repos as $repo) {
+        $ghrepo = $repo['github'];
+        $modules[] = [
+            'ghrepo' => $ghrepo,
+            'account' => explode('/', $ghrepo)[0],
+            'repo' => explode('/', $ghrepo)[1],
+            'cloneUrl' => "git@github.com:$ghrepo.git",
+        ];
+    }
+    return $modules;
+}
diff --git a/labels_command.php b/labels_command.php
@@ -1,5 +1,6 @@
 <?php
 
+use SilverStripe\SupportedModules\MetaData;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Command\Command;
@@ -73,8 +74,8 @@
     // modules
     $modules = [];
     $repos = [];
-    $modulesCurrentMajor = filtered_modules(CURRENT_CMS_MAJOR, $input);
-    $modulesPreviousMajor = filtered_modules(CURRENT_CMS_MAJOR - 1, $input);
+    $modulesCurrentMajor = filtered_modules(MetaData::HIGHEST_STABLE_CMS_MAJOR, $input);
+    $modulesPreviousMajor = filtered_modules(MetaData::HIGHEST_STABLE_CMS_MAJOR - 1, $input);
     foreach ([$modulesCurrentMajor, $modulesPreviousMajor] as $modulesList) {
         foreach ($modulesList as $module) {
             $repo = $module['repo'];

diff --git a/run.php b/run.php
@@ -6,11 +6,11 @@
 include 'update_command.php';
 include 'labels_command.php';
 
+use SilverStripe\SupportedModules\MetaData;
 use Symfony\Component\Console\Application;
 use Symfony\Component\Console\Input\InputOption;
 
 // consts
-const CURRENT_CMS_MAJOR = '5';
 const BRANCH_OPTIONS = ['next-minor', 'next-patch', 'github-default'];
 const DEFAULT_BRANCH = 'next-patch';
 const DEFAULT_ACCOUNT = 'creative-commoners';
@@ -22,6 +22,7 @@
 
 // global variables
 $MODULE_DIR = '';
+$GITHUB_REF = '';
 $PRS_CREATED = [];
 $REPOS_WITH_PRS_CREATED = [];
 $REPOS_WITH_LABELS_UPDATED = [];
@@ -32,7 +33,7 @@
     'cms-major',
     null,
     InputOption::VALUE_REQUIRED,
-    'The CMS major version to use (default: '. CURRENT_CMS_MAJOR .')'
+    'The CMS major version to use (default: '. MetaData::HIGHEST_STABLE_CMS_MAJOR .')'
 ];
 $optionBranch = [
     'branch',