[DOCS-1988] Update Lychee link check actions to support crawling

.github/workflows/linkcheck-pr.yml

@@ -1,33 +1,213 @@
-name: Lychee PR link checker
+name: Link checker - PR changed files
+
+# Avoid collisions by ensuring only one run per ref
+concurrency:
+  group: linkcheck-pr-${{ github.ref_name }}
+  cancel-in-progress: false
 
 on:
+  workflow_dispatch:
   deployment_status:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - '**.md'
+      - '**.mdx'
 
 permissions:
   contents: read
   deployments: read
+  pull-requests: write
 
 jobs:
   linkChecker:
     runs-on: ubuntu-latest
-
-    # Only run when Mintlify PR deployment succeeds
+    
+    # Run on: manual trigger, successful Mintlify deployment, or PR events (for forks)
     if: |
-      github.event.deployment_status.state == 'success' && 
-      github.event.deployment.environment == 'staging' &&
-      contains(github.event.deployment_status.creator.login, 'mintlify') &&
-      contains(github.event.deployment_status.environment_url, 'mintlify')
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'pull_request' ||
+      (github.event_name == 'deployment_status' && 
+       github.event.deployment_status.state == 'success' && 
+       github.event.deployment.environment == 'staging' &&
+       contains(github.event.deployment_status.creator.login, 'mintlify') &&
+       contains(github.event.deployment_status.environment_url, 'mintlify'))
+
     steps:
-      # check URLs with Lychee
       - uses: actions/checkout@v6
+        with:
+          # Needed to diff base..head for the associated PR
+          fetch-depth: 0
+
+      - name: Resolve PR and deployment URL
+        id: pr-context
+        if: github.event_name == 'deployment_status'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const { owner, repo } = context.repo;
+            const sha = context.payload.deployment?.sha;
+            const deployUrl =
+              context.payload.deployment_status?.environment_url ||
+              context.payload.deployment_status?.target_url ||
+              '';
+
+            core.info(`Deployment SHA: ${sha}`);
+            core.info(`Deployment URL: ${deployUrl}`);
+
+            // Find PR(s) associated with this deployment commit SHA
+            const prsResp = await github.rest.repos.listPullRequestsAssociatedWithCommit({
+              owner,
+              repo,
+              commit_sha: sha,
+            });
+
+            const pr = prsResp.data?.[0];
+            if (!pr) {
+              core.warning(`No PR associated with commit ${sha}. Skipping linkcheck + PR comment.`);
+              core.setOutput('pr_number', '');
+              core.setOutput('deploy_url', deployUrl);
+              return;
+            }
+
+            core.info(`Associated PR: #${pr.number} (${pr.html_url})`);
+            core.setOutput('pr_number', String(pr.number));
+            core.setOutput('base_sha', pr.base.sha);
+            core.setOutput('head_sha', pr.head.sha);
+            core.setOutput('deploy_url', deployUrl);
+
+      - name: Get changed documentation files
+        id: changed-files
+        if: steps.pr-context.outputs.pr_number != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request'
+        uses: tj-actions/changed-files@v47.0.1
+        with:
+          base_sha: ${{ steps.pr-context.outputs.base_sha || github.event.pull_request.base.sha }}
+          sha: ${{ steps.pr-context.outputs.head_sha || github.event.pull_request.head.sha }}
+          files: |
+            **/*.md
+            **/*.mdx
 
       - name: Link Checker
         id: lychee
+        if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch'
         uses: lycheeverse/lychee-action@v2
         with:
-          args: "--threads 5 --max-retries 5 --retry-wait-time 2 --include '^https?://' --include '^http?://' --base-url='${{ github.event.deployment_status.environment_url }}' '${{ github.event.deployment_status.environment_url }}'"
-          format: markdown
           fail: false
-        env:
-          # to be used in case rate limits are surpassed
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          # Don't fail if no files to check
+          failIfEmpty: false
+          # Output format for reports
+          format: markdown
+          # GitHub token for API rate limiting
+          token: ${{ secrets.GITHUB_TOKEN }}
+          # Override base_url with deployment URL (if available) or use production
+          # For forks without Mintlify preview: checks against production site
+          args: >-
+            --base-url ${{ steps.pr-context.outputs.deploy_url || 'https://docs.wandb.ai' }}
+            ${{ steps.changed-files.outputs.all_changed_files || '.' }}
+
+      - name: Comment PR with link check results
+        if: (steps.pr-context.outputs.pr_number != '' || github.event_name == 'pull_request') && (steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch')
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const identifier = '<!-- lychee-link-checker-comment -->';
+
+            let commentBody = identifier + '\n';
+
+            const isFork = context.payload.pull_request?.head?.repo?.fork || false;
+            const deployUrl = '${{ steps.pr-context.outputs.deploy_url }}';
+
+            if (${{ steps.lychee.outputs.exit_code }} === 0) {
+              // Success - no broken links
+              commentBody += '## 🔗 Link Checker Results\n\n';
+              commentBody += '✅ **All links are valid!**\n\n';
+              commentBody += 'No broken links were detected in the changed files.\n';
+              if (isFork && !deployUrl) {
+                commentBody += '\n_Note: Checked against production site (https://docs.wandb.ai) since preview deployments are not available for forks._\n';
+              }
+
+              // Check if there were redirects in the report
+              try {
+                const report = fs.readFileSync('./lychee/out.md', 'utf8');
+                if (report.includes('Redirect') || report.includes('redirect')) {
+                  commentBody += '\n\n> [!TIP]\n';
+                  commentBody += '> **Redirects detected**: If you see redirects for internal docs.wandb.ai links, check if they have trailing slashes.\n';
+                  commentBody += '> \n';
+                  commentBody += '> Mintlify automatically removes trailing slashes, causing redirects like:\n';
+                  commentBody += '> - `https://docs.wandb.ai/models/` → `https://docs.wandb.ai/models`\n';
+                  commentBody += '> \n';
+                  commentBody += '> **Fix**: Remove trailing slashes from links to avoid unnecessary redirects.\n';
+                }
+              } catch (e) {
+                // Ignore if report file doesn't exist
+              }
+            } else {
+              // Issues found - include report
+              const report = fs.readFileSync('./lychee/out.md', 'utf8');
+
+              commentBody += '## 🔗 Link Checker Results\n\n';
+              commentBody += '> [!NOTE]\n';
+              if (isFork && !deployUrl) {
+                commentBody += '> This PR is from a fork, so links were checked against the **production site** (https://docs.wandb.ai).\n';
+                commentBody += '> \n';
+                commentBody += '> Links to **newly created files** in this PR will be reported as broken until the PR is merged.\n';
+              } else {
+                commentBody += '> Links to **newly created files** in this PR may be reported as broken because this checks links against the **preview deployment**.\n';
+              }
+              commentBody += '> \n';
+              commentBody += '> Warnings about **new** files in this PR can be safely ignored.\n\n';
+
+              // Add trailing slash tip if redirects are present
+              if (report.includes('Redirect') || report.includes('redirect')) {
+                commentBody += '> [!TIP]\n';
+                commentBody += '> **Redirects detected**: If you see redirects for internal docs.wandb.ai links, check if they have trailing slashes.\n';
+                commentBody += '> \n';
+                commentBody += '> Mintlify automatically removes trailing slashes, causing redirects like:\n';
+                commentBody += '> - `https://docs.wandb.ai/models/` → `https://docs.wandb.ai/models`\n';
+                commentBody += '> - `/weave/quickstart/` → `/weave/quickstart`\n';
+                commentBody += '> \n';
+                commentBody += '> **Fix**: Remove trailing slashes from links to avoid unnecessary redirects.\n\n';
+              }
+
+              commentBody += '---\n\n';
+              commentBody += report;
+            }
+
+            // Determine PR number
+            const prNumber = Number('${{ steps.pr-context.outputs.pr_number }}') || 
+                           context.payload.pull_request?.number;
+
+            if (!prNumber) {
+              core.info('No PR number available, skipping comment');
+              return;
+            }
+
+            // Find existing comment
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+            });
+
+            const existingComment = comments.find(comment =>
+              comment.body?.includes(identifier) && comment.user?.login === 'github-actions[bot]'
+            );
+
+            if (existingComment) {
+              // Update existing comment
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existingComment.id,
+                body: commentBody
+              });
+            } else {
+              // Create new comment
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body: commentBody
+              });
+            }

.github/workflows/linkcheck-prod.yml

@@ -1,30 +1,44 @@
-name: Lychee production link checker
+name: Link checker - production site
+
 on:
   repository_dispatch:
   workflow_dispatch:
   schedule:
-    - cron: "5 0 1 * *" # In UTC, currently 12:05 AM on the 1st of each month
+    - cron: "5 0 1 * *" # Monthly on the 1st at 12:05 AM UTC
 
 jobs:
   linkChecker:
     runs-on: ubuntu-latest
     permissions:
-      issues: write # required for peter-evans/create-issue-from-file
+      issues: write # Required for creating issues
     steps:
-      # check URLs with Lychee
       - uses: actions/checkout@v6
 
+      - name: Download and parse sitemap
+        run: |
+          echo "Fetching sitemap from https://docs.wandb.ai/sitemap.xml..."
+          curl -s https://docs.wandb.ai/sitemap.xml | \
+          grep -o '<loc>[^<]*</loc>' | \
+          sed 's/<loc>//g; s/<\/loc>//g' > urls.txt
+
+          URL_COUNT=$(wc -l < urls.txt | tr -d ' ')
+          echo "Found ${URL_COUNT} URLs in sitemap"
+
+          # Show first few URLs for verification
+          echo "Sample URLs:"
+          head -5 urls.txt
+
       - name: Link Checker
         id: lychee
         uses: lycheeverse/lychee-action@v2
         with:
-          args: "--threads 5 --max-retries 5 --retry-wait-time 2 --include '^https?://' --include '^http?://' --base-url='http://docs.wandb.ai' 'http://docs.wandb.ai'"
+          # Configuration is in lychee.toml
+          # Check all URLs from sitemap
+          args: "urls.txt"
           output: ./lychee-report.md
           format: markdown
           fail: false
-        env:
-          # to be used in case rate limits are surpassed
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create Issue From File
         if: steps.lychee.outputs.exit_code != 0

index.mdx

@@ -19,8 +19,8 @@ import {ProductCard} from "/snippets/ProductCard.jsx";
         Use W&B Models to manage AI model development. Features include training, fine-tuning, reporting, automating hyperparameter sweeps, and utilizing the model registry for versioning and reproducibility.
         <br/>
         <br/>
-        • <a href="/models/">Introduction</a><br/>
-        • <a href="/models/quickstart/">Quickstart</a><br/>
+        • <a href="/models">Introduction</a><br/>
+        • <a href="/models/quickstart">Quickstart</a><br/>
         • <a href="https://www.youtube.com/watch?v=tHAFujRhZLA">YouTube Tutorial</a><br/>
     </ProductCard>
 
@@ -33,35 +33,35 @@ import {ProductCard} from "/snippets/ProductCard.jsx";
         Use W&B Weave to manage AI models in your code. Features include tracing, output evaluation, cost estimates, and a hosted inference service and playground for comparing different large language models (LLMs) and settings.
         <br/>
         <br/>
-        • <a href="/weave/">Introduction</a><br/>
+        • <a href="/weave">Introduction</a><br/>
         • <a href="/weave/quickstart">Quickstart</a><br/>
         • <a href="https://www.youtube.com/watch?v=IQcGGNLN3zo">YouTube Demo</a><br/>
     </ProductCard>
 
     <ProductCard
         title="W&B Inference"
         iconSrc="icons/cropped-inference.svg"
-        href="/inference/"
+        href="/inference"
         subtitle="Access foundation models"
     >
         Use W&B Inference to access leading open-source foundation models through an OpenAI-compatible API. Features include multiple model options, usage tracking, and integration with Weave for tracing and evaluation.
         <br/>
         <br/>
-        • <a href="/inference/">Introduction</a><br/>
+        • <a href="/inference">Introduction</a><br/>
         • <a href="https://wandb.ai/inference">Try in Playground</a>
     </ProductCard>
 
     <ProductCard
         title="W&B Training"
         iconSrc="icons/cropped-training.svg"
-        href="/training/"
+        href="/training"
         subtitle="Post-train your models"
     >
         Now in public preview, use W&B Training to post-train large language models using serverless reinforcement learning (RL). Features include fully managed GPU infrastructure, integration with ART and RULER, and automatic scaling for multi-turn agentic tasks.
         <br/>
         <br/>
-        • <a href="/training/">Introduction</a><br/>
-        • <a href="/training/prerequisites/">Quickstart</a><br/>
+        • <a href="/training">Introduction</a><br/>
+        • <a href="/training/prerequisites">Quickstart</a><br/>
     </ProductCard>
 </div>
 </HomeWrapper>