chore: check-in work

crawl.js

@@ -0,0 +1,83 @@
+import fs from 'node:fs/promises'
+import got from 'got'
+import {parse} from 'node-html-parser'
+
+async function main() {
+  const queue = ['https://primer.style/']
+  const visited = new Set()
+  const failed = new Set()
+  const pages = new Map()
+
+  while (queue.length > 0) {
+    const url = queue.shift()
+    if (visited.has(url)) {
+      continue
+    }
+
+    console.log('Visiting %s (%s visited, %s remaining)', url, visited.size, queue.length)
+
+    visited.add(url)
+
+    try {
+      const response = await got(url)
+      console.log(response.redirectUrls)
+      const root = parse(response.body)
+      const links = root.querySelectorAll('a').flatMap(link => {
+        const href = link.getAttribute('href')
+
+        if (!href) {
+          return []
+        }
+
+        if (href.startsWith('http')) {
+          return href
+        }
+
+        if (href.includes('*')) {
+          return []
+        }
+
+        if (href.startsWith('/') || href.startsWith('.')) {
+          const normalized = new URL(href, url)
+          return `https://${normalized.host}${normalized.pathname}`
+        }
+
+        return []
+      })
+
+      pages.set(url, [])
+
+      for (const link of links) {
+        if (
+          link.startsWith('https://primer.style/view-components/lookbook') ||
+          link.startsWith('https://primer.style/react/storybook') ||
+          link.startsWith('https://primer.style/css/storybook')
+        ) {
+          continue
+        }
+
+        if (link.startsWith('https://primer.style')) {
+          const url = new URL(link)
+          const bareUrl = `https://${url.host}${url.pathname}`
+          if (!visited.has(bareUrl) && !queue.includes(bareUrl)) {
+            queue.push(bareUrl)
+            pages.get(url).push(bareUrl)
+          }
+        } else {
+          // console.log('Skipping link: %s', link)
+        }
+      }
+    } catch (error) {
+      failed.add(url)
+      console.log(error)
+    }
+  }
+
+  // await fs.writeFile('pages.json', JSON.stringify(Object.fromEntries(pages), null, 2))
+  // await fs.writeFile('failed.json', JSON.stringify(Object.fromEntries(pages), null, 2))
+}
+
+main().catch(error => {
+  console.log(error)
+  process.exit(1)
+})

examples/nextjs/content/components/banner.mdx

@@ -59,4 +59,5 @@ If you're unsure about which technique is appropriate for your usecase, please r
 ### Known accessibility issues (GitHub staff only)
 
 {' '}
+
 <AccessibilityLink label="Banner" />

examples/nextjs/develop.js

@@ -0,0 +1,77 @@
+import {createServer} from 'node:http'
+import {parse} from 'node:url'
+import {WebSocketServer} from 'ws'
+import next from 'next'
+import chokidar from 'chokidar'
+
+const hostname = 'localhost'
+const port = 3000
+const app = next({
+  dev: true,
+  hostname,
+  port,
+})
+const handler = app.getRequestHandler()
+const wss = new WebSocketServer({noServer: true})
+const watchCallbacks = []
+
+const watcher = chokidar.watch('./content', {
+  ignored: /(^|[\/\\])\../,
+})
+
+watcher.on('change', event => {
+  watchCallbacks.forEach(callback => {
+    callback()
+  })
+})
+
+wss.on('connection', function connection(ws) {
+  console.log('connection open')
+  ws.on('error', console.error)
+
+  ws.on('close', function close() {
+    console.log('closed')
+    const index = watchCallbacks.findIndex(onChange)
+    watchCallbacks.splice(index, 1)
+  })
+
+  watchCallbacks.push(onChange)
+
+  function onChange() {
+    ws.send('refresh')
+  }
+})
+
+app.prepare().then(() => {
+  const server = createServer(async (req, res) => {
+    try {
+      // Be sure to pass `true` as the second argument to `url.parse`.
+      // This tells it to parse the query portion of the URL.
+      const parsedUrl = parse(req.url, true)
+      await handler(req, res, parsedUrl)
+    } catch (err) {
+      console.error('Error occurred handling', req.url, err)
+      res.statusCode = 500
+      res.end('internal server error')
+    }
+  })
+
+  server.once('error', err => {
+    console.error(err)
+    process.exit(1)
+  })
+
+  server.listen(port, () => {
+    console.log(`> Ready on http://${hostname}:${port}`)
+  })
+
+  server.on('upgrade', function upgrade(request, socket, head) {
+    const {pathname} = parse(request.url, true)
+
+    if (pathname === '/_doctocat/refresh-page') {
+      wss.handleUpgrade(request, socket, head, function done(ws) {
+        wss.emit('connection', ws, request)
+      })
+    }
+  })
+})

examples/nextjs/next.config.js

@@ -1,10 +1,32 @@
 import {doctocat} from '@primer/doctocat-nextjs/config'
+import {createRequire} from 'module'
 
+const require = createRequire(import.meta.url)
 const withDoctocat = doctocat()
 
 /**
  * @type {import('next').NextConfig}
  */
-const config = {}
+const config = {
+  // pageExtensions: ['ts', 'tsx', 'md', 'mdx'],
+  // webpack(config, options) {
+  // const loader = {
+  // loader: require.resolve('./src/loader/index.cjs'),
+  // options: {
+  // providerImportSource: 'next-mdx-import-source-file',
+  // },
+  // }
+  // config.resolve.alias['next-mdx-import-source-file'] = [
+  // 'private-next-root-dir/src/mdx-components',
+  // 'private-next-root-dir/mdx-components',
+  // '@mdx-js/react',
+  // ]
+  // config.module.rules.push({
+  // test: /\.mdx$/,
+  // use: [options.defaultLoaders.babel, loader],
+  // })
+  // return config
+  // },
+}
 
 export default withDoctocat(config)

examples/nextjs/package.json

@@ -10,14 +10,26 @@
     "type-check": "tsc --noEmit"
   },
   "dependencies": {
+    "@mdx-js/loader": "^3.0.1",
+    "@mdx-js/mdx": "^3.0.1",
+    "@primer/doctocat-content": "0.0.0",
     "@primer/doctocat-nextjs": "0.0.0",
+    "@primer/octicons-react": "^19.9.0",
+    "@primer/primitives": "^8.2.0",
+    "@primer/react": "^36.19.0",
+    "@primer/react-brand": "^0.34.0",
     "next": "^14.2.3",
+    "next-mdx-remote": "^5.0.0",
     "react": "^18.3.1",
-    "react-dom": "^18.3.1"
+    "react-dom": "^18.3.1",
+    "styled-components": "^5.3.11",
+    "ws": "^8.17.0"
   },
   "devDependencies": {
     "@types/node": "^20.12.12",
     "@types/react": "^18.3.2",
+    "@types/styled-components": "^5.1.34",
+    "chokidar": "^3.6.0",
     "rimraf": "^5.0.7",
     "typescript": "^5.4.5"
   }

examples/nextjs/src/app/[...slug]/page.tsx

@@ -0,0 +1,28 @@
+import {getPages, getPageBySlug} from '@primer/doctocat-content'
+import path from 'node:path'
+import process from 'node:process'
+
+interface PageProps {
+  params: {
+    slug: Array<string>
+  }
+}
+
+export default async function Page({params}: PageProps) {
+  const directory = path.join(process.cwd(), 'content')
+  const page = await getPageBySlug(directory, params.slug)
+  const relativePath = path.relative(directory, page.filepath)
+  const {default: MDXContent, frontmatter} = await import(`../../../content/${relativePath}`)
+
+  return <MDXContent components={{}} />
+}
+
+export async function generateStaticParams(): Promise<Array<PageProps['params']>> {
+  const directory = path.join(process.cwd(), 'content')
+  const pages = await getPages(directory)
+  return pages.map(page => {
+    return {
+      slug: page.getSlug(),
+    }
+  })
+}

examples/nextjs/src/app/about/page.tsx

@@ -0,0 +1,3 @@
+export default function AboutPage() {
+  return 'about'
+}