Extract data (#16)

* refactor(periodic-data): extract periodic data into separate package

* ci(*): set up proper ci

* fix(ci): fix faulty commands and its order

* fix(ci/generate-data): pass `secrets` as environment variables

* fix(ci/command): fix `Unrecognized named-value: 'secrests'`

* fix(apps/web/lint): fix `knip` lint issue

* fix(ci): generated web stuffs

* fix(ci): generate schema and images

* fix(ci/run): split command

* fix(ci/run): chain command with two ampersand

* fix(apps/web/schema): create directory if not created before writing
schema

* fix(ci): generated web page instead of web stuffs

* fix(ci): generate web pages first before schema

* fix(apps/web): generate dummy schema file before generating real one to
solve chicken and egg problem

* fix(apps/web): terminate schema generation upon completion

* fix(packages/data): added `typecheck` command

* fix(packages/data): parse environment variables as string

* fix(apps/data): added format commands

* fix(apps/web): formatted all files with `prettier`

* fix(apps/web): ignore cache

* fix(apps/web): fixed `pretest-ci` puppeteer installation command

* test(apps/web): increase snapshot timeout

* test(apps/web): reduce number of test files during CI

.github/workflows/action.yaml

@@ -27,14 +27,45 @@ jobs:
             - name: Install Depedencies
               run: pnpm i
 
+            - name: Generate Data
+              working-directory: packages/data
+              env:
+                  OWNER: ${{ secrets.OWNER }}
+                  TOKEN: ${{ secrets.TOKEN }}
+                  REPO: ${{ secrets.REPO }}
+                  DATA_PATH: ${{ secrets.DATA_PATH }}
+              run: make generate-data
+
+            - name: Create Environment Variable
+              working-directory: apps/web
+              run: make copy-env-testing
+
+            - name: Generate Env Type Definitions
+              working-directory: apps/web
+              run: make generate-environment-type-definition
+
+            - name: Generate Web Pages
+              working-directory: apps/web
+              run: make generate-web-pages
+
+            - name: Generate Seo Schema
+              working-directory: apps/web
+              run: make generate-schema
+
+            - name: Generate Assets
+              working-directory: apps/web
+              run: make generate-images
+
             - name: Lint code
+              working-directory: apps/web
               run: make lint
 
             - name: Typecheck code
               run: make typecheck
 
-            - name: Generate prettier config
-              run: make format-generate-config
-
             - name: Format Check
               run: make format-check
+
+            - name: Build & Test code
+              working-directory: apps/web
+              run: make pretest-ci && make test

Makefile

@@ -5,6 +5,9 @@ format-write:
 format-check:
 	pnpm --stream -r format-check
 
+typecheck:
+	pnpm --stream -r typecheck
+
 # web deployment
 web-deploy-production:
 	vercel --prod

apps/web/.prettierignore

@@ -13,6 +13,7 @@ public/workbox-*.js*
 src/web/generated/*.ts
 env.d.ts
 public/images/generated
+cache/config.json
 
 # next custom generated files
 next.config.mjs

apps/web/Makefile

@@ -118,4 +118,7 @@ test-snapshot: clear-snapshot
 	pnpm vite-node script/test/snapshot.ts &&\
 		make test-type path="snapshot" arguments="$(arguments)"
 
+pretest-ci:
+	node node_modules/puppeteer/install.mjs
+
 test: build-testing test-unit test-snapshot

apps/web/knip.ts

@@ -8,8 +8,18 @@ const config: KnipConfig = {
 		'src/**/*.ts',
 		'script/**/*.ts',
 	],
-	ignore: ['next-sitemap.config.js'],
-	ignoreDependencies: ['vite-node', 'next-sitemap', 'prettier', 'eslint'],
+	ignore: ['next-sitemap.config.js', 'next/**.mjs', 'test/**/**.ts'],
+	ignoreBinaries: ['make'],
+	ignoreDependencies: [
+		'vite-node',
+		'next-sitemap',
+		'eslint',
+		'@periotable/data',
+		'@ducanh2912/next-pwa',
+		'@types/jest-image-snapshot',
+		'jest-image-snapshot',
+		'puppeteer',
+	],
 };
 
 export default config;

apps/web/package.json

@@ -8,13 +8,15 @@
 	"scripts": {
 		"postinstall": "make opt-out-telemetry",
 		"format-check": "make format-check",
-		"format-write": "make format-write"
+		"format-write": "make format-write",
+		"typecheck": "make typecheck"
 	},
 	"dependencies": {
 		"@emotion/react": "^11.11.4",
 		"@emotion/styled": "^11.11.5",
 		"@fontsource-variable/jetbrains-mono": "^5.0.20",
 		"@mui/joy": "5.0.0-beta.32",
+		"@periotable/data": "workspace:^",
 		"@poolofdeath20/util": "^0.8.0",
 		"bowser": "^2.11.0",
 		"next": "^14.1.4",

apps/web/pages/compounds/index.tsx

@@ -4,7 +4,8 @@ import Box from '@mui/joy/Box';
 
 import { Optional } from '@poolofdeath20/util';
 
-import data from '../../src/web/generated/data';
+import data from '@periotable/data';
+
 import useSearchQuery from '../../src/web/hooks/search';
 import Seo from '../../src/web/components/seo';
 import ListOfCompounds, {

apps/web/pages/elements/[name]/[section].tsx

@@ -2,7 +2,8 @@ import type { GetStaticPaths, GetStaticProps } from 'next';
 
 import { Defined } from '@poolofdeath20/util';
 
-import data from '../../../src/web/generated/data';
+import data from '@periotable/data';
+
 import Element, {
 	listOfPropertiesTitle,
 	titleToId,

apps/web/pages/elements/[name]/index.tsx

@@ -23,7 +23,8 @@ import {
 	Defined,
 } from '@poolofdeath20/util';
 
-import data from '../../../src/web/generated/data';
+import data from '@periotable/data';
+
 import Seo from '../../../src/web/components/seo';
 import BohrTwoDimensional from '../../../src/web/components/bohr/two-dimensional';
 import BohrThreeDimensional from '../../../src/web/components/bohr/three-dimensional';

apps/web/script/assets/images.ts

@@ -4,7 +4,8 @@ import axios from 'axios';
 
 import { isNotUndefined } from '@poolofdeath20/util';
 
-import data from '../../src/web/generated/data';
+import data from '@periotable/data';
+
 import constants from '../../src/web/constant';
 import { obtainNameFromUrl } from '../../src/web/util/asset';
 

apps/web/script/seo/schema.ts

@@ -1,14 +1,26 @@
 import fs from 'fs';
 
-import { generatePaths } from '../../test/snapshot/data';
+const main = async () => {
+	if (!fs.existsSync('src/web/generated')) {
+		fs.mkdirSync('src/web/generated');
+	}
 
-const main = () => {
-	const paths = generatePaths();
+	// create a dummy file to solve chicken and egg problem
+	fs.writeFileSync(
+		'src/web/generated/schema.ts',
+		`const paths = [] as ReadonlyArray<string>\n; export default paths;`
+	);
+
+	const paths = await import('../../test/snapshot/data').then((data) => {
+		return data.generatePaths();
+	});
 
 	fs.writeFileSync(
 		'src/web/generated/schema.ts',
-		`const paths = ${JSON.stringify(paths, undefined, 4)}; export default paths;`
+		`const paths = ${JSON.stringify(paths, undefined, 4)}\n; export default paths;`
 	);
+
+	process.exit(0);
 };
 
 main();

apps/web/script/test/snapshot.ts

@@ -1,5 +1,7 @@
 import fs from 'fs';
 
+import ci from 'ci-info';
+
 import type { DeepReadonly } from '@poolofdeath20/util';
 
 import { generatePaths } from '../../test/snapshot/data';
@@ -23,7 +25,7 @@ const main = () => {
 	// which should not include error page
 	const paths = generatePaths().concat('/error');
 
-	const numberOfParallelTests = 10;
+	const numberOfParallelTests = ci.isCI ? 3 : 10;
 
 	const numberOfTests = Math.ceil(paths.length / numberOfParallelTests);
 

apps/web/src/web/components/pages/index/common.tsx

@@ -24,7 +24,8 @@ import {
 
 import { CgClose } from 'react-icons/cg';
 
-import data from '../../../generated/data';
+import data from '@periotable/data';
+
 import Seo from '../../../components/seo';
 import { DemoTile, EmptyTile, Tile } from '../../../components/table/element';
 import SearchBar from '../../../components/common/input';

apps/web/vitest.config.ts

@@ -7,7 +7,7 @@ import { defineConfig } from 'vitest/config';
 import { Defined } from '@poolofdeath20/util';
 
 export default defineConfig(() => {
-	const timeOut = 300_000;
+	const timeOut = ci.isCI ? 6_000_000 : 300_000;
 
 	return {
 		clearScreen: ci.isCI,
@@ -16,23 +16,19 @@ export default defineConfig(() => {
 			testTimeout: timeOut,
 			hookTimeout: timeOut,
 			teardownTimeout: timeOut,
-			env: ci.isCI
-				? undefined
-				: fs
-						.readFileSync('.env', {
-							encoding: 'utf-8',
-						})
-						.split('\n')
-						.filter(Boolean)
-						.reduce((prev, keyValuePair) => {
-							const [key, value] = keyValuePair.split('=');
-							return {
-								...prev,
-								[Defined.parse(key).orThrow(
-									'key is undefined'
-								)]: value,
-							};
-						}, {}),
+			env: fs
+				.readFileSync('.env', {
+					encoding: 'utf-8',
+				})
+				.split('\n')
+				.filter(Boolean)
+				.reduce((prev, keyValuePair) => {
+					const [key, value] = keyValuePair.split('=');
+					return {
+						...prev,
+						[Defined.parse(key).orThrow('key is undefined')]: value,
+					};
+				}, {}),
 		},
 	};
 });

packages/data/.gitignore

@@ -0,0 +1,3 @@
+index.ts
+.env
+env.d.ts

packages/data/.prettierignore

@@ -0,0 +1,3 @@
+index.ts
+.env
+env.d.ts

packages/data/Makefile

@@ -0,0 +1,20 @@
+# .env
+generate-environment-type-definition:
+	pnpm vite-node script/type-def.ts
+
+generate-data:
+	pnpm vite-node script/data.ts
+
+# format
+format:
+	pnpm prettier --$(type) .
+
+format-check:
+	make format type=check
+
+format-write:
+	make format type=write
+
+# typecheck
+typecheck:
+	pnpm tsc -p tsconfig.json

packages/data/package.json

@@ -0,0 +1,20 @@
+{
+	"name": "@periotable/data",
+	"version": "0.0.0",
+	"author": "PoolOfDeath20",
+	"private": true,
+	"license": "GPL",
+	"scripts": {
+		"format-check": "make format-check",
+		"format-write": "make format-write",
+		"typecheck": "make typecheck"
+	},
+	"devDependencies": {
+		"@poolofdeath20/tsconfig": "^0.0.0",
+		"axios": "^1.6.8",
+		"dotenv": "^16.4.5",
+		"gen-env-type-def": "^0.0.4",
+		"octokit": "^3.2.0",
+		"vite-node": "^1.4.0"
+	}
+}

packages/data/script/data.ts

@@ -0,0 +1,44 @@
+import fs from 'fs';
+
+import { Octokit } from 'octokit';
+
+import dotenv from 'dotenv';
+
+import axios from 'axios';
+
+const asString = <Str extends string>(string: Str | undefined) => {
+	if (string) {
+		return string;
+	}
+
+	throw new Error('');
+};
+
+const main = async () => {
+	dotenv.config();
+
+	const octokit = new Octokit({
+		auth: process.env.TOKEN,
+	});
+
+	const result = await octokit.rest.repos.getContent({
+		owner: asString(process.env.OWNER),
+		repo: asString(process.env.REPO),
+		path: asString(process.env.DATA_PATH),
+	});
+
+	// @ts-expect-error: Download URL exists because data is of type "file" but it doesn't have type
+	const url = result.data.download_url as string;
+
+	const content = await axios.get(url).then((response) => {
+		if (typeof response.data !== 'string') {
+			throw new Error('Data is not a string');
+		}
+
+		return response.data;
+	});
+
+	fs.writeFileSync('index.ts', content);
+};
+
+main();

packages/data/script/type-def.ts

@@ -0,0 +1,12 @@
+import { genEnvTypeDef } from 'gen-env-type-def';
+
+const main = () => {
+	genEnvTypeDef([
+		{
+			inDir: '.',
+			envType: 'process.env',
+		},
+	]);
+};
+
+main();

packages/data/tsconfig.json

@@ -0,0 +1,5 @@
+{
+	"extends": ["@poolofdeath20/tsconfig/node"],
+	"include": ["*.d.ts", "**/*.ts"],
+	"exclude": ["node_modules"]
+}