Skip to content

Commit

Permalink
chore: translation data migrations (#643)
Browse files Browse the repository at this point in the history
# Pull Request type



Please check the type of change your PR introduces:

- [x] Bugfix - Link up nav menu pages
- [ ] Feature
- [ ] Code style update (formatting, renaming)
- [ ] Refactoring (no functional changes, no API changes)
- [ ] Build-related changes
- [ ] Documentation content changes
- [x] Other (please describe):
  - update translation keys in db
  - update translation keys on CrowdIn

## What is the current behavior?



Issue Number: N/A

## What is the new behavior?



-
-
-

## Does this introduce a breaking change?

- [ ] Yes
- [ ] No



## Other information




PR-URL: #643
Co-authored-by: Joe Karow <58997957+JoeKarow@users.noreply.github.com>
  • Loading branch information
kodiakhq[bot] and JoeKarow authored Jun 30, 2023
2 parents 54cfadd + 1aa5ecb commit 8e3fe66
Show file tree
Hide file tree
Showing 48 changed files with 568 additions and 131 deletions.
34 changes: 7 additions & 27 deletions apps/app/next-i18next.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,16 @@
/* eslint-disable node/no-process-env */
// @ts-check
/* eslint-disable import/no-unused-modules */
import axios from 'axios'
// import axios from 'axios'
import ChainedBackend from 'i18next-chained-backend'
import HttpBackend from 'i18next-http-backend'
import intervalPlural from 'i18next-intervalplural-postprocessor'
// import LocalStorageBackend from 'i18next-localstorage-backend'
import MultiBackend from 'i18next-multiload-backend-adapter'
import { z } from 'zod'
// import { z } from 'zod'

import path from 'path'

import { I18nextKeysOnDemand } from '@weareinreach/i18next-keys-ondemand'

export const namespaces = [
'attribute',
'common',
Expand All @@ -28,7 +26,7 @@ export const namespaces = [
]
const isBrowser = typeof window !== 'undefined'

const Keys = z.record(z.string())
// const Keys = z.record(z.string())

/**
* Gets full path based on environment
Expand All @@ -44,30 +42,12 @@ const getUrl = (path) => {
return `http://localhost:${process.env.PORT ?? 3000}${parsedPath}` // dev SSR should use localhost
}

/** @type {import('@weareinreach/i18next-keys-ondemand').TranslationGetter} */
const onDemandFetcher = async (keys, lang, ns, defaultValues) => {
console.log('fetcher called', { keys, lang, ns, defaultValues })
try {
const { data } = await axios.post(getUrl('/api/i18n/get'), { keys, lang, ns, defaultValues })
const translations = Keys.parse(data)
return translations
} catch (err) {
console.error(err)
return {}
}
}

const onDemand = new I18nextKeysOnDemand({
translationGetter: onDemandFetcher,
debounceDelay: 50,
})

// const crowdinBackend = new CrowdinOtaBackend(undefined, )
const apiPath = '/api/i18n/load?lng={{lng}}&ns={{ns}}'
const httpBackend = new HttpBackend(null, {
loadPath: getUrl(apiPath), //typeof window !== 'undefined' ? apiPath : `http://localhost:3000${apiPath}`,
allowMultiLoading: true,
})
// const httpBackend = new HttpBackend(null, {
// loadPath: getUrl(apiPath), //typeof window !== 'undefined' ? apiPath : `http://localhost:3000${apiPath}`,
// allowMultiLoading: true,
// })

const multi = new MultiBackend(null, {
backend: HttpBackend,
Expand Down
3 changes: 2 additions & 1 deletion apps/app/nextjs-routes.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ declare module "nextjs-routes" {

export type Route =
| StaticRoute<"/account">
| StaticRoute<"/account/reviews">
| StaticRoute<"/account/saved">
| StaticRoute<"/admin/quicklink/email">
| StaticRoute<"/admin/quicklink">
| StaticRoute<"/admin/quicklink/phone">
Expand All @@ -29,7 +31,6 @@ declare module "nextjs-routes" {
| DynamicRoute<"/org/[slug]", { "slug": string }>
| DynamicRoute<"/org/[slug]/remote", { "slug": string }>
| StaticRoute<"/profile">
| StaticRoute<"/saved">
| DynamicRoute<"/search/[...params]", { "params": string[] }>
| StaticRoute<"/search">
| StaticRoute<"/suggest">
Expand Down
1 change: 0 additions & 1 deletion apps/app/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@
"@weareinreach/auth": "workspace:*",
"@weareinreach/db": "workspace:*",
"@weareinreach/env": "workspace:*",
"@weareinreach/i18next-keys-ondemand": "0.3.2",
"@weareinreach/ui": "workspace:*",
"axios": "1.4.0",
"cookies-next": "2.1.2",
Expand Down
53 changes: 53 additions & 0 deletions apps/app/src/pages/account/reviews.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import { Center, Grid, Loader, Overlay, Stack, Title } from '@mantine/core'
import { type GetServerSideProps, InferGetServerSidePropsType } from 'next'
import { useRouter } from 'next/router'
import { useSession } from 'next-auth/react'
import { useTranslation } from 'next-i18next'

import { getServerSession } from '@weareinreach/auth'
import { QuickPromotionModal } from '@weareinreach/ui/modals'
import { getServerSideTranslations } from '~app/utils/i18n'

const Reviews = () => {
const { t } = useTranslation('common')
const { data: session, status } = useSession()
const router = useRouter()
if (status === 'loading') {
return (
<Center>
<Loader />
</Center>
)
}
if (status === 'unauthenticated' || session === null) {
return (
<Overlay blur={2}>
<QuickPromotionModal autoLaunch onClose={() => router.replace('/')} />
</Overlay>
)
}

return (
<Grid.Col xs={12} sm={12}>
<Stack h='100vh' align='flex-start' w='100%'>
<Title order={2}>{t('words.reviews')}</Title>
<Stack spacing={0} align='center' justify='center' h='100%' w='100%'>
<Title order={2}>🚧</Title>
<Title order={2}>{t('words.coming-soon')}</Title>
</Stack>
</Stack>
</Grid.Col>
)
}

export const getServerSideProps: GetServerSideProps = async ({ locale, req, res }) => {
const session = await getServerSession({ req, res })

return {
props: {
session,
...(await getServerSideTranslations(locale, ['common'])),
},
}
}
export default Reviews
File renamed without changes.
22 changes: 8 additions & 14 deletions apps/app/src/pages/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -315,27 +315,21 @@ Home.omitGrid = true

export const getStaticProps: GetStaticProps = async ({ locale }) => {
const ssg = await trpcServerClient({ session: null })
await ssg.review.getFeatured.prefetch(3)

const [i18n] = await Promise.allSettled([
getServerSideTranslations(locale, ['common', 'landingPage', 'attribute']),
ssg.review.getFeatured.prefetch(3),
])

// await ssg.review.getFeatured.prefetch(3)
return {
props: {
trpcState: ssg.dehydrate(),
...(await getServerSideTranslations(locale, ['common', 'landingPage', 'attribute'])),
// ...(await getServerSideTranslations(locale, ['common', 'landingPage', 'attribute'])),
...(i18n.status === 'fulfilled' ? i18n.value : {}),
},
revalidate: 60 * 60 * 24, // 24 hours
}
}
// export const getServerSideProps = async ({ locale, req, res }: GetServerSidePropsContext) => {
// const session = await getServerSession({ req, res })
// const ssg = await trpcServerClient({ session })
// await ssg.review.getFeatured.prefetch(3)

// return {
// props: {
// session,
// trpcState: ssg.dehydrate(),
// ...(await getServerSideTranslations(locale, ['common', 'landingPage'])),
// },
// }
// }
export default Home
8 changes: 6 additions & 2 deletions apps/app/src/pages/org/[slug]/[orgLocationId]/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -187,10 +187,14 @@ export const getStaticProps: GetStaticProps = async ({ locale, params }) => {
const { slug, orgLocationId } = urlParams.data

const ssg = await trpcServerClient({ session: null })

const orgId = await ssg.organization.getIdFromSlug.fetch({ slug })
if (!orgId?.id) return { notFound: true }

const [i18n] = await Promise.allSettled([
getServerSideTranslations(locale, ['common', 'services', 'attribute', 'phone-type', slug]),
getServerSideTranslations(locale, ['common', 'services', 'attribute', 'phone-type', orgId.id]),
ssg.organization.getBySlug.prefetch({ slug }),
ssg.organization.getIdFromSlug.prefetch({ slug }),
// ssg.organization.getIdFromSlug.prefetch({ slug }),
ssg.location.forLocationPage.prefetch({ id: orgLocationId }),
ssg.organization.forLocationPage.prefetch({ slug }),
])
Expand Down
5 changes: 4 additions & 1 deletion apps/app/src/pages/org/[slug]/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -254,15 +254,18 @@ export const getStaticProps: GetStaticProps<Record<string, unknown>, RoutedQuery
const { slug } = params

const ssg = await trpcServerClient({ session: null })
const orgId = await ssg.organization.getIdFromSlug.fetch({ slug })
if (!orgId?.id) return { notFound: true }

const [i18n] = await Promise.allSettled([
getServerSideTranslations(locale, ['common', 'services', 'attribute', 'phone-type', slug]),
getServerSideTranslations(locale, ['common', 'services', 'attribute', 'phone-type', orgId.id]),
ssg.organization.forOrgPage.prefetch({ slug }),
])
// await ssg.organization.getBySlug.prefetch({ slug })

const props = {
trpcState: ssg.dehydrate(),
organizationId: orgId.id,
// ...(await getServerSideTranslations(locale, ['common', 'services', 'attribute', 'phone-type', slug])),
...(i18n.status === 'fulfilled' ? i18n.value : {}),
}
Expand Down
7 changes: 5 additions & 2 deletions apps/app/src/pages/org/[slug]/remote.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -151,10 +151,13 @@ export const getStaticProps: GetStaticProps<
const { slug } = params

const ssg = await trpcServerClient({ session: null })
const orgId = await ssg.organization.getIdFromSlug.fetch({ slug })
if (!orgId?.id) return { notFound: true }

const [i18n] = await Promise.allSettled([
getServerSideTranslations(locale, ['common', 'services', 'attribute', 'phone-type', slug]),
getServerSideTranslations(locale, ['common', 'services', 'attribute', 'phone-type', orgId.id]),
ssg.organization.getNameFromSlug.prefetch(slug),
ssg.organization.getIdFromSlug.prefetch({ slug }),
// ssg.organization.getIdFromSlug.prefetch({ slug }),
])
const props = {
trpcState: ssg.dehydrate(),
Expand Down
1 change: 0 additions & 1 deletion apps/app/src/pages/suggest.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import { useSession } from 'next-auth/react'
import { useState } from 'react'

import { trpcServerClient } from '@weareinreach/api/trpc'
import { getServerSession } from '@weareinreach/auth'
import { SuggestOrg } from '@weareinreach/ui/components/sections/SuggestOrg'
import { QuickPromotionModal } from '@weareinreach/ui/modals'
import { getServerSideTranslations } from '~app/utils/i18n'
Expand Down
2 changes: 1 addition & 1 deletion apps/web/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"version": "0.100.0",
"private": true,
"scripts": {
"build": "next build",
"buildx": "next build",
"clean:node": "rm -rf ./node_modules/ || true && rm -rf ./.next || true",
"dev": "next dev",
"preinstall": "npx only-allow pnpm",
Expand Down
1 change: 1 addition & 0 deletions packages/api/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
crowdin/generated/*.json
30 changes: 30 additions & 0 deletions packages/api/cache/slugToOrgId.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import { kv as redis } from '@vercel/kv'
import { Logger } from 'tslog'

const log = new Logger({ name: 'Cache - Slug to OrgId' })

export const readSlugCache = async (slug: string) => {
try {
if ((await redis.ping()) !== 'PONG') {
log.warn('Skipping cache read - Redis client not connected')
return null
}
const organizationId = await redis.hget<string>('slugToOrgId', slug)
return organizationId
} catch (err) {
log.error(err)
return null
}
}

export const writeSlugCache = async (slug: string, organizationId: string) => {
try {
if ((await redis.ping()) !== 'PONG') {
log.warn('Skipping cache write - Redis client not connected')
return
}
await redis.hset('slugToOrgId', { [slug]: organizationId })
} catch (err) {
log.error(err)
}
}
Empty file.
15 changes: 9 additions & 6 deletions packages/api/crowdin/getIdsForDb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import { prisma } from '@weareinreach/db'
import { getStringIdByKey } from '.'

const queue = new PQueue({
concurrency: 20,
intervalCap: 20,
concurrency: 15,
intervalCap: 15,
interval: 500,
autoStart: false,
carryoverConcurrencyCount: true,
Expand All @@ -36,9 +36,9 @@ const output: { crowdinId: number; key: string; ns: string }[] = []
// queue.on('completed', () => {
// console.info(`--> Job #${qcount} finished in ${Date.now() - qtimer}ms`)
// })
// queue.on('error', (error) => {
// console.error(error)
// })
queue.on('error', (error) => {
console.error(error)
})
queue.on('idle', () => {
// console.info(`<==> Writing data to file.`)
fs.writeFileSync(path.resolve(__dirname, './generated/out.json'), JSON.stringify(output))
Expand All @@ -59,7 +59,10 @@ const task = async (key: string, ns: string) => {
}
let recordTotal = 0
const run = async () => {
const records = await prisma.translationKey.findMany({ select: { key: true, ns: true } })
const records = await prisma.translationKey.findMany({
where: { ns: 'org-data', crowdinId: null },
select: { key: true, ns: true },
})
recordTotal = records.length
for (const record of records) {
const { key, ns } = record
Expand Down
75 changes: 75 additions & 0 deletions packages/api/crowdin/temp.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import fs from 'fs'
import path from 'path'

import { prisma } from '@weareinreach/db'
import { crowdinApi } from '~api/crowdin/client'

interface KeyData {
key: string
crowdinId: number
slug: string
}

interface TestData {
id: number
key: string
}

const run = async () => {
const data = await prisma.translationKey.findMany({
where: { ns: 'org-data' },
select: { key: true, crowdinId: true },
})

const batchSize = 100
const totalBatches = Math.ceil(data.length / batchSize)
let batchCount = 1
while (data.length) {
const batch = data.splice(0, batchSize)

console.log(`[${batchCount}/${totalBatches}] Processing ${batch.length} items.`)

const result = await crowdinApi.sourceStringsApi.stringBatchOperations(
12,
batch.map(({ key, crowdinId }) => ({
op: 'replace',
path: `/${crowdinId}/identifier`,
value: key,
}))
)
// const result = { data: batch }
console.log(`\tResult count: ${result.data.length}`)
batchCount++
}

// const updates = await crowdinApi.sourceStringsApi.stringBatchOperations(
// 12,
// data.map(({ key, crowdinId, slug }) => ({
// op: 'replace',
// path: `/${crowdinId}/context`,
// value: `https://app.inreach.org/org/${slug}`,
// }))
// )
// console.log(updates)
}

run()

const test = async () => {
// const out: Record<string, string> = {}
// for (let i = 0; i < 5000; i++) {
// out[`key-${i.toString().padStart(4, '0')}`] = `Text item ${i.toString().padStart(4, '0')}`
// }
const data: unknown[] = []
for (let i = 0; i < 10; i++) {
console.log(`Batch ${i + 1}`)
const result = await crowdinApi.sourceStringsApi.listProjectStrings(14, {
fileId: 3141,
limit: 500,
offset: 500 * i,
})
data.push(...result.data.map(({ data }) => ({ id: data.id, key: data.identifier })))
}
fs.writeFileSync('data.json', JSON.stringify(data))
}
// test()
Loading

0 comments on commit 8e3fe66

Please sign in to comment.