Merge pull request #64 from krystxf/feat/be-route-stops

feat: routes endpoint

apps/backend/package.json

@@ -25,6 +25,7 @@
     },
     "dependencies": {
         "@apollo/server": "^4.11.2",
+        "@fast-csv/parse": "^5.0.2",
         "@nestjs/apollo": "^12.2.1",
         "@nestjs/cache-manager": "^2.3.0",
         "@nestjs/common": "^10.4.13",
@@ -41,6 +42,7 @@
         "radash": "^12.1.0",
         "reflect-metadata": "^0.2.2",
         "rxjs": "^7.8.1",
+        "unzipper": "^0.12.3",
         "zod": "^3.23.8"
     },
     "devDependencies": {
@@ -51,6 +53,7 @@
         "@types/jest": "^29.5.14",
         "@types/node": "^20.17.9",
         "@types/supertest": "^6.0.2",
+        "@types/unzipper": "^0.10.10",
         "@typescript-eslint/eslint-plugin": "^6.21.0",
         "@typescript-eslint/parser": "^6.21.0",
         "eslint": "^8.57.1",

apps/backend/prisma/migrations/20241205183812_gtfs_routes/migration.sql

@@ -0,0 +1,33 @@
+-- CreateTable
+CREATE TABLE "GtfsRoute" (
+    "id" TEXT NOT NULL,
+    "type" TEXT NOT NULL,
+    "shortName" TEXT NOT NULL,
+    "longName" TEXT,
+    "url" TEXT,
+    "color" TEXT,
+    "isNight" BOOLEAN,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "GtfsRoute_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "GtfsRouteStop" (
+    "id" TEXT NOT NULL,
+    "routeId" TEXT NOT NULL,
+    "directionId" TEXT NOT NULL,
+    "stopId" TEXT NOT NULL,
+    "stopSequence" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "GtfsRouteStop_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "GtfsRouteStop_routeId_directionId_stopId_stopSequence_key" ON "GtfsRouteStop"("routeId", "directionId", "stopId", "stopSequence");
+
+-- AddForeignKey
+ALTER TABLE "GtfsRouteStop" ADD CONSTRAINT "GtfsRouteStop_routeId_fkey" FOREIGN KEY ("routeId") REFERENCES "GtfsRoute"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

apps/backend/prisma/schema.prisma

@@ -71,6 +71,36 @@ model PlatformsOnRoutes {
   @@unique([platformId, routeId])
 }
 
+model GtfsRoute {
+  id        String   @id
+  type      String
+  shortName String
+  longName  String?
+  url       String?
+  color     String?
+  isNight   Boolean?
+
+  createdAt     DateTime        @default(now())
+  updatedAt     DateTime        @updatedAt
+  GtfsRouteStop GtfsRouteStop[]
+}
+
+model GtfsRouteStop {
+  id String @id @default(uuid())
+
+  routeId String
+  route   GtfsRoute @relation(fields: [routeId], references: [id])
+
+  directionId  String
+  stopId       String
+  stopSequence Int
+
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  @@unique([routeId, directionId, stopId, stopSequence])
+}
+
 model Log {
   id        BigInt   @id @default(autoincrement())
   level     LogLevel

apps/backend/src/app.module.ts

@@ -9,10 +9,12 @@ import { cacheModuleConfig } from "src/config/cache-module.config";
 import { configModuleConfig } from "src/config/config-module.config";
 import { GRAPHQL_PATH } from "src/constants/api";
 import { DepartureModule } from "src/modules/departure/departure.module";
+import { GtfsModule } from "src/modules/gtfs/gtfs.module";
 import { ImportModule } from "src/modules/import/import.module";
 import { LoggerModule } from "src/modules/logger/logger.module";
 import { PlatformModule } from "src/modules/platform/platform.module";
 import { PrismaModule } from "src/modules/prisma/prisma.module";
+import { RouteModule } from "src/modules/route/route.module";
 import { StatusModule } from "src/modules/status/status.module";
 import { StopModule } from "src/modules/stop/stop.module";
 
@@ -25,6 +27,8 @@ import { StopModule } from "src/modules/stop/stop.module";
         PrismaModule,
         LoggerModule,
         StatusModule,
+        GtfsModule,
+        RouteModule,
         ConfigModule.forRoot(configModuleConfig),
         ScheduleModule.forRoot(),
         CacheModule.registerAsync(cacheModuleConfig),

apps/backend/src/modules/gtfs/gtfs.controller.ts

@@ -0,0 +1,26 @@
+import { Controller, OnModuleInit } from "@nestjs/common";
+import { Cron, CronExpression } from "@nestjs/schedule";
+
+import { GtfsService } from "src/modules/gtfs/gtfs.service";
+
+@Controller("gtfs")
+export class GtfsController implements OnModuleInit {
+    constructor(private readonly gtfsService: GtfsService) {}
+
+    async onModuleInit(): Promise<void> {
+        try {
+            this.gtfsService.syncGtfsData();
+        } catch (error) {
+            console.error(error);
+        }
+    }
+
+    @Cron(CronExpression.EVERY_7_HOURS)
+    async cronSyncStops(): Promise<void> {
+        try {
+            await this.gtfsService.syncGtfsData();
+        } catch (error) {
+            console.error(error);
+        }
+    }
+}

apps/backend/src/modules/gtfs/gtfs.module.ts

@@ -0,0 +1,11 @@
+import { Module } from "@nestjs/common";
+
+import { GtfsController } from "src/modules/gtfs/gtfs.controller";
+import { GtfsService } from "src/modules/gtfs/gtfs.service";
+
+@Module({
+    controllers: [GtfsController],
+    providers: [GtfsService],
+    imports: [],
+})
+export class GtfsModule {}

apps/backend/src/modules/gtfs/gtfs.service.ts

@@ -0,0 +1,86 @@
+import { Injectable } from "@nestjs/common";
+import { Open as unzipperOpen } from "unzipper";
+
+import { PrismaService } from "src/modules/prisma/prisma.service";
+import { parseCsvString } from "src/utils/csv.utils";
+
+@Injectable()
+export class GtfsService {
+    constructor(private readonly prisma: PrismaService) {}
+
+    async syncGtfsData() {
+        const response = await fetch("https://data.pid.cz/PID_GTFS.zip");
+        const arrayBuffer = await response.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+        const directory = await unzipperOpen.buffer(buffer);
+
+        const routes = directory.files.find(
+            (file) => file.path === "routes.txt",
+        );
+        if (!routes) {
+            console.log("routes.txt not found");
+            return;
+        }
+        const routeStops = directory.files.find(
+            (file) => file.path === "route_stops.txt",
+        );
+        if (!routeStops) {
+            console.log("route_stops.txt not found");
+            return;
+        }
+
+        const routesBuffer = await routes.buffer();
+        const routeStopsBuffer = await routeStops.buffer();
+
+        type RouteRecord = {
+            route_id: string;
+            route_short_name: string;
+            route_long_name: string;
+            route_type: string;
+            route_color?: string | undefined;
+            is_night: string;
+            route_url?: string | undefined;
+        };
+        // FIXME: validate with zod
+        const routesData = await parseCsvString<RouteRecord>(
+            routesBuffer.toString(),
+        );
+
+        type RouteStopRecord = {
+            route_id: string;
+            direction_id: string;
+            stop_id: string;
+            stop_sequence: string;
+        };
+        // FIXME: validate with zod
+        const routeStopsData = await parseCsvString<RouteStopRecord>(
+            routeStopsBuffer.toString(),
+        );
+
+        await this.prisma.$transaction(async (transaction) => {
+            await transaction.gtfsRouteStop.deleteMany();
+            await transaction.gtfsRoute.deleteMany();
+
+            await transaction.gtfsRoute.createMany({
+                data: routesData.map((route) => ({
+                    id: route.route_id,
+                    shortName: route.route_short_name,
+                    longName: route.route_long_name ?? null,
+                    type: route.route_type,
+                    isNight: Boolean(route.is_night),
+                    color: route.route_color ?? null,
+                    url: route.route_url ?? null,
+                })),
+            });
+
+            await transaction.gtfsRouteStop.createMany({
+                data: routeStopsData.map((routeStop) => ({
+                    routeId: routeStop.route_id,
+                    directionId: routeStop.direction_id,
+                    stopId: routeStop.stop_id,
+                    stopSequence: Number(routeStop.stop_sequence),
+                })),
+            });
+        });
+    }
+}

apps/backend/src/modules/route/route.controller.ts

@@ -0,0 +1,38 @@
+import { CacheInterceptor } from "@nestjs/cache-manager";
+import {
+    Controller,
+    Get,
+    HttpException,
+    HttpStatus,
+    Param,
+    UseInterceptors,
+    Version,
+} from "@nestjs/common";
+import { ApiParam, ApiTags } from "@nestjs/swagger";
+
+import { EndpointVersion } from "src/enums/endpoint-version";
+import { RouteService } from "src/modules/route/route.service";
+
+@ApiTags("route")
+@Controller("route")
+@UseInterceptors(CacheInterceptor)
+export class RouteController {
+    constructor(private readonly routeService: RouteService) {}
+
+    @Get(":id")
+    @Version([EndpointVersion.v1])
+    @ApiParam({
+        name: "id",
+        description: "Route ID",
+        required: true,
+        example: "L991",
+        type: "string",
+    })
+    async getRoute(@Param("id") id: unknown) {
+        if (typeof id !== "string") {
+            throw new HttpException("Missing route ID", HttpStatus.BAD_REQUEST);
+        }
+
+        return this.routeService.getRoute(id);
+    }
+}

apps/backend/src/modules/route/route.module.ts

@@ -0,0 +1,11 @@
+import { Module } from "@nestjs/common";
+
+import { RouteController } from "src/modules/route/route.controller";
+import { RouteService } from "src/modules/route/route.service";
+
+@Module({
+    controllers: [RouteController],
+    providers: [RouteService],
+    imports: [],
+})
+export class RouteModule {}

apps/backend/src/modules/route/route.service.ts

@@ -0,0 +1,65 @@
+import { Injectable } from "@nestjs/common";
+import { group, unique } from "radash";
+
+import { platformSelect } from "src/modules/platform/platform.service";
+import { PrismaService } from "src/modules/prisma/prisma.service";
+
+const gtfsRouteSelect = {
+    id: true,
+    shortName: true,
+    longName: true,
+    isNight: true,
+    color: true,
+    url: true,
+    type: true,
+};
+
+const gtfsRouteStopSelect = {
+    directionId: true,
+    stopId: true,
+    stopSequence: true,
+};
+
+@Injectable()
+export class RouteService {
+    constructor(private prisma: PrismaService) {}
+
+    async getRoute(id: string) {
+        const route = await this.prisma.gtfsRoute.findFirst({
+            select: gtfsRouteSelect,
+            where: {
+                id,
+            },
+        });
+
+        const routeStops = await this.prisma.gtfsRouteStop.findMany({
+            select: gtfsRouteStopSelect,
+            where: {
+                routeId: id,
+            },
+            orderBy: {
+                stopSequence: "asc",
+            },
+        });
+
+        const stops = await this.prisma.platform.findMany({
+            select: platformSelect,
+            where: {
+                id: {
+                    in: unique(routeStops.map((item) => item.stopId)),
+                },
+            },
+        });
+
+        return {
+            ...route,
+            directions: group(
+                routeStops.map((routeStop) => ({
+                    ...routeStop,
+                    stop: stops.find((stop) => stop.id === routeStop.stopId),
+                })),
+                (item) => item.directionId,
+            ),
+        };
+    }
+}

apps/backend/src/utils/csv.utils.ts

@@ -0,0 +1,14 @@
+import { parseString } from "@fast-csv/parse";
+
+export async function parseCsvString<T>(csvString: string): Promise<T[]> {
+    return new Promise((resolve) => {
+        const rows: T[] = [];
+
+        parseString(csvString, { headers: true })
+            .on("error", (error) => console.error(error))
+            .on("data", (row) => rows.push(row))
+            .on("end", () => {
+                resolve(rows);
+            });
+    });
+}