feature: OCR capabilities (#210)

* chore: changed text extraction into API

* feature: working OCR capabilities

apps/expo/app.config.ts

@@ -33,6 +33,7 @@ const defineConfig = (_ctx: ConfigContext): ExpoConfig => ({
       foregroundImage: "./assets/icon.png",
       backgroundColor: "#fff",
     },
+    permissions: ["READ_EXTERNAL_STORAGE", "WRITE_EXTERNAL_STORAGE"],
   },
   extra: {
     eas: {

apps/expo/src/screens/create-reviewer/index.tsx

@@ -43,8 +43,19 @@ import { NUMBER_OF_QUESTIONS_OPTIONS } from "./constants";
 import useQuestionStore, { QuestionType } from "../../stores/useQuestionStore";
 import { AppButton } from "../../components/buttons/AppButton";
 import * as DocumentPicker from "expo-document-picker";
+import * as FileSystem from "expo-file-system";
 import { mapQuestionType } from "../../utils/helpers/strings";
 
+interface CacheOptions {
+  name: string;
+  uri: string;
+}
+
+interface FilePickerType {
+  fileType: string;
+  base64ContentType: string;
+}
+
 export const CreateReviewerScreen = ({
   navigation,
   route,
@@ -73,6 +84,8 @@ export const CreateReviewerScreen = ({
   );
   const [isHighlighterToggled, setIsHighlighterToggled] = useState(false);
   const [isChoiceModalToggled, setIsChoiceModalToggled] = useState(false);
+  const [isPDFUploading, setIsPDFUploading] = useState(false);
+  const [isImageUploading, setIsImageUploading] = useState(false);
   const [openAlert, setOpenAlert] = useState(false);
   const [numberOfQuestionOptions, setNumberOfQuestionOptions] = useState<
     Option[]
@@ -101,6 +114,8 @@ export const CreateReviewerScreen = ({
     },
   );
 
+  const { mutate: readFile } = trpc.pdfTextExtraction.extractText.useMutation();
+
   const { mutate: generateMultipleQuestions, isLoading: isGenerating } =
     trpc.gptApi.generateMultipleQuestions.useMutation();
 
@@ -201,14 +216,79 @@ export const CreateReviewerScreen = ({
     });
   }, [reviewerImage]);
 
-  const filePicker = async () => {
+  const createCacheFile = async ({ name, uri }: CacheOptions) => {
+    if (
+      !(await FileSystem.getInfoAsync(FileSystem.cacheDirectory + "uploads/"))
+        .exists
+    ) {
+      await FileSystem.makeDirectoryAsync(
+        FileSystem.cacheDirectory + "uploads/",
+      );
+    }
+    const cacheFilePath = FileSystem.cacheDirectory + "uploads/" + name;
+    await FileSystem.copyAsync({ from: uri, to: cacheFilePath });
+    return cacheFilePath;
+  };
+
+  const filePicker = async ({
+    fileType,
+    base64ContentType,
+  }: FilePickerType) => {
     const result = await DocumentPicker.getDocumentAsync({
-      type: "application/pdf",
-      copyToCacheDirectory: true,
+      type: fileType,
+      copyToCacheDirectory: false,
+      multiple: false,
     });
 
-    if (result.type === "success") {
-      console.log(result.uri);
+    if (result.type !== "cancel") {
+      if (fileType === "image/*") {
+        setIsImageUploading(true);
+      } else {
+        setIsPDFUploading(true);
+      }
+
+      const convertedFile = await createCacheFile({
+        name: result.name,
+        uri: result.uri,
+      });
+
+      const base64 = await FileSystem.readAsStringAsync(convertedFile, {
+        encoding: "base64",
+      });
+
+      if (base64) {
+        const formatBase64 = `data:${base64ContentType};base64,${base64}`;
+        readFile(
+          {
+            file: formatBase64,
+            fileType: `${fileType === "image/*" ? "JPG" : "PDF"}`,
+          },
+          {
+            onSuccess: (data) => {
+              if (richText.current) {
+                richText.current.insertHTML(data.text);
+              }
+              successToast({
+                title: "Success",
+                message: `${
+                  fileType === "image/*" ? "Image" : "File"
+                } transcribed successfully`,
+              });
+
+              setIsImageUploading(false);
+              setIsPDFUploading(false);
+            },
+            onError: (error) => {
+              errorToast({
+                title: "Error",
+                message: error.message,
+              });
+              setIsImageUploading(false);
+              setIsPDFUploading(false);
+            },
+          },
+        );
+      }
     }
   };
 
@@ -569,17 +649,40 @@ export const CreateReviewerScreen = ({
             name="content"
           />
         </View>
-        <View>
+        <View className="w-[90%] flex-row justify-around self-center">
+          <AppButton
+            text="Upload PDF"
+            buttonColor="violet-600"
+            borderShadowColor="indigo-800"
+            borderRadius="full"
+            fontStyle="bold"
+            textColor="white"
+            TOwidth="36"
+            Vwidth="36"
+            isLoading={isPDFUploading}
+            onPress={() =>
+              filePicker({
+                fileType: "application/pdf",
+                base64ContentType: "application/pdf",
+              })
+            }
+          />
           <AppButton
-            text="Upload PDF file"
+            text="Upload Image"
             buttonColor="violet-600"
             borderShadowColor="indigo-800"
             borderRadius="full"
             fontStyle="bold"
             textColor="white"
             TOwidth="36"
             Vwidth="36"
-            onPress={filePicker}
+            isLoading={isImageUploading}
+            onPress={() =>
+              filePicker({
+                fileType: "image/*",
+                base64ContentType: "image/jpg",
+              })
+            }
           />
         </View>
       </ScrollView>

packages/api/src/router/pdfTextExtraction.ts

@@ -1,19 +1,45 @@
 import { router, protectedProcedure } from "../trpc";
 import { z } from "zod";
-import { readPDFFile } from "../services/pdfTextExtractor";
+
+type OCRResult = {
+  text: string;
+};
 
 export const textExtractionRouter = router({
   extractText: protectedProcedure
     .input(
       z.object({
         file: z.string(),
+        fileType: z.string(),
       }),
     )
     .mutation(async ({ input }) => {
-      const { file } = input;
+      const { file, fileType } = input;
+
+      const apiKey = process.env.OCR_API;
+      const apiEndpoint = "https://api.ocr.space/parse/image";
+
+      const formData = new FormData();
+      formData.append("base64image", file);
+      formData.append("filetype", fileType);
+      if (apiKey) {
+        formData.append("apikey", apiKey);
+      }
+
+      const response = await fetch(apiEndpoint, {
+        method: "POST",
+        body: formData,
+      });
+
+      if (!response.ok) {
+        throw new Error("OCR request failed");
+      }
 
-      const content = await readPDFFile(file);
+      const data = await response.json();
+      const ocrResult: OCRResult = {
+        text: data.ParsedResults?.[0]?.ParsedText || "",
+      };
 
-      return content;
+      return ocrResult;
     }),
 });