Merge pull request #5 from RyoJerryYu/feat-use-proto-memo-for-plugin

feat: use proto memo for plugin

README.md

@@ -38,7 +38,7 @@ It implements a simple URL resource download feature. All url [you-get](https://
 
 2. Run `docker-compose up -d` to start the services.
 
-3. Access `localhost:5230` , login and make sure the Memos server work. Create a webhook to `http://webhook:8000/webhook` .
+3. Access `localhost:5230` , login and make sure the Memos server work. Create a webhook to `http://webhook:8000/webhook` . (It's for Memos server after 0.22.2. For before 0.22.1, use `http://webhook:8000/webhook_old`)
 
 4. Post a memo with contents containing a twitter url. If that tweet was attached with some image, the webhook will download them and upload to the Memo server automatically.
 

memos_webhook/app.py

@@ -2,9 +2,9 @@
 
 import asyncio
 import contextlib
-from typing import Annotated
+from typing import Annotated, Any
 
-from fastapi import BackgroundTasks, Depends, FastAPI
+from fastapi import BackgroundTasks, Depends, FastAPI, Request
 
 import memos_webhook.proto_gen.memos.api.v1 as v1
 from memos_webhook.dependencies.config import get_config, new_config
@@ -16,6 +16,7 @@
 from memos_webhook.plugins.you_get_plugin import YouGetPlugin
 from memos_webhook.utils.logger import logger as util_logger
 from memos_webhook.utils.logger import logging_config
+from memos_webhook.webhook.type_transform import old_payload_to_proto
 from memos_webhook.webhook.types.webhook_payload import WebhookPayload
 
 logger = util_logger.getChild("app")
@@ -37,26 +38,47 @@ async def lifespan(app: FastAPI):
 
 
 async def webhook_task(
-    payload: WebhookPayload,
+    payload: v1.WebhookRequestPayload,
     executor: PluginExecutor,
 ):
     await executor.execute(payload)
 
 
-@app.post("/webhook")
-async def webhook_hanlder(
+@app.post("/webhook_old")
+async def webhook_old_hanlder(
     payload: WebhookPayload,
     background_tasks: BackgroundTasks,
     executor: Annotated[PluginExecutor, Depends(get_plugin_executor)],
 ):
+    """The old webhook handler, use specific json schema."""
     # 添加后台任务
-    background_tasks.add_task(webhook_task, payload, executor)
+    proto_payload = old_payload_to_proto(payload)
+    background_tasks.add_task(webhook_task, proto_payload, executor)
     return {
         "code": 0,
         "message": f"Task started in background with param: {payload.model_dump_json()}",
     }
 
 
+@app.post("/webhook")
+async def webhook_handler(
+    req: Request,
+    background_tasks: BackgroundTasks,
+    executor: Annotated[PluginExecutor, Depends(get_plugin_executor)],
+):
+    """The new webhook handler, use protojson."""
+    dict_json = await req.json()
+    logger.debug(f"webhook handler received request: {dict_json}")
+    logger.debug(f"type: {type(dict_json)}")
+
+    proto_payload = v1.WebhookRequestPayload().from_dict(dict_json)
+    background_tasks.add_task(webhook_task, proto_payload, executor)
+    return {
+        "code": 0,
+        "message": f"Task started with param: {proto_payload.to_json()}",
+    }
+
+
 async def root_task(memos_cli: MemosCli):
     logger.info("root task started")
     await asyncio.sleep(3)

memos_webhook/plugins/base_plugin.py

@@ -19,8 +19,8 @@ class PluginProtocol(Protocol):
     Unless you know what you are doing."""
     def positive_tag(self) -> str: ...
     def negative_tag(self) -> str: ...
-    async def task(self, payload: WebhookPayload, memos_cli: MemosCli) -> v1.Memo: ...
-    def should_trigger(self, payload: WebhookPayload) -> bool: ...
+    async def task(self, payload: v1.WebhookRequestPayload, memos_cli: MemosCli) -> v1.Memo: ...
+    def should_trigger(self, payload: v1.WebhookRequestPayload) -> bool: ...
 
 
 class BasePlugin(PluginProtocol, ABC):
@@ -60,7 +60,7 @@ def tag(self) -> str:
         ...
 
     @abstractmethod
-    async def task(self, payload: WebhookPayload, memos_cli: MemosCli) -> v1.Memo:
+    async def task(self, payload: v1.WebhookRequestPayload, memos_cli: MemosCli) -> v1.Memo:
         """The webhook task function.
 
         Return the modified memo, and the plugin will auto update the memo with modified content and negative tag.
@@ -79,14 +79,14 @@ def negative_tag(self) -> str:
         """The negative tag for the webhook plugin."""
         return f"#{self.tag()}/done"
 
-    def additional_trigger(self, payload: WebhookPayload) -> bool:
+    def additional_trigger(self, payload: v1.WebhookRequestPayload) -> bool:
         """The additional trigger besides the tag.
         If return True and negative tag not exists,
         the webhook will be triggered even if the tag not exists.
         """
         return False
 
-    def should_trigger(self, payload: WebhookPayload) -> bool:
+    def should_trigger(self, payload: v1.WebhookRequestPayload) -> bool:
         """Check if the rule should trigger by the payload.
 
         First check if the payload activity type is in the trigger activity types.
@@ -96,8 +96,8 @@ def should_trigger(self, payload: WebhookPayload) -> bool:
         """
         assert payload.memo is not None, "payload memo is None"
 
-        if payload.activityType not in self.activity_types():
-            self.logger.info(f"activityType not match: {payload.activityType}")
+        if payload.activity_type not in self.activity_types():
+            self.logger.info(f"activityType not match: {payload.activity_type}")
             return False
 
         negative_tag, positive_tag = self.negative_tag(), self.positive_tag()
@@ -132,14 +132,14 @@ def __init__(self, memos_cli: MemosCli, plugins: list[PluginProtocol]) -> None:
         self.logger = logger.getChild("PluginExecutor")
 
     async def update_memo_content(
-        self, plugin: PluginProtocol, payload: WebhookPayload
+        self, plugin: PluginProtocol, payload: v1.WebhookRequestPayload
     ) -> v1.Memo:
         """update memo content
         Once the task triggered, will replace the `#tag` with `#tag/done`.
         If the `#tag` not exists, will add the `#tag/done` to first line.
         """
         self.logger.debug(
-            f"Background task started with param: {payload.model_dump_json()}"
+            f"Background task started with param: {payload.to_json()}"
         )
 
         res_memo = await plugin.task(payload, self.memos_cli)
@@ -161,11 +161,11 @@ async def update_memo_content(
         )
         self.logger.debug(f"Updated memo content {updated_memo.content}")
 
-    async def execute(self, payload: WebhookPayload) -> None:
+    async def execute(self, payload: v1.WebhookRequestPayload) -> None:
         """Execute the webhook task by the rule."""
         for plugin in self.plugins:
             self.logger.info(f"Execute plugin: {plugin}")
-            if not plugin.should_trigger(payload):
+            if not plugin.should_trigger(payload=payload):
                 continue
 
             await self.update_memo_content(plugin, payload)

memos_webhook/plugins/base_plugin_test.py

@@ -4,7 +4,6 @@
 import memos_webhook.webhook.types.memo_service as webhook_types
 from memos_webhook.dependencies.memos_cli import MemosCli
 from memos_webhook.proto_gen.memos.api import v1
-from memos_webhook.webhook.types.webhook_payload import WebhookPayload
 
 from .base_plugin import BasePlugin
 
@@ -19,7 +18,7 @@ def tag(self) -> str:
         return "hook/download"
 
     @override
-    async def task(self, payload: WebhookPayload, memos_cli: MemosCli) -> v1.Memo:
+    async def task(self, payload: v1.WebhookRequestPayload, memos_cli: MemosCli) -> v1.Memo:
         return v1.Memo()
 
 
@@ -33,14 +32,14 @@ def tag(self) -> str:
         return "hook/download"
 
     @override
-    def additional_trigger(self, payload: WebhookPayload) -> bool:
+    def additional_trigger(self, payload: v1.WebhookRequestPayload) -> bool:
         if payload.memo:
             return "overwrite" in payload.memo.content
 
         return False
 
     @override
-    async def task(self, payload: WebhookPayload, memos_cli: MemosCli) -> v1.Memo:
+    async def task(self, payload: v1.WebhookRequestPayload, memos_cli: MemosCli) -> v1.Memo:
         return v1.Memo()
 
 
@@ -85,8 +84,8 @@ async def test_should_trigger(self):
             plugin = MockPlugin()
             self.assertEqual(
                 plugin.should_trigger(
-                    WebhookPayload(
-                        activityType=case["activityType"],
+                    v1.WebhookRequestPayload(
+                        activity_type=case["activityType"],
                         memo=webhook_types.Memo(content=case["content"]),
                     )
                 ),
@@ -132,8 +131,8 @@ async def test_should_trigger_overwrite(self):
             plugin = MockOverwritePlugin()
             self.assertEqual(
                 plugin.should_trigger(
-                    WebhookPayload(
-                        activityType=case["activityType"],
+                    v1.WebhookRequestPayload(
+                        activity_type=case["activityType"],
                         memo=webhook_types.Memo(content=case["content"]),
                     )
                 ),

memos_webhook/plugins/you_get_plugin.py

@@ -42,14 +42,14 @@ def tag(self) -> str:
         return self.cfg.tag
 
     @override
-    def additional_trigger(self, payload: WebhookPayload) -> bool:
+    def additional_trigger(self, payload: v1.WebhookRequestPayload) -> bool:
         urls = extract_urls(payload.memo.content, self.patterns)
         if urls:
             return True
         return False
 
     @override
-    async def task(self, payload: WebhookPayload, memos_cli: MemosCli) -> v1.Memo:
+    async def task(self, payload: v1.WebhookRequestPayload, memos_cli: MemosCli) -> v1.Memo:
         memo_name = payload.memo.name
         self.logger.info(f"Start {self.cfg.name} webhook task for memo: {memo_name}")
 

memos_webhook/webhook/type_transform.py

@@ -0,0 +1,78 @@
+from datetime import datetime
+
+import memos_webhook.proto_gen.memos.api.v1 as v1
+
+from .types.common import RowStatus as OldRowStatus
+from .types.google_protobuf import PbTimestamp as OldPbTimestamp
+from .types.memo_relation_service import MemoRelation as OldRelation
+from .types.memo_service import Memo as OldMemo
+from .types.memo_service import Visibility as OldVisibility
+from .types.resource_service import Resource as OldResource
+from .types.webhook_payload import WebhookPayload as OldPayload
+
+
+def old_row_status_to_proto(row_status: OldRowStatus) -> v1.RowStatus:
+    return v1.RowStatus(int(row_status))
+
+
+def old_timestamp_to_proto(timestamp: OldPbTimestamp | None) -> datetime:
+    if timestamp is None:
+        return None
+    return datetime.fromtimestamp(timestamp.seconds + (timestamp.nanos) * 0.001)
+
+
+def old_visibility_to_proto(visibility: OldVisibility) -> v1.Visibility:
+    return v1.Visibility(int(visibility))
+
+
+def old_resource_to_proto(input: OldResource) -> v1.Resource:
+    return v1.Resource(
+        name=input.name,
+        uid=input.uid,
+        create_time=input.create_time,
+        filename=input.filename,
+        content=input.content,
+        external_link=input.external_link,
+        type=input.type,
+        size=input.size,
+        memo=input.memo,
+    )
+
+
+def old_relation_to_proto(input: OldRelation) -> v1.MemoRelation:
+    return v1.MemoRelation(
+        memo=input.memo,
+        related_memo=input.related_memo,
+        type=v1.MemoRelationType(int(input.type)),
+    )
+
+
+def old_memo_to_proto(input: OldMemo) -> v1.Memo:
+    return v1.Memo(
+        name=input.name,
+        uid=input.uid,
+        row_status=old_row_status_to_proto(input.row_status),
+        creator=input.creator,
+        create_time=old_timestamp_to_proto(input.create_time),
+        update_time=old_timestamp_to_proto(input.update_time),
+        display_time=old_timestamp_to_proto(input.display_time),
+        content=input.content,
+        nodes=[],  # we do not handle nodes transformation for old version.
+        visibility=old_visibility_to_proto(input.visibility),
+        tags=input.tags,
+        pinned=input.pinned,
+        parent_id=input.parent_id,
+        resources=[old_resource_to_proto(resource) for resource in input.resources],
+        relations=[old_relation_to_proto(relation) for relation in input.relations],
+        parent=input.parent,
+    )
+
+
+def old_payload_to_proto(input: OldPayload) -> v1.WebhookRequestPayload:
+    return v1.WebhookRequestPayload(
+        url=input.url,
+        activity_type=input.activityType,
+        creator_id=input.creatorId,
+        create_time=datetime.fromtimestamp(input.createdTs),
+        memo=old_memo_to_proto(input.memo),
+    )

proto/api/v1/memo_service.proto

@@ -207,6 +207,8 @@ message MemoProperty {
   repeated string tags = 1;
   bool has_link = 2;
   bool has_task_list = 3;
+  bool has_code = 4;
+  bool has_incomplete_tasks = 5;
 }
 
 message CreateMemoRequest {