diff --git a/README.md b/README.md index d38f3e6..0fbf02e 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ It implements a simple URL resource download feature. All url [you-get](https:// 2. Run `docker-compose up -d` to start the services. -3. Access `localhost:5230` , login and make sure the Memos server work. Create a webhook to `http://webhook:8000/webhook` . +3. Access `localhost:5230` , login and make sure the Memos server work. Create a webhook to `http://webhook:8000/webhook` . (It's for Memos server after 0.22.2. For before 0.22.1, use `http://webhook:8000/webhook_old`) 4. Post a memo with contents containing a twitter url. If that tweet was attached with some image, the webhook will download them and upload to the Memo server automatically. diff --git a/memos_webhook/app.py b/memos_webhook/app.py index d519e81..47a48b0 100644 --- a/memos_webhook/app.py +++ b/memos_webhook/app.py @@ -2,9 +2,9 @@ import asyncio import contextlib -from typing import Annotated +from typing import Annotated, Any -from fastapi import BackgroundTasks, Depends, FastAPI +from fastapi import BackgroundTasks, Depends, FastAPI, Request import memos_webhook.proto_gen.memos.api.v1 as v1 from memos_webhook.dependencies.config import get_config, new_config @@ -16,6 +16,7 @@ from memos_webhook.plugins.you_get_plugin import YouGetPlugin from memos_webhook.utils.logger import logger as util_logger from memos_webhook.utils.logger import logging_config +from memos_webhook.webhook.type_transform import old_payload_to_proto from memos_webhook.webhook.types.webhook_payload import WebhookPayload logger = util_logger.getChild("app") @@ -37,26 +38,47 @@ async def lifespan(app: FastAPI): async def webhook_task( - payload: WebhookPayload, + payload: v1.WebhookRequestPayload, executor: PluginExecutor, ): await executor.execute(payload) -@app.post("/webhook") -async def webhook_hanlder( +@app.post("/webhook_old") +async def webhook_old_hanlder( payload: WebhookPayload, background_tasks: BackgroundTasks, executor: Annotated[PluginExecutor, Depends(get_plugin_executor)], ): + """The old webhook handler, use specific json schema.""" # 添加后台任务 - background_tasks.add_task(webhook_task, payload, executor) + proto_payload = old_payload_to_proto(payload) + background_tasks.add_task(webhook_task, proto_payload, executor) return { "code": 0, "message": f"Task started in background with param: {payload.model_dump_json()}", } +@app.post("/webhook") +async def webhook_handler( + req: Request, + background_tasks: BackgroundTasks, + executor: Annotated[PluginExecutor, Depends(get_plugin_executor)], +): + """The new webhook handler, use protojson.""" + dict_json = await req.json() + logger.debug(f"webhook handler received request: {dict_json}") + logger.debug(f"type: {type(dict_json)}") + + proto_payload = v1.WebhookRequestPayload().from_dict(dict_json) + background_tasks.add_task(webhook_task, proto_payload, executor) + return { + "code": 0, + "message": f"Task started with param: {proto_payload.to_json()}", + } + + async def root_task(memos_cli: MemosCli): logger.info("root task started") await asyncio.sleep(3) diff --git a/memos_webhook/plugins/base_plugin.py b/memos_webhook/plugins/base_plugin.py index fc6984e..b82bedb 100644 --- a/memos_webhook/plugins/base_plugin.py +++ b/memos_webhook/plugins/base_plugin.py @@ -19,8 +19,8 @@ class PluginProtocol(Protocol): Unless you know what you are doing.""" def positive_tag(self) -> str: ... def negative_tag(self) -> str: ... - async def task(self, payload: WebhookPayload, memos_cli: MemosCli) -> v1.Memo: ... - def should_trigger(self, payload: WebhookPayload) -> bool: ... + async def task(self, payload: v1.WebhookRequestPayload, memos_cli: MemosCli) -> v1.Memo: ... + def should_trigger(self, payload: v1.WebhookRequestPayload) -> bool: ... class BasePlugin(PluginProtocol, ABC): @@ -60,7 +60,7 @@ def tag(self) -> str: ... @abstractmethod - async def task(self, payload: WebhookPayload, memos_cli: MemosCli) -> v1.Memo: + async def task(self, payload: v1.WebhookRequestPayload, memos_cli: MemosCli) -> v1.Memo: """The webhook task function. Return the modified memo, and the plugin will auto update the memo with modified content and negative tag. @@ -79,14 +79,14 @@ def negative_tag(self) -> str: """The negative tag for the webhook plugin.""" return f"#{self.tag()}/done" - def additional_trigger(self, payload: WebhookPayload) -> bool: + def additional_trigger(self, payload: v1.WebhookRequestPayload) -> bool: """The additional trigger besides the tag. If return True and negative tag not exists, the webhook will be triggered even if the tag not exists. """ return False - def should_trigger(self, payload: WebhookPayload) -> bool: + def should_trigger(self, payload: v1.WebhookRequestPayload) -> bool: """Check if the rule should trigger by the payload. First check if the payload activity type is in the trigger activity types. @@ -96,8 +96,8 @@ def should_trigger(self, payload: WebhookPayload) -> bool: """ assert payload.memo is not None, "payload memo is None" - if payload.activityType not in self.activity_types(): - self.logger.info(f"activityType not match: {payload.activityType}") + if payload.activity_type not in self.activity_types(): + self.logger.info(f"activityType not match: {payload.activity_type}") return False negative_tag, positive_tag = self.negative_tag(), self.positive_tag() @@ -132,14 +132,14 @@ def __init__(self, memos_cli: MemosCli, plugins: list[PluginProtocol]) -> None: self.logger = logger.getChild("PluginExecutor") async def update_memo_content( - self, plugin: PluginProtocol, payload: WebhookPayload + self, plugin: PluginProtocol, payload: v1.WebhookRequestPayload ) -> v1.Memo: """update memo content Once the task triggered, will replace the `#tag` with `#tag/done`. If the `#tag` not exists, will add the `#tag/done` to first line. """ self.logger.debug( - f"Background task started with param: {payload.model_dump_json()}" + f"Background task started with param: {payload.to_json()}" ) res_memo = await plugin.task(payload, self.memos_cli) @@ -161,11 +161,11 @@ async def update_memo_content( ) self.logger.debug(f"Updated memo content {updated_memo.content}") - async def execute(self, payload: WebhookPayload) -> None: + async def execute(self, payload: v1.WebhookRequestPayload) -> None: """Execute the webhook task by the rule.""" for plugin in self.plugins: self.logger.info(f"Execute plugin: {plugin}") - if not plugin.should_trigger(payload): + if not plugin.should_trigger(payload=payload): continue await self.update_memo_content(plugin, payload) diff --git a/memos_webhook/plugins/base_plugin_test.py b/memos_webhook/plugins/base_plugin_test.py index b4e08ac..f6d8e65 100644 --- a/memos_webhook/plugins/base_plugin_test.py +++ b/memos_webhook/plugins/base_plugin_test.py @@ -4,7 +4,6 @@ import memos_webhook.webhook.types.memo_service as webhook_types from memos_webhook.dependencies.memos_cli import MemosCli from memos_webhook.proto_gen.memos.api import v1 -from memos_webhook.webhook.types.webhook_payload import WebhookPayload from .base_plugin import BasePlugin @@ -19,7 +18,7 @@ def tag(self) -> str: return "hook/download" @override - async def task(self, payload: WebhookPayload, memos_cli: MemosCli) -> v1.Memo: + async def task(self, payload: v1.WebhookRequestPayload, memos_cli: MemosCli) -> v1.Memo: return v1.Memo() @@ -33,14 +32,14 @@ def tag(self) -> str: return "hook/download" @override - def additional_trigger(self, payload: WebhookPayload) -> bool: + def additional_trigger(self, payload: v1.WebhookRequestPayload) -> bool: if payload.memo: return "overwrite" in payload.memo.content return False @override - async def task(self, payload: WebhookPayload, memos_cli: MemosCli) -> v1.Memo: + async def task(self, payload: v1.WebhookRequestPayload, memos_cli: MemosCli) -> v1.Memo: return v1.Memo() @@ -85,8 +84,8 @@ async def test_should_trigger(self): plugin = MockPlugin() self.assertEqual( plugin.should_trigger( - WebhookPayload( - activityType=case["activityType"], + v1.WebhookRequestPayload( + activity_type=case["activityType"], memo=webhook_types.Memo(content=case["content"]), ) ), @@ -132,8 +131,8 @@ async def test_should_trigger_overwrite(self): plugin = MockOverwritePlugin() self.assertEqual( plugin.should_trigger( - WebhookPayload( - activityType=case["activityType"], + v1.WebhookRequestPayload( + activity_type=case["activityType"], memo=webhook_types.Memo(content=case["content"]), ) ), diff --git a/memos_webhook/plugins/you_get_plugin.py b/memos_webhook/plugins/you_get_plugin.py index 2838027..5bb8ff0 100644 --- a/memos_webhook/plugins/you_get_plugin.py +++ b/memos_webhook/plugins/you_get_plugin.py @@ -42,14 +42,14 @@ def tag(self) -> str: return self.cfg.tag @override - def additional_trigger(self, payload: WebhookPayload) -> bool: + def additional_trigger(self, payload: v1.WebhookRequestPayload) -> bool: urls = extract_urls(payload.memo.content, self.patterns) if urls: return True return False @override - async def task(self, payload: WebhookPayload, memos_cli: MemosCli) -> v1.Memo: + async def task(self, payload: v1.WebhookRequestPayload, memos_cli: MemosCli) -> v1.Memo: memo_name = payload.memo.name self.logger.info(f"Start {self.cfg.name} webhook task for memo: {memo_name}") diff --git a/memos_webhook/proto_gen/__init__.py b/memos_webhook/proto_gen/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/memos_webhook/proto_gen/memos/api/v1/__init__.py b/memos_webhook/proto_gen/memos/api/v1/__init__.py index f3216db..9583d2b 100644 --- a/memos_webhook/proto_gen/memos/api/v1/__init__.py +++ b/memos_webhook/proto_gen/memos/api/v1/__init__.py @@ -907,6 +907,8 @@ class MemoProperty(betterproto.Message): tags: List[str] = betterproto.string_field(1) has_link: bool = betterproto.bool_field(2) has_task_list: bool = betterproto.bool_field(3) + has_code: bool = betterproto.bool_field(4) + has_incomplete_tasks: bool = betterproto.bool_field(5) @dataclass(eq=False, repr=False) @@ -1172,8 +1174,8 @@ class DeleteMemoReactionRequest(betterproto.Message): class Webhook(betterproto.Message): id: int = betterproto.int32_field(1) creator_id: int = betterproto.int32_field(2) - created_time: datetime = betterproto.message_field(3) - updated_time: datetime = betterproto.message_field(4) + create_time: datetime = betterproto.message_field(3) + update_time: datetime = betterproto.message_field(4) row_status: "RowStatus" = betterproto.enum_field(5) name: str = betterproto.string_field(6) url: str = betterproto.string_field(7) @@ -1213,6 +1215,15 @@ class DeleteWebhookRequest(betterproto.Message): id: int = betterproto.int32_field(1) +@dataclass(eq=False, repr=False) +class WebhookRequestPayload(betterproto.Message): + url: str = betterproto.string_field(1) + activity_type: str = betterproto.string_field(2) + creator_id: int = betterproto.int32_field(3) + create_time: datetime = betterproto.message_field(4) + memo: "Memo" = betterproto.message_field(5) + + @dataclass(eq=False, repr=False) class WorkspaceProfile(betterproto.Message): owner: str = betterproto.string_field(1) @@ -1248,22 +1259,19 @@ class WorkspaceSetting(betterproto.Message): @dataclass(eq=False, repr=False) class WorkspaceGeneralSetting(betterproto.Message): - instance_url: str = betterproto.string_field(1) - """instance_url is the instance URL.""" - - disallow_signup: bool = betterproto.bool_field(2) + disallow_signup: bool = betterproto.bool_field(1) """disallow_signup is the flag to disallow signup.""" - disallow_password_login: bool = betterproto.bool_field(3) + disallow_password_login: bool = betterproto.bool_field(2) """disallow_password_login is the flag to disallow password login.""" - additional_script: str = betterproto.string_field(4) + additional_script: str = betterproto.string_field(3) """additional_script is the additional script.""" - additional_style: str = betterproto.string_field(5) + additional_style: str = betterproto.string_field(4) """additional_style is the additional style.""" - custom_profile: "WorkspaceCustomProfile" = betterproto.message_field(6) + custom_profile: "WorkspaceCustomProfile" = betterproto.message_field(5) """custom_profile is the custom profile.""" @@ -1315,6 +1323,12 @@ class WorkspaceMemoRelatedSetting(betterproto.Message): content_length_limit: int = betterproto.int32_field(3) """content_length_limit is the limit of content length. Unit is byte.""" + enable_auto_compact: bool = betterproto.bool_field(4) + """enable_auto_compact enables auto compact for large content.""" + + enable_double_click_edit: bool = betterproto.bool_field(5) + """enable_double_click_edit enables editing on double click.""" + @dataclass(eq=False, repr=False) class GetWorkspaceSettingRequest(betterproto.Message): diff --git a/memos_webhook/webhook/type_transform.py b/memos_webhook/webhook/type_transform.py new file mode 100644 index 0000000..6bedb4a --- /dev/null +++ b/memos_webhook/webhook/type_transform.py @@ -0,0 +1,78 @@ +from datetime import datetime + +import memos_webhook.proto_gen.memos.api.v1 as v1 + +from .types.common import RowStatus as OldRowStatus +from .types.google_protobuf import PbTimestamp as OldPbTimestamp +from .types.memo_relation_service import MemoRelation as OldRelation +from .types.memo_service import Memo as OldMemo +from .types.memo_service import Visibility as OldVisibility +from .types.resource_service import Resource as OldResource +from .types.webhook_payload import WebhookPayload as OldPayload + + +def old_row_status_to_proto(row_status: OldRowStatus) -> v1.RowStatus: + return v1.RowStatus(int(row_status)) + + +def old_timestamp_to_proto(timestamp: OldPbTimestamp | None) -> datetime: + if timestamp is None: + return None + return datetime.fromtimestamp(timestamp.seconds + (timestamp.nanos) * 0.001) + + +def old_visibility_to_proto(visibility: OldVisibility) -> v1.Visibility: + return v1.Visibility(int(visibility)) + + +def old_resource_to_proto(input: OldResource) -> v1.Resource: + return v1.Resource( + name=input.name, + uid=input.uid, + create_time=input.create_time, + filename=input.filename, + content=input.content, + external_link=input.external_link, + type=input.type, + size=input.size, + memo=input.memo, + ) + + +def old_relation_to_proto(input: OldRelation) -> v1.MemoRelation: + return v1.MemoRelation( + memo=input.memo, + related_memo=input.related_memo, + type=v1.MemoRelationType(int(input.type)), + ) + + +def old_memo_to_proto(input: OldMemo) -> v1.Memo: + return v1.Memo( + name=input.name, + uid=input.uid, + row_status=old_row_status_to_proto(input.row_status), + creator=input.creator, + create_time=old_timestamp_to_proto(input.create_time), + update_time=old_timestamp_to_proto(input.update_time), + display_time=old_timestamp_to_proto(input.display_time), + content=input.content, + nodes=[], # we do not handle nodes transformation for old version. + visibility=old_visibility_to_proto(input.visibility), + tags=input.tags, + pinned=input.pinned, + parent_id=input.parent_id, + resources=[old_resource_to_proto(resource) for resource in input.resources], + relations=[old_relation_to_proto(relation) for relation in input.relations], + parent=input.parent, + ) + + +def old_payload_to_proto(input: OldPayload) -> v1.WebhookRequestPayload: + return v1.WebhookRequestPayload( + url=input.url, + activity_type=input.activityType, + creator_id=input.creatorId, + create_time=datetime.fromtimestamp(input.createdTs), + memo=old_memo_to_proto(input.memo), + ) diff --git a/proto/api/v1/memo_service.proto b/proto/api/v1/memo_service.proto index 26f94e2..a4946d3 100644 --- a/proto/api/v1/memo_service.proto +++ b/proto/api/v1/memo_service.proto @@ -207,6 +207,8 @@ message MemoProperty { repeated string tags = 1; bool has_link = 2; bool has_task_list = 3; + bool has_code = 4; + bool has_incomplete_tasks = 5; } message CreateMemoRequest { diff --git a/proto/api/v1/webhook_service.proto b/proto/api/v1/webhook_service.proto index c949d43..145c24f 100644 --- a/proto/api/v1/webhook_service.proto +++ b/proto/api/v1/webhook_service.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package memos.api.v1; import "api/v1/common.proto"; +import "api/v1/memo_service.proto"; import "google/api/annotations.proto"; import "google/api/client.proto"; import "google/protobuf/empty.proto"; @@ -48,9 +49,9 @@ message Webhook { int32 creator_id = 2; - google.protobuf.Timestamp created_time = 3; + google.protobuf.Timestamp create_time = 3; - google.protobuf.Timestamp updated_time = 4; + google.protobuf.Timestamp update_time = 4; RowStatus row_status = 5; @@ -86,3 +87,15 @@ message UpdateWebhookRequest { message DeleteWebhookRequest { int32 id = 1; } + +message WebhookRequestPayload { + string url = 1; + + string activity_type = 2; + + int32 creator_id = 3; + + google.protobuf.Timestamp create_time = 4; + + Memo memo = 5; +} diff --git a/proto/api/v1/workspace_setting_service.proto b/proto/api/v1/workspace_setting_service.proto index ac5693c..69bf8d6 100644 --- a/proto/api/v1/workspace_setting_service.proto +++ b/proto/api/v1/workspace_setting_service.proto @@ -36,18 +36,16 @@ message WorkspaceSetting { } message WorkspaceGeneralSetting { - // instance_url is the instance URL. - string instance_url = 1; // disallow_signup is the flag to disallow signup. - bool disallow_signup = 2; + bool disallow_signup = 1; // disallow_password_login is the flag to disallow password login. - bool disallow_password_login = 3; + bool disallow_password_login = 2; // additional_script is the additional script. - string additional_script = 4; + string additional_script = 3; // additional_style is the additional style. - string additional_style = 5; + string additional_style = 4; // custom_profile is the custom profile. - WorkspaceCustomProfile custom_profile = 6; + WorkspaceCustomProfile custom_profile = 5; } message WorkspaceCustomProfile { @@ -94,6 +92,10 @@ message WorkspaceMemoRelatedSetting { bool display_with_update_time = 2; // content_length_limit is the limit of content length. Unit is byte. int32 content_length_limit = 3; + // enable_auto_compact enables auto compact for large content. + bool enable_auto_compact = 4; + // enable_double_click_edit enables editing on double click. + bool enable_double_click_edit = 5; } message GetWorkspaceSettingRequest { diff --git a/tests/.gitignore b/tests/.gitignore index b79834a..25b18de 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1 +1,2 @@ -.download \ No newline at end of file +.download +docker_compose.yaml diff --git a/tests/docker_compose.yaml b/tests/docker_compose.example.yaml similarity index 92% rename from tests/docker_compose.yaml rename to tests/docker_compose.example.yaml index bfa4670..8519d48 100644 --- a/tests/docker_compose.yaml +++ b/tests/docker_compose.example.yaml @@ -3,7 +3,7 @@ services: memos: networks: - memos_webhook_test - image: neosmemo/memos:0.22.1 + image: neosmemo/memos:0.22.2 ports: - 5240:5230 webhook: diff --git a/tests/run.sh b/tests/run.sh index b400aef..9e85fad 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1 +1,5 @@ -docker-compose -f tests/docker_compose.yaml up -d \ No newline at end of file +if [ ! -f tests/docker_compose.yaml ]; then + cp tests/docker_compose.example.yaml tests/docker_compose.yaml +fi + +docker-compose -f tests/docker_compose.yaml up -d --no-deps --build \ No newline at end of file