feat(platform): Support manually setting up webhooks (#8750)

- Resolves #8748

The webhooks system as is works really well for full blown enterprise
webhooks managed via a UI. It does not work for more "chill guy" webhook
tools that just send notifications sometimes.

## Changes 🏗️

- feat(blocks): Add Compass transcription trigger block

- feat(backend): Amend webhooks system to support manual-set-up webhooks
   - Make event filter input optional on webhook-triggered nodes
   - Make credentials optional on webhook-triggered nodes
   - Add code path to re-use existing manual webhook on graph update
   - Add `ManualWebhookManagerBase`

- feat(frontend): Add UI to pass webhook URL to user on manual-set-up
webhook blocks

![image](https://github.com/user-attachments/assets/1c35f161-7fe4-4916-8506-5ca9a838f398)

- fix(backend): Strip webhook info from node objects for graph export

- refactor(backend): Rename `backend.integrations.webhooks.base` to
`._base`

---------

Co-authored-by: Reinier van der Leer <pwuts@agpt.co>
Co-authored-by: Zamil Majdy <zamil.majdy@agpt.co>

autogpt_platform/backend/backend/blocks/compass/triggers.py

@@ -0,0 +1,59 @@
+from pydantic import BaseModel
+
+from backend.data.block import (
+    Block,
+    BlockCategory,
+    BlockManualWebhookConfig,
+    BlockOutput,
+    BlockSchema,
+)
+from backend.data.model import SchemaField
+from backend.integrations.webhooks.compass import CompassWebhookType
+
+
+class Transcription(BaseModel):
+    text: str
+    speaker: str
+    end: float
+    start: float
+    duration: float
+
+
+class TranscriptionDataModel(BaseModel):
+    date: str
+    transcription: str
+    transcriptions: list[Transcription]
+
+
+class CompassAITriggerBlock(Block):
+    class Input(BlockSchema):
+        payload: TranscriptionDataModel = SchemaField(hidden=True)
+
+    class Output(BlockSchema):
+        transcription: str = SchemaField(
+            description="The contents of the compass transcription."
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="9464a020-ed1d-49e1-990f-7f2ac924a2b7",
+            description="This block will output the contents of the compass transcription.",
+            categories={BlockCategory.HARDWARE},
+            input_schema=CompassAITriggerBlock.Input,
+            output_schema=CompassAITriggerBlock.Output,
+            webhook_config=BlockManualWebhookConfig(
+                provider="compass",
+                webhook_type=CompassWebhookType.TRANSCRIPTION,
+            ),
+            test_input=[
+                {"input": "Hello, World!"},
+                {"input": "Hello, World!", "data": "Existing Data"},
+            ],
+            # test_output=[
+            #     ("output", "Hello, World!"),  # No data provided, so trigger is returned
+            #     ("output", "Existing Data"),  # Data is provided, so data is returned.
+            # ],
+        )
+
+    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+        yield "transcription", input_data.payload.transcription

autogpt_platform/backend/backend/data/block.py

@@ -42,6 +42,7 @@ class BlockType(Enum):
     OUTPUT = "Output"
     NOTE = "Note"
     WEBHOOK = "Webhook"
+    WEBHOOK_MANUAL = "Webhook (manual)"
     AGENT = "Agent"
 
 
@@ -57,6 +58,7 @@ class BlockCategory(Enum):
     COMMUNICATION = "Block that interacts with communication platforms."
     DEVELOPER_TOOLS = "Developer tools such as GitHub blocks."
     DATA = "Block that interacts with structured data."
+    HARDWARE = "Block that interacts with hardware."
     AGENT = "Block that interacts with other agents."
     CRM = "Block that interacts with CRM services."
 
@@ -197,7 +199,12 @@ class EmptySchema(BlockSchema):
 
 
 # --8<-- [start:BlockWebhookConfig]
-class BlockWebhookConfig(BaseModel):
+class BlockManualWebhookConfig(BaseModel):
+    """
+    Configuration model for webhook-triggered blocks on which
+    the user has to manually set up the webhook at the provider.
+    """
+
     provider: str
     """The service provider that the webhook connects to"""
 
@@ -208,6 +215,27 @@ class BlockWebhookConfig(BaseModel):
     Only for use in the corresponding `WebhooksManager`.
     """
 
+    event_filter_input: str = ""
+    """
+    Name of the block's event filter input.
+    Leave empty if the corresponding webhook doesn't have distinct event/payload types.
+    """
+
+    event_format: str = "{event}"
+    """
+    Template string for the event(s) that a block instance subscribes to.
+    Applied individually to each event selected in the event filter input.
+
+    Example: `"pull_request.{event}"` -> `"pull_request.opened"`
+    """
+
+
+class BlockWebhookConfig(BlockManualWebhookConfig):
+    """
+    Configuration model for webhook-triggered blocks for which
+    the webhook can be automatically set up through the provider's API.
+    """
+
     resource_format: str
     """
     Template string for the resource that a block instance subscribes to.
@@ -217,17 +245,6 @@ class BlockWebhookConfig(BaseModel):
 
     Only for use in the corresponding `WebhooksManager`.
     """
-
-    event_filter_input: str
-    """Name of the block's event filter input."""
-
-    event_format: str = "{event}"
-    """
-    Template string for the event(s) that a block instance subscribes to.
-    Applied individually to each event selected in the event filter input.
-
-    Example: `"pull_request.{event}"` -> `"pull_request.opened"`
-    """
     # --8<-- [end:BlockWebhookConfig]
 
 
@@ -247,7 +264,7 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):
         disabled: bool = False,
         static_output: bool = False,
         block_type: BlockType = BlockType.STANDARD,
-        webhook_config: Optional[BlockWebhookConfig] = None,
+        webhook_config: Optional[BlockWebhookConfig | BlockManualWebhookConfig] = None,
     ):
         """
         Initialize the block with the given schema.
@@ -278,27 +295,38 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):
         self.contributors = contributors or set()
         self.disabled = disabled
         self.static_output = static_output
-        self.block_type = block_type if not webhook_config else BlockType.WEBHOOK
+        self.block_type = block_type
         self.webhook_config = webhook_config
         self.execution_stats = {}
 
         if self.webhook_config:
-            # Enforce shape of webhook event filter
+            if isinstance(self.webhook_config, BlockWebhookConfig):
-            event_filter_field = self.input_schema.model_fields[
+                # Enforce presence of credentials field on auto-setup webhook blocks
-                self.webhook_config.event_filter_input
+                if CREDENTIALS_FIELD_NAME not in self.input_schema.model_fields:
-            ]
+                    raise TypeError(
-            if not (
+                        "credentials field is required on auto-setup webhook blocks"
-                isinstance(event_filter_field.annotation, type)
+                    )
-                and issubclass(event_filter_field.annotation, BaseModel)
+                self.block_type = BlockType.WEBHOOK
-                and all(
+            else:
-                    field.annotation is bool
+                self.block_type = BlockType.WEBHOOK_MANUAL
-                    for field in event_filter_field.annotation.model_fields.values()
+
-                )
+            # Enforce shape of webhook event filter, if present
-            ):
+            if self.webhook_config.event_filter_input:
-                raise NotImplementedError(
+                event_filter_field = self.input_schema.model_fields[
-                    f"{self.name} has an invalid webhook event selector: "
+                    self.webhook_config.event_filter_input
-                    "field must be a BaseModel and all its fields must be boolean"
+                ]
-                )
+                if not (
+                    isinstance(event_filter_field.annotation, type)
+                    and issubclass(event_filter_field.annotation, BaseModel)
+                    and all(
+                        field.annotation is bool
+                        for field in event_filter_field.annotation.model_fields.values()
+                    )
+                ):
+                    raise NotImplementedError(
+                        f"{self.name} has an invalid webhook event selector: "
+                        "field must be a BaseModel and all its fields must be boolean"
+                    )
 
             # Enforce presence of 'payload' input
             if "payload" not in self.input_schema.model_fields:

autogpt_platform/backend/backend/data/graph.py

@@ -84,6 +84,8 @@ class NodeModel(Node):
             raise ValueError(f"Block #{self.block_id} not found for node #{self.id}")
         if not block.webhook_config:
             raise TypeError("This method can't be used on non-webhook blocks")
+        if not block.webhook_config.event_filter_input:
+            return True
         event_filter = self.input_default.get(block.webhook_config.event_filter_input)
         if not event_filter:
             raise ValueError(f"Event filter is not configured on node #{self.id}")
@@ -268,11 +270,19 @@ class GraphModel(Graph):
                 + [sanitize(link.sink_name) for link in input_links.get(node.id, [])]
             )
             for name in block.input_schema.get_required_fields():
-                if name not in provided_inputs and (
+                if (
-                    for_run  # Skip input completion validation, unless when executing.
+                    name not in provided_inputs
-                    or block.block_type == BlockType.INPUT
+                    and not (
-                    or block.block_type == BlockType.OUTPUT
+                        name == "payload"
-                    or block.block_type == BlockType.AGENT
+                        and block.block_type
+                        in (BlockType.WEBHOOK, BlockType.WEBHOOK_MANUAL)
+                    )
+                    and (
+                        for_run  # Skip input completion validation, unless when executing.
+                        or block.block_type == BlockType.INPUT
+                        or block.block_type == BlockType.OUTPUT
+                        or block.block_type == BlockType.AGENT
+                    )
                 ):
                     raise ValueError(
                         f"Node {block.name} #{node.id} required input missing: `{name}`"
@@ -292,7 +302,6 @@ class GraphModel(Graph):
 
             # Validate dependencies between fields
             for field_name, field_info in input_schema.items():
-
                 # Apply input dependency validation only on run & field with depends_on
                 json_schema_extra = field_info.json_schema_extra or {}
                 dependencies = json_schema_extra.get("depends_on", [])
@@ -359,7 +368,7 @@ class GraphModel(Graph):
                 link.is_static = True  # Each value block output should be static.
 
     @staticmethod
-    def from_db(graph: AgentGraph, hide_credentials: bool = False):
+    def from_db(graph: AgentGraph, for_export: bool = False):
         return GraphModel(
             id=graph.id,
             user_id=graph.userId,
@@ -369,7 +378,7 @@ class GraphModel(Graph):
             name=graph.name or "",
             description=graph.description or "",
             nodes=[
-                GraphModel._process_node(node, hide_credentials)
+                NodeModel.from_db(GraphModel._process_node(node, for_export))
                 for node in graph.AgentNodes or []
             ],
             links=list(
@@ -382,23 +391,29 @@ class GraphModel(Graph):
         )
 
     @staticmethod
-    def _process_node(node: AgentNode, hide_credentials: bool) -> NodeModel:
+    def _process_node(node: AgentNode, for_export: bool) -> AgentNode:
-        node_dict = {field: getattr(node, field) for field in node.model_fields}
+        if for_export:
-        if hide_credentials and "constantInput" in node_dict:
+            # Remove credentials from node input
-            constant_input = json.loads(
+            if node.constantInput:
-                node_dict["constantInput"], target_type=dict[str, Any]
+                constant_input = json.loads(
-            )
+                    node.constantInput, target_type=dict[str, Any]
-            constant_input = GraphModel._hide_credentials_in_input(constant_input)
+                )
-            node_dict["constantInput"] = json.dumps(constant_input)
+                constant_input = GraphModel._hide_node_input_credentials(constant_input)
-        return NodeModel.from_db(AgentNode(**node_dict))
+                node.constantInput = json.dumps(constant_input)
+
+            # Remove webhook info
+            node.webhookId = None
+            node.Webhook = None
+
+        return node
 
     @staticmethod
-    def _hide_credentials_in_input(input_data: dict[str, Any]) -> dict[str, Any]:
+    def _hide_node_input_credentials(input_data: dict[str, Any]) -> dict[str, Any]:
         sensitive_keys = ["credentials", "api_key", "password", "token", "secret"]
         result = {}
         for key, value in input_data.items():
             if isinstance(value, dict):
-                result[key] = GraphModel._hide_credentials_in_input(value)
+                result[key] = GraphModel._hide_node_input_credentials(value)
             elif isinstance(value, str) and any(
                 sensitive_key in key.lower() for sensitive_key in sensitive_keys
             ):
@@ -495,7 +510,7 @@ async def get_graph(
     version: int | None = None,
     template: bool = False,
     user_id: str | None = None,
-    hide_credentials: bool = False,
+    for_export: bool = False,
 ) -> GraphModel | None:
     """
     Retrieves a graph from the DB.
@@ -521,7 +536,7 @@ async def get_graph(
         include=AGENT_GRAPH_INCLUDE,
         order={"version": "desc"},
     )
-    return GraphModel.from_db(graph, hide_credentials) if graph else None
+    return GraphModel.from_db(graph, for_export) if graph else None
 
 
 async def set_graph_active_version(graph_id: str, version: int, user_id: str) -> None:

autogpt_platform/backend/backend/data/integrations.py

@@ -3,11 +3,12 @@ from typing import TYPE_CHECKING, AsyncGenerator, Optional
 
 from prisma import Json
 from prisma.models import IntegrationWebhook
-from pydantic import Field
+from pydantic import Field, computed_field
 
 from backend.data.includes import INTEGRATION_WEBHOOK_INCLUDE
 from backend.data.queue import AsyncRedisEventBus
 from backend.integrations.providers import ProviderName
+from backend.integrations.webhooks.utils import webhook_ingress_url
 
 from .db import BaseDbModel
 
@@ -31,6 +32,11 @@ class Webhook(BaseDbModel):
 
     attached_nodes: Optional[list["NodeModel"]] = None
 
+    @computed_field
+    @property
+    def url(self) -> str:
+        return webhook_ingress_url(self.provider.value, self.id)
+
     @staticmethod
     def from_db(webhook: IntegrationWebhook):
         from .graph import NodeModel
@@ -84,8 +90,10 @@ async def get_webhook(webhook_id: str) -> Webhook:
     return Webhook.from_db(webhook)
 
 
-async def get_all_webhooks(credentials_id: str) -> list[Webhook]:
+async def get_all_webhooks_by_creds(credentials_id: str) -> list[Webhook]:
     """⚠️ No `user_id` check: DO NOT USE without check in user-facing endpoints."""
+    if not credentials_id:
+        raise ValueError("credentials_id must not be empty")
     webhooks = await IntegrationWebhook.prisma().find_many(
         where={"credentialsId": credentials_id},
         include=INTEGRATION_WEBHOOK_INCLUDE,
@@ -93,7 +101,7 @@ async def get_all_webhooks(credentials_id: str) -> list[Webhook]:
     return [Webhook.from_db(webhook) for webhook in webhooks]
 
 
-async def find_webhook(
+async def find_webhook_by_credentials_and_props(
     credentials_id: str, webhook_type: str, resource: str, events: list[str]
 ) -> Webhook | None:
     """⚠️ No `user_id` check: DO NOT USE without check in user-facing endpoints."""
@@ -109,6 +117,22 @@ async def find_webhook(
     return Webhook.from_db(webhook) if webhook else None
 
 
+async def find_webhook_by_graph_and_props(
+    graph_id: str, provider: str, webhook_type: str, events: list[str]
+) -> Webhook | None:
+    """⚠️ No `user_id` check: DO NOT USE without check in user-facing endpoints."""
+    webhook = await IntegrationWebhook.prisma().find_first(
+        where={
+            "provider": provider,
+            "webhookType": webhook_type,
+            "events": {"has_every": events},
+            "AgentNodes": {"some": {"agentGraphId": graph_id}},
+        },
+        include=INTEGRATION_WEBHOOK_INCLUDE,
+    )
+    return Webhook.from_db(webhook) if webhook else None
+
+
 async def update_webhook_config(webhook_id: str, updated_config: dict) -> Webhook:
     """⚠️ No `user_id` check: DO NOT USE without check in user-facing endpoints."""
     _updated_webhook = await IntegrationWebhook.prisma().update(

autogpt_platform/backend/backend/executor/manager.py

@@ -798,10 +798,13 @@ class ExecutionManager(AppService):
             # Extract webhook payload, and assign it to the input pin
             webhook_payload_key = f"webhook_{node.webhook_id}_payload"
             if (
-                block.block_type == BlockType.WEBHOOK
+                block.block_type in (BlockType.WEBHOOK, BlockType.WEBHOOK_MANUAL)
                 and node.webhook_id
-                and webhook_payload_key in data
             ):
+                if webhook_payload_key not in data:
+                    raise ValueError(
+                        f"Node {block.name} #{node.id} webhook payload is missing"
+                    )
                 input_data = {"payload": data[webhook_payload_key]}
 
             input_data, error = validate_exec(node, input_data)

autogpt_platform/backend/backend/integrations/providers.py

@@ -4,6 +4,7 @@ from enum import Enum
 # --8<-- [start:ProviderName]
 class ProviderName(str, Enum):
     ANTHROPIC = "anthropic"
+    COMPASS = "compass"
     DISCORD = "discord"
     D_ID = "d_id"
     E2B = "e2b"

autogpt_platform/backend/backend/integrations/webhooks/__init__.py

@@ -1,16 +1,18 @@
 from typing import TYPE_CHECKING
 
+from .compass import CompassWebhookManager
 from .github import GithubWebhooksManager
 from .slant3d import Slant3DWebhooksManager
 
 if TYPE_CHECKING:
     from ..providers import ProviderName
-    from .base import BaseWebhooksManager
+    from ._base import BaseWebhooksManager
 
 # --8<-- [start:WEBHOOK_MANAGERS_BY_NAME]
 WEBHOOK_MANAGERS_BY_NAME: dict["ProviderName", type["BaseWebhooksManager"]] = {
     handler.PROVIDER_NAME: handler
     for handler in [
+        CompassWebhookManager,
         GithubWebhooksManager,
         Slant3DWebhooksManager,
     ]

autogpt_platform/backend/backend/integrations/webhooks/_base.py

@@ -1,7 +1,7 @@
 import logging
 import secrets
 from abc import ABC, abstractmethod
-from typing import ClassVar, Generic, TypeVar
+from typing import ClassVar, Generic, Optional, TypeVar
 from uuid import uuid4
 
 from fastapi import Request
@@ -10,6 +10,7 @@ from strenum import StrEnum
 from backend.data import integrations
 from backend.data.model import Credentials
 from backend.integrations.providers import ProviderName
+from backend.integrations.webhooks.utils import webhook_ingress_url
 from backend.util.exceptions import MissingConfigError
 from backend.util.settings import Config
 
@@ -26,7 +27,7 @@ class BaseWebhooksManager(ABC, Generic[WT]):
 
     WebhookType: WT
 
-    async def get_suitable_webhook(
+    async def get_suitable_auto_webhook(
         self,
         user_id: str,
         credentials: Credentials,
@@ -39,16 +40,34 @@ class BaseWebhooksManager(ABC, Generic[WT]):
                 "PLATFORM_BASE_URL must be set to use Webhook functionality"
             )
 
-        if webhook := await integrations.find_webhook(
+        if webhook := await integrations.find_webhook_by_credentials_and_props(
             credentials.id, webhook_type, resource, events
         ):
             return webhook
         return await self._create_webhook(
-            user_id, credentials, webhook_type, resource, events
+            user_id, webhook_type, events, resource, credentials
+        )
+
+    async def get_manual_webhook(
+        self,
+        user_id: str,
+        graph_id: str,
+        webhook_type: WT,
+        events: list[str],
+    ):
+        if current_webhook := await integrations.find_webhook_by_graph_and_props(
+            graph_id, self.PROVIDER_NAME, webhook_type, events
+        ):
+            return current_webhook
+        return await self._create_webhook(
+            user_id,
+            webhook_type,
+            events,
+            register=False,
         )
 
     async def prune_webhook_if_dangling(
-        self, webhook_id: str, credentials: Credentials
+        self, webhook_id: str, credentials: Optional[Credentials]
     ) -> bool:
         webhook = await integrations.get_webhook(webhook_id)
         if webhook.attached_nodes is None:
@@ -57,7 +76,8 @@ class BaseWebhooksManager(ABC, Generic[WT]):
             # Don't prune webhook if in use
             return False
 
-        await self._deregister_webhook(webhook, credentials)
+        if credentials:
+            await self._deregister_webhook(webhook, credentials)
         await integrations.delete_webhook(webhook.id)
         return True
 
@@ -135,27 +155,36 @@ class BaseWebhooksManager(ABC, Generic[WT]):
     async def _create_webhook(
         self,
         user_id: str,
-        credentials: Credentials,
         webhook_type: WT,
-        resource: str,
         events: list[str],
+        resource: str = "",
+        credentials: Optional[Credentials] = None,
+        register: bool = True,
     ) -> integrations.Webhook:
+        if not app_config.platform_base_url:
+            raise MissingConfigError(
+                "PLATFORM_BASE_URL must be set to use Webhook functionality"
+            )
+
         id = str(uuid4())
         secret = secrets.token_hex(32)
         provider_name = self.PROVIDER_NAME
-        ingress_url = (
+        ingress_url = webhook_ingress_url(provider_name=provider_name, webhook_id=id)
-            f"{app_config.platform_base_url}/api/integrations/{provider_name.value}"
+        if register:
-            f"/webhooks/{id}/ingress"
+            if not credentials:
-        )
+                raise TypeError("credentials are required if register = True")
-        provider_webhook_id, config = await self._register_webhook(
+            provider_webhook_id, config = await self._register_webhook(
-            credentials, webhook_type, resource, events, ingress_url, secret
+                credentials, webhook_type, resource, events, ingress_url, secret
-        )
+            )
+        else:
+            provider_webhook_id, config = "", {}
+
         return await integrations.create_webhook(
             integrations.Webhook(
                 id=id,
                 user_id=user_id,
                 provider=provider_name,
-                credentials_id=credentials.id,
+                credentials_id=credentials.id if credentials else "",
                 webhook_type=webhook_type,
                 resource=resource,
                 events=events,

autogpt_platform/backend/backend/integrations/webhooks/_manual_base.py

@@ -0,0 +1,30 @@
+import logging
+
+from backend.data import integrations
+from backend.data.model import APIKeyCredentials, Credentials, OAuth2Credentials
+
+from ._base import WT, BaseWebhooksManager
+
+logger = logging.getLogger(__name__)
+
+
+class ManualWebhookManagerBase(BaseWebhooksManager[WT]):
+    async def _register_webhook(
+        self,
+        credentials: Credentials,
+        webhook_type: WT,
+        resource: str,
+        events: list[str],
+        ingress_url: str,
+        secret: str,
+    ) -> tuple[str, dict]:
+        print(ingress_url)  # FIXME: pass URL to user in front end
+
+        return "", {}
+
+    async def _deregister_webhook(
+        self,
+        webhook: integrations.Webhook,
+        credentials: OAuth2Credentials | APIKeyCredentials,
+    ) -> None:
+        pass

autogpt_platform/backend/backend/integrations/webhooks/compass.py

@@ -0,0 +1,30 @@
+import logging
+
+from fastapi import Request
+from strenum import StrEnum
+
+from backend.data import integrations
+from backend.integrations.providers import ProviderName
+
+from ._manual_base import ManualWebhookManagerBase
+
+logger = logging.getLogger(__name__)
+
+
+class CompassWebhookType(StrEnum):
+    TRANSCRIPTION = "transcription"
+    TASK = "task"
+
+
+class CompassWebhookManager(ManualWebhookManagerBase):
+    PROVIDER_NAME = ProviderName.COMPASS
+    WebhookType = CompassWebhookType
+
+    @classmethod
+    async def validate_payload(
+        cls, webhook: integrations.Webhook, request: Request
+    ) -> tuple[dict, str]:
+        payload = await request.json()
+        event_type = CompassWebhookType.TRANSCRIPTION  # currently the only type
+
+        return payload, event_type

autogpt_platform/backend/backend/integrations/webhooks/github.py

@@ -10,7 +10,7 @@ from backend.data import integrations
 from backend.data.model import Credentials
 from backend.integrations.providers import ProviderName
 
-from .base import BaseWebhooksManager
+from ._base import BaseWebhooksManager
 
 logger = logging.getLogger(__name__)
 

autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py

@@ -1,7 +1,7 @@
 import logging
 from typing import TYPE_CHECKING, Callable, Optional, cast
 
-from backend.data.block import get_block
+from backend.data.block import BlockWebhookConfig, get_block
 from backend.data.graph import set_node_webhook
 from backend.data.model import CREDENTIALS_FIELD_NAME
 from backend.integrations.webhooks import WEBHOOK_MANAGERS_BY_NAME
@@ -10,7 +10,7 @@ if TYPE_CHECKING:
     from backend.data.graph import GraphModel, NodeModel
     from backend.data.model import Credentials
 
-    from .base import BaseWebhooksManager
+    from ._base import BaseWebhooksManager
 
 logger = logging.getLogger(__name__)
 
@@ -108,50 +108,79 @@ async def on_node_activate(
 
     webhooks_manager = WEBHOOK_MANAGERS_BY_NAME[provider]()
 
-    try:
+    if auto_setup_webhook := isinstance(block.webhook_config, BlockWebhookConfig):
-        resource = block.webhook_config.resource_format.format(**node.input_default)
+        try:
-    except KeyError:
+            resource = block.webhook_config.resource_format.format(**node.input_default)
-        resource = None
+        except KeyError:
-    logger.debug(
+            resource = None
-        f"Constructed resource string {resource} from input {node.input_default}"
+        logger.debug(
-    )
+            f"Constructed resource string {resource} from input {node.input_default}"
+        )
+    else:
+        resource = ""  # not relevant for manual webhooks
 
+    needs_credentials = CREDENTIALS_FIELD_NAME in block.input_schema.model_fields
+    credentials_meta = (
+        node.input_default.get(CREDENTIALS_FIELD_NAME) if needs_credentials else None
+    )
     event_filter_input_name = block.webhook_config.event_filter_input
     has_everything_for_webhook = (
         resource is not None
-        and CREDENTIALS_FIELD_NAME in node.input_default
+        and (credentials_meta or not needs_credentials)
-        and event_filter_input_name in node.input_default
+        and (
-        and any(is_on for is_on in node.input_default[event_filter_input_name].values())
+            not event_filter_input_name
+            or (
+                event_filter_input_name in node.input_default
+                and any(
+                    is_on
+                    for is_on in node.input_default[event_filter_input_name].values()
+                )
+            )
+        )
     )
 
-    if has_everything_for_webhook and resource:
+    if has_everything_for_webhook and resource is not None:
         logger.debug(f"Node #{node} has everything for a webhook!")
-        if not credentials:
+        if credentials_meta and not credentials:
-            credentials_meta = node.input_default[CREDENTIALS_FIELD_NAME]
             raise ValueError(
                 f"Cannot set up webhook for node #{node.id}: "
                 f"credentials #{credentials_meta['id']} not available"
             )
 
-        # Shape of the event filter is enforced in Block.__init__
+        if event_filter_input_name:
-        event_filter = cast(dict, node.input_default[event_filter_input_name])
+            # Shape of the event filter is enforced in Block.__init__
-        events = [
+            event_filter = cast(dict, node.input_default[event_filter_input_name])
-            block.webhook_config.event_format.format(event=event)
+            events = [
-            for event, enabled in event_filter.items()
+                block.webhook_config.event_format.format(event=event)
-            if enabled is True
+                for event, enabled in event_filter.items()
-        ]
+                if enabled is True
-        logger.debug(f"Webhook events to subscribe to: {', '.join(events)}")
+            ]
+            logger.debug(f"Webhook events to subscribe to: {', '.join(events)}")
+        else:
+            events = []
 
         # Find/make and attach a suitable webhook to the node
-        new_webhook = await webhooks_manager.get_suitable_webhook(
+        if auto_setup_webhook:
-            user_id,
+            assert credentials is not None
-            credentials,
+            new_webhook = await webhooks_manager.get_suitable_auto_webhook(
-            block.webhook_config.webhook_type,
+                user_id,
-            resource,
+                credentials,
-            events,
+                block.webhook_config.webhook_type,
-        )
+                resource,
+                events,
+            )
+        else:
+            # Manual webhook -> no credentials -> don't register but do create
+            new_webhook = await webhooks_manager.get_manual_webhook(
+                user_id,
+                node.graph_id,
+                block.webhook_config.webhook_type,
+                events,
+            )
         logger.debug(f"Acquired webhook: {new_webhook}")
         return await set_node_webhook(node.id, new_webhook.id)
+    else:
+        logger.debug(f"Node #{node.id} does not have everything for a webhook")
 
     return node
 
@@ -194,12 +223,16 @@ async def on_node_deactivate(
         updated_node = await set_node_webhook(node.id, None)
 
         # Prune and deregister the webhook if it is no longer used anywhere
-        logger.debug("Pruning and deregistering webhook if dangling")
         webhook = node.webhook
-        if credentials:
+        logger.debug(
-            logger.debug(f"Pruning webhook #{webhook.id} with credentials")
+            f"Pruning{' and deregistering' if credentials else ''} "
-            await webhooks_manager.prune_webhook_if_dangling(webhook.id, credentials)
+            f"webhook #{webhook.id}"
-        else:
+        )
+        await webhooks_manager.prune_webhook_if_dangling(webhook.id, credentials)
+        if (
+            CREDENTIALS_FIELD_NAME in block.input_schema.model_fields
+            and not credentials
+        ):
             logger.warning(
                 f"Cannot deregister webhook #{webhook.id}: credentials "
                 f"#{webhook.credentials_id} not available "

autogpt_platform/backend/backend/integrations/webhooks/slant3d.py

@@ -6,7 +6,7 @@ from fastapi import Request
 from backend.data import integrations
 from backend.data.model import APIKeyCredentials, Credentials
 from backend.integrations.providers import ProviderName
-from backend.integrations.webhooks.base import BaseWebhooksManager
+from backend.integrations.webhooks._base import BaseWebhooksManager
 
 logger = logging.getLogger(__name__)
 

autogpt_platform/backend/backend/integrations/webhooks/utils.py

@@ -0,0 +1,11 @@
+from backend.util.settings import Config
+
+app_config = Config()
+
+
+# TODO: add test to assert this matches the actual API route
+def webhook_ingress_url(provider_name: str, webhook_id: str) -> str:
+    return (
+        f"{app_config.platform_base_url}/api/integrations/{provider_name}"
+        f"/webhooks/{webhook_id}/ingress"
+    )

autogpt_platform/backend/backend/server/integrations/router.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field, SecretStr
 from backend.data.graph import set_node_webhook
 from backend.data.integrations import (
     WebhookEvent,
-    get_all_webhooks,
+    get_all_webhooks_by_creds,
     get_webhook,
     publish_webhook_event,
     wait_for_webhook_event,
@@ -363,7 +363,7 @@ async def remove_all_webhooks_for_credentials(
     Raises:
         NeedConfirmation: If any of the webhooks are still in use and `force` is `False`
     """
-    webhooks = await get_all_webhooks(credentials.id)
+    webhooks = await get_all_webhooks_by_creds(credentials.id)
     if credentials.provider not in WEBHOOK_MANAGERS_BY_NAME:
         if webhooks:
             logger.error(

autogpt_platform/backend/backend/server/routers/v1.py

@@ -149,7 +149,7 @@ class DeleteGraphResponse(TypedDict):
 @v1_router.get(path="/graphs", tags=["graphs"], dependencies=[Depends(auth_middleware)])
 async def get_graphs(
     user_id: Annotated[str, Depends(get_user_id)]
-) -> Sequence[graph_db.Graph]:
+) -> Sequence[graph_db.GraphModel]:
     return await graph_db.get_graphs(filter_by="active", user_id=user_id)
 
 
@@ -166,9 +166,9 @@ async def get_graph(
     user_id: Annotated[str, Depends(get_user_id)],
     version: int | None = None,
     hide_credentials: bool = False,
-) -> graph_db.Graph:
+) -> graph_db.GraphModel:
     graph = await graph_db.get_graph(
-        graph_id, version, user_id=user_id, hide_credentials=hide_credentials
+        graph_id, version, user_id=user_id, for_export=hide_credentials
     )
     if not graph:
         raise HTTPException(status_code=404, detail=f"Graph #{graph_id} not found.")
@@ -187,7 +187,7 @@ async def get_graph(
 )
 async def get_graph_all_versions(
     graph_id: str, user_id: Annotated[str, Depends(get_user_id)]
-) -> Sequence[graph_db.Graph]:
+) -> Sequence[graph_db.GraphModel]:
     graphs = await graph_db.get_graph_all_versions(graph_id, user_id=user_id)
     if not graphs:
         raise HTTPException(status_code=404, detail=f"Graph #{graph_id} not found.")
@@ -199,7 +199,7 @@ async def get_graph_all_versions(
 )
 async def create_new_graph(
     create_graph: CreateGraph, user_id: Annotated[str, Depends(get_user_id)]
-) -> graph_db.Graph:
+) -> graph_db.GraphModel:
     return await do_create_graph(create_graph, is_template=False, user_id=user_id)
 
 
@@ -209,7 +209,7 @@ async def do_create_graph(
     # user_id doesn't have to be annotated like on other endpoints,
     # because create_graph isn't used directly as an endpoint
     user_id: str,
-) -> graph_db.Graph:
+) -> graph_db.GraphModel:
     if create_graph.graph:
         graph = graph_db.make_graph_model(create_graph.graph, user_id)
     elif create_graph.template_id:
@@ -270,7 +270,7 @@ async def update_graph(
     graph_id: str,
     graph: graph_db.Graph,
     user_id: Annotated[str, Depends(get_user_id)],
-) -> graph_db.Graph:
+) -> graph_db.GraphModel:
     # Sanity check
     if graph.id and graph.id != graph_id:
         raise HTTPException(400, detail="Graph ID does not match ID in URI")
@@ -440,7 +440,7 @@ async def get_graph_run_node_execution_results(
 )
 async def get_templates(
     user_id: Annotated[str, Depends(get_user_id)]
-) -> Sequence[graph_db.Graph]:
+) -> Sequence[graph_db.GraphModel]:
     return await graph_db.get_graphs(filter_by="template", user_id=user_id)
 
 
@@ -449,7 +449,9 @@ async def get_templates(
     tags=["templates", "graphs"],
     dependencies=[Depends(auth_middleware)],
 )
-async def get_template(graph_id: str, version: int | None = None) -> graph_db.Graph:
+async def get_template(
+    graph_id: str, version: int | None = None
+) -> graph_db.GraphModel:
     graph = await graph_db.get_graph(graph_id, version, template=True)
     if not graph:
         raise HTTPException(status_code=404, detail=f"Template #{graph_id} not found.")
@@ -463,7 +465,7 @@ async def get_template(graph_id: str, version: int | None = None) -> graph_db.Gr
 )
 async def create_new_template(
     create_graph: CreateGraph, user_id: Annotated[str, Depends(get_user_id)]
-) -> graph_db.Graph:
+) -> graph_db.GraphModel:
     return await do_create_graph(create_graph, is_template=True, user_id=user_id)
 
 

autogpt_platform/frontend/src/components/CustomNode.tsx

@@ -6,7 +6,7 @@ import React, {
   useContext,
   useMemo,
 } from "react";
-import { NodeProps, useReactFlow, Node, Edge } from "@xyflow/react";
+import { NodeProps, useReactFlow, Node as XYNode, Edge } from "@xyflow/react";
 import "@xyflow/react/dist/style.css";
 import "./customnode.css";
 import InputModalComponent from "./InputModalComponent";
@@ -16,6 +16,7 @@ import {
   BlockIOSubSchema,
   BlockIOStringSubSchema,
   Category,
+  Node,
   NodeExecutionResult,
   BlockUIType,
   BlockCost,
@@ -71,7 +72,7 @@ export type CustomNodeData = {
   outputSchema: BlockIORootSchema;
   hardcodedValues: { [key: string]: any };
   connections: ConnectionData;
-  webhookId?: string;
+  webhook?: Node["webhook"];
   isOutputOpen: boolean;
   status?: NodeExecutionResult["status"];
   /** executionResults contains outputs across multiple executions
@@ -87,7 +88,7 @@ export type CustomNodeData = {
   uiType: BlockUIType;
 };
 
-export type CustomNode = Node<CustomNodeData, "custom">;
+export type CustomNode = XYNode<CustomNodeData, "custom">;
 
 export function CustomNode({
   data,
@@ -237,7 +238,11 @@ export function CustomNode({
           const isHidden = propSchema.hidden;
           const isConnectable =
             // No input connection handles on INPUT and WEBHOOK blocks
-            ![BlockUIType.INPUT, BlockUIType.WEBHOOK].includes(nodeType) &&
+            ![
+              BlockUIType.INPUT,
+              BlockUIType.WEBHOOK,
+              BlockUIType.WEBHOOK_MANUAL,
+            ].includes(nodeType) &&
             // No input connection handles for credentials
             propKey !== "credentials" &&
             // For OUTPUT blocks, only show the 'value' (hides 'name') input connection handle
@@ -549,22 +554,25 @@ export function CustomNode({
   >(null);
 
   useEffect(() => {
-    if (data.uiType != BlockUIType.WEBHOOK) return;
+    if (
-    if (!data.webhookId) {
+      ![BlockUIType.WEBHOOK, BlockUIType.WEBHOOK_MANUAL].includes(data.uiType)
+    )
+      return;
+    if (!data.webhook) {
       setWebhookStatus("none");
       return;
     }
 
     setWebhookStatus("pending");
     api
-      .pingWebhook(data.webhookId)
+      .pingWebhook(data.webhook.id)
       .then((pinged) => setWebhookStatus(pinged ? "works" : "exists"))
       .catch((error: Error) =>
         error.message.includes("ping timed out")
           ? setWebhookStatus("broken")
           : setWebhookStatus("none"),
       );
-  }, [data.uiType, data.webhookId, api, setWebhookStatus]);
+  }, [data.uiType, data.webhook, api, setWebhookStatus]);
 
   const webhookStatusDot = useMemo(
     () =>
@@ -726,6 +734,33 @@ export function CustomNode({
             data-id="input-handles"
           >
             <div>
+              {data.uiType === BlockUIType.WEBHOOK_MANUAL &&
+                (data.webhook ? (
+                  <div className="nodrag mr-5 flex flex-col gap-1">
+                    Webhook URL:
+                    <div className="flex gap-2 rounded-md bg-gray-50 p-2">
+                      <code className="select-all text-sm">
+                        {data.webhook.url}
+                      </code>
+                      <Button
+                        variant="outline"
+                        size="icon"
+                        className="size-7 flex-none"
+                        onClick={() =>
+                          data.webhook &&
+                          navigator.clipboard.writeText(data.webhook.url)
+                        }
+                        title="Copy webhook URL"
+                      >
+                        <CopyIcon className="size-4" />
+                      </Button>
+                    </div>
+                  </div>
+                ) : (
+                  <p className="italic text-gray-500">
+                    (A Webhook URL will be generated when you save the agent)
+                  </p>
+                ))}
               {data.inputSchema &&
                 generateInputHandles(data.inputSchema, data.uiType)}
             </div>

autogpt_platform/frontend/src/hooks/useAgentGraph.ts

@@ -169,7 +169,7 @@ export default function useAgentGraph(
               inputSchema: block.inputSchema,
               outputSchema: block.outputSchema,
               hardcodedValues: node.input_default,
-              webhookId: node.webhook_id,
+              webhook: node.webhook,
               uiType: block.uiType,
               connections: graph.links
                 .filter((l) => [l.source_id, l.sink_id].includes(node.id))
@@ -815,7 +815,7 @@ export default function useAgentGraph(
                   ),
                   status: undefined,
                   backend_id: backendNode.id,
-                  webhookId: backendNode.webhook_id,
+                  webhook: backendNode.webhook,
                   executionResults: [],
                 },
               }

autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts

@@ -172,7 +172,7 @@ export type Node = {
     position: { x: number; y: number };
     [key: string]: any;
   };
-  webhook_id?: string;
+  webhook?: Webhook;
 };
 
 /* Mirror of backend/data/graph.py:Link */
@@ -314,6 +314,20 @@ export type APIKeyCredentials = BaseCredentials & {
   expires_at?: number;
 };
 
+/* Mirror of backend/data/integrations.py:Webhook */
+type Webhook = {
+  id: string;
+  url: string;
+  provider: CredentialsProviderName;
+  credentials_id: string;
+  webhook_type: string;
+  resource?: string;
+  events: string[];
+  secret: string;
+  config: Record<string, any>;
+  provider_webhook_id?: string;
+};
+
 export type User = {
   id: string;
   email: string;
@@ -325,6 +339,7 @@ export enum BlockUIType {
   OUTPUT = "Output",
   NOTE = "Note",
   WEBHOOK = "Webhook",
+  WEBHOOK_MANUAL = "Webhook (manual)",
   AGENT = "Agent",
 }