Significant-Gravitas
/
AutoGPT
mirror of https://github.com/Significant-Gravitas/AutoGPT.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

feat(platform): Support manually setting up webhooks (#8750) 

- Resolves #8748

The webhooks system as is works really well for full blown enterprise
webhooks managed via a UI. It does not work for more "chill guy" webhook
tools that just send notifications sometimes.

## Changes 🏗️

- feat(blocks): Add Compass transcription trigger block

- feat(backend): Amend webhooks system to support manual-set-up webhooks
   - Make event filter input optional on webhook-triggered nodes
   - Make credentials optional on webhook-triggered nodes
   - Add code path to re-use existing manual webhook on graph update
   - Add `ManualWebhookManagerBase`

- feat(frontend): Add UI to pass webhook URL to user on manual-set-up
webhook blocks

![image](https://github.com/user-attachments/assets/1c35f161-7fe4-4916-8506-5ca9a838f398)

- fix(backend): Strip webhook info from node objects for graph export

- refactor(backend): Rename `backend.integrations.webhooks.base` to
`._base`

---------

Co-authored-by: Reinier van der Leer <pwuts@agpt.co>
Co-authored-by: Zamil Majdy <zamil.majdy@agpt.co>
pull/9061/head
Nicholas Tindle 2024-12-18 13:24:34 -06:00 committed by GitHub
parent 89a9354acb
commit 746f3d4e41
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 450 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
autogpt_platform/backend/backend/blocks/compass/triggers.py Normal file
View File

@ -0,0 +1,59 @@
from pydantic import BaseModel
from backend.data.block import (
Block,
BlockCategory,
BlockManualWebhookConfig,
BlockOutput,
BlockSchema,
)
from backend.data.model import SchemaField
from backend.integrations.webhooks.compass import CompassWebhookType
class Transcription(BaseModel):
text: str
speaker: str
end: float
start: float
duration: float
class TranscriptionDataModel(BaseModel):
date: str
transcription: str
transcriptions: list[Transcription]
class CompassAITriggerBlock(Block):
class Input(BlockSchema):
payload: TranscriptionDataModel = SchemaField(hidden=True)
class Output(BlockSchema):
transcription: str = SchemaField(
description="The contents of the compass transcription."
)
def __init__(self):
super().__init__(
id="9464a020-ed1d-49e1-990f-7f2ac924a2b7",
description="This block will output the contents of the compass transcription.",
categories={BlockCategory.HARDWARE},
input_schema=CompassAITriggerBlock.Input,
output_schema=CompassAITriggerBlock.Output,
webhook_config=BlockManualWebhookConfig(
provider="compass",
webhook_type=CompassWebhookType.TRANSCRIPTION,
),
test_input=[
{"input": "Hello, World!"},
{"input": "Hello, World!", "data": "Existing Data"},
],
# test_output=[
# ("output", "Hello, World!"), # No data provided, so trigger is returned
# ("output", "Existing Data"), # Data is provided, so data is returned.
# ],
)
def run(self, input_data: Input, **kwargs) -> BlockOutput:
yield "transcription", input_data.payload.transcription

88
autogpt_platform/backend/backend/data/block.py
View File

@ -42,6 +42,7 @@ class BlockType(Enum):
OUTPUT = "Output"
NOTE = "Note"
WEBHOOK = "Webhook"
WEBHOOK_MANUAL = "Webhook (manual)"
AGENT = "Agent"
@ -57,6 +58,7 @@ class BlockCategory(Enum):
COMMUNICATION = "Block that interacts with communication platforms."
DEVELOPER_TOOLS = "Developer tools such as GitHub blocks."
DATA = "Block that interacts with structured data."
HARDWARE = "Block that interacts with hardware."
AGENT = "Block that interacts with other agents."
CRM = "Block that interacts with CRM services."
@ -197,7 +199,12 @@ class EmptySchema(BlockSchema):
# --8<-- [start:BlockWebhookConfig]
class BlockWebhookConfig(BaseModel):
class BlockManualWebhookConfig(BaseModel):
"""
Configuration model for webhook-triggered blocks on which
the user has to manually set up the webhook at the provider.
"""
provider: str
"""The service provider that the webhook connects to"""
@ -208,6 +215,27 @@ class BlockWebhookConfig(BaseModel):
Only for use in the corresponding `WebhooksManager`.
"""
event_filter_input: str = ""
"""
Name of the block's event filter input.
Leave empty if the corresponding webhook doesn't have distinct event/payload types.
"""
event_format: str = "{event}"
"""
Template string for the event(s) that a block instance subscribes to.
Applied individually to each event selected in the event filter input.
Example: `"pull_request.{event}"` -> `"pull_request.opened"`
"""
class BlockWebhookConfig(BlockManualWebhookConfig):
"""
Configuration model for webhook-triggered blocks for which
the webhook can be automatically set up through the provider's API.
"""
resource_format: str
"""
Template string for the resource that a block instance subscribes to.
@ -217,17 +245,6 @@ class BlockWebhookConfig(BaseModel):
Only for use in the corresponding `WebhooksManager`.
"""
event_filter_input: str
"""Name of the block's event filter input."""
event_format: str = "{event}"
"""
Template string for the event(s) that a block instance subscribes to.
Applied individually to each event selected in the event filter input.
Example: `"pull_request.{event}"` -> `"pull_request.opened"`
"""
# --8<-- [end:BlockWebhookConfig]
@ -247,7 +264,7 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):
disabled: bool = False,
static_output: bool = False,
block_type: BlockType = BlockType.STANDARD,
webhook_config: Optional[BlockWebhookConfig] = None,
webhook_config: Optional[BlockWebhookConfig | BlockManualWebhookConfig] = None,
):
"""
Initialize the block with the given schema.
@ -278,27 +295,38 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):
self.contributors = contributors or set()
self.disabled = disabled
self.static_output = static_output
self.block_type = block_type if not webhook_config else BlockType.WEBHOOK
self.block_type = block_type
self.webhook_config = webhook_config
self.execution_stats = {}
if self.webhook_config:
# Enforce shape of webhook event filter
event_filter_field = self.input_schema.model_fields[
self.webhook_config.event_filter_input
]
if not (
isinstance(event_filter_field.annotation, type)
and issubclass(event_filter_field.annotation, BaseModel)
and all(
field.annotation is bool
for field in event_filter_field.annotation.model_fields.values()
)
):
raise NotImplementedError(
f"{self.name} has an invalid webhook event selector: "
"field must be a BaseModel and all its fields must be boolean"
)
if isinstance(self.webhook_config, BlockWebhookConfig):
# Enforce presence of credentials field on auto-setup webhook blocks
if CREDENTIALS_FIELD_NAME not in self.input_schema.model_fields:
raise TypeError(
"credentials field is required on auto-setup webhook blocks"
)
self.block_type = BlockType.WEBHOOK
else:
self.block_type = BlockType.WEBHOOK_MANUAL
# Enforce shape of webhook event filter, if present
if self.webhook_config.event_filter_input:
event_filter_field = self.input_schema.model_fields[
self.webhook_config.event_filter_input
]
if not (
isinstance(event_filter_field.annotation, type)
and issubclass(event_filter_field.annotation, BaseModel)
and all(
field.annotation is bool
for field in event_filter_field.annotation.model_fields.values()
)
):
raise NotImplementedError(
f"{self.name} has an invalid webhook event selector: "
"field must be a BaseModel and all its fields must be boolean"
)
# Enforce presence of 'payload' input
if "payload" not in self.input_schema.model_fields:

57
autogpt_platform/backend/backend/data/graph.py
View File

@ -84,6 +84,8 @@ class NodeModel(Node):
raise ValueError(f"Block #{self.block_id} not found for node #{self.id}")
if not block.webhook_config:
raise TypeError("This method can't be used on non-webhook blocks")
if not block.webhook_config.event_filter_input:
return True
event_filter = self.input_default.get(block.webhook_config.event_filter_input)
if not event_filter:
raise ValueError(f"Event filter is not configured on node #{self.id}")
@ -268,11 +270,19 @@ class GraphModel(Graph):
+ [sanitize(link.sink_name) for link in input_links.get(node.id, [])]
)
for name in block.input_schema.get_required_fields():
if name not in provided_inputs and (
for_run # Skip input completion validation, unless when executing.
or block.block_type == BlockType.INPUT
or block.block_type == BlockType.OUTPUT
or block.block_type == BlockType.AGENT
if (
name not in provided_inputs
and not (
name == "payload"
and block.block_type
in (BlockType.WEBHOOK, BlockType.WEBHOOK_MANUAL)
)
and (
for_run # Skip input completion validation, unless when executing.
or block.block_type == BlockType.INPUT
or block.block_type == BlockType.OUTPUT
or block.block_type == BlockType.AGENT
)
):
raise ValueError(
f"Node {block.name} #{node.id} required input missing: `{name}`"
@ -292,7 +302,6 @@ class GraphModel(Graph):
# Validate dependencies between fields
for field_name, field_info in input_schema.items():
# Apply input dependency validation only on run & field with depends_on
json_schema_extra = field_info.json_schema_extra or {}
dependencies = json_schema_extra.get("depends_on", [])
@ -359,7 +368,7 @@ class GraphModel(Graph):
link.is_static = True # Each value block output should be static.
@staticmethod
def from_db(graph: AgentGraph, hide_credentials: bool = False):
def from_db(graph: AgentGraph, for_export: bool = False):
return GraphModel(
id=graph.id,
user_id=graph.userId,
@ -369,7 +378,7 @@ class GraphModel(Graph):
name=graph.name or "",
description=graph.description or "",
nodes=[
GraphModel._process_node(node, hide_credentials)
NodeModel.from_db(GraphModel._process_node(node, for_export))
for node in graph.AgentNodes or []
],
links=list(
@ -382,23 +391,29 @@ class GraphModel(Graph):
)
@staticmethod
def _process_node(node: AgentNode, hide_credentials: bool) -> NodeModel:
node_dict = {field: getattr(node, field) for field in node.model_fields}
if hide_credentials and "constantInput" in node_dict:
constant_input = json.loads(
node_dict["constantInput"], target_type=dict[str, Any]
)
constant_input = GraphModel._hide_credentials_in_input(constant_input)
node_dict["constantInput"] = json.dumps(constant_input)
return NodeModel.from_db(AgentNode(**node_dict))
def _process_node(node: AgentNode, for_export: bool) -> AgentNode:
if for_export:
# Remove credentials from node input
if node.constantInput:
constant_input = json.loads(
node.constantInput, target_type=dict[str, Any]
)
constant_input = GraphModel._hide_node_input_credentials(constant_input)
node.constantInput = json.dumps(constant_input)
# Remove webhook info
node.webhookId = None
node.Webhook = None
return node
@staticmethod
def _hide_credentials_in_input(input_data: dict[str, Any]) -> dict[str, Any]:
def _hide_node_input_credentials(input_data: dict[str, Any]) -> dict[str, Any]:
sensitive_keys = ["credentials", "api_key", "password", "token", "secret"]
result = {}
for key, value in input_data.items():
if isinstance(value, dict):
result[key] = GraphModel._hide_credentials_in_input(value)
result[key] = GraphModel._hide_node_input_credentials(value)
elif isinstance(value, str) and any(
sensitive_key in key.lower() for sensitive_key in sensitive_keys
):
@ -495,7 +510,7 @@ async def get_graph(
version: int | None = None,
template: bool = False,
user_id: str | None = None,
hide_credentials: bool = False,
for_export: bool = False,
) -> GraphModel | None:
"""
Retrieves a graph from the DB.
@ -521,7 +536,7 @@ async def get_graph(
include=AGENT_GRAPH_INCLUDE,
order={"version": "desc"},
)
return GraphModel.from_db(graph, hide_credentials) if graph else None
return GraphModel.from_db(graph, for_export) if graph else None
async def set_graph_active_version(graph_id: str, version: int, user_id: str) -> None:

30
autogpt_platform/backend/backend/data/integrations.py
View File

@ -3,11 +3,12 @@ from typing import TYPE_CHECKING, AsyncGenerator, Optional
from prisma import Json
from prisma.models import IntegrationWebhook
from pydantic import Field
from pydantic import Field, computed_field
from backend.data.includes import INTEGRATION_WEBHOOK_INCLUDE
from backend.data.queue import AsyncRedisEventBus
from backend.integrations.providers import ProviderName
from backend.integrations.webhooks.utils import webhook_ingress_url
from .db import BaseDbModel
@ -31,6 +32,11 @@ class Webhook(BaseDbModel):
attached_nodes: Optional[list["NodeModel"]] = None
@computed_field
@property
def url(self) -> str:
return webhook_ingress_url(self.provider.value, self.id)
@staticmethod
def from_db(webhook: IntegrationWebhook):
from .graph import NodeModel
@ -84,8 +90,10 @@ async def get_webhook(webhook_id: str) -> Webhook:
return Webhook.from_db(webhook)
async def get_all_webhooks(credentials_id: str) -> list[Webhook]:
async def get_all_webhooks_by_creds(credentials_id: str) -> list[Webhook]:
"""⚠️ No `user_id` check: DO NOT USE without check in user-facing endpoints."""
if not credentials_id:
raise ValueError("credentials_id must not be empty")
webhooks = await IntegrationWebhook.prisma().find_many(
where={"credentialsId": credentials_id},
include=INTEGRATION_WEBHOOK_INCLUDE,
@ -93,7 +101,7 @@ async def get_all_webhooks(credentials_id: str) -> list[Webhook]:
return [Webhook.from_db(webhook) for webhook in webhooks]
async def find_webhook(
async def find_webhook_by_credentials_and_props(
credentials_id: str, webhook_type: str, resource: str, events: list[str]
) -> Webhook | None:
"""⚠️ No `user_id` check: DO NOT USE without check in user-facing endpoints."""
@ -109,6 +117,22 @@ async def find_webhook(
return Webhook.from_db(webhook) if webhook else None
async def find_webhook_by_graph_and_props(
graph_id: str, provider: str, webhook_type: str, events: list[str]
) -> Webhook | None:
"""⚠️ No `user_id` check: DO NOT USE without check in user-facing endpoints."""
webhook = await IntegrationWebhook.prisma().find_first(
where={
"provider": provider,
"webhookType": webhook_type,
"events": {"has_every": events},
"AgentNodes": {"some": {"agentGraphId": graph_id}},
},
include=INTEGRATION_WEBHOOK_INCLUDE,
)
return Webhook.from_db(webhook) if webhook else None
async def update_webhook_config(webhook_id: str, updated_config: dict) -> Webhook:
"""⚠️ No `user_id` check: DO NOT USE without check in user-facing endpoints."""
_updated_webhook = await IntegrationWebhook.prisma().update(

7
autogpt_platform/backend/backend/executor/manager.py
View File

@ -798,10 +798,13 @@ class ExecutionManager(AppService):
# Extract webhook payload, and assign it to the input pin
webhook_payload_key = f"webhook_{node.webhook_id}_payload"
if (
block.block_type == BlockType.WEBHOOK
block.block_type in (BlockType.WEBHOOK, BlockType.WEBHOOK_MANUAL)
and node.webhook_id
and webhook_payload_key in data
):
if webhook_payload_key not in data:
raise ValueError(
f"Node {block.name} #{node.id} webhook payload is missing"
)
input_data = {"payload": data[webhook_payload_key]}
input_data, error = validate_exec(node, input_data)

1
autogpt_platform/backend/backend/integrations/providers.py
View File

@ -4,6 +4,7 @@ from enum import Enum
# --8<-- [start:ProviderName]
class ProviderName(str, Enum):
ANTHROPIC = "anthropic"
COMPASS = "compass"
DISCORD = "discord"
D_ID = "d_id"
E2B = "e2b"

4
autogpt_platform/backend/backend/integrations/webhooks/__init__.py
View File

@ -1,16 +1,18 @@
from typing import TYPE_CHECKING
from .compass import CompassWebhookManager
from .github import GithubWebhooksManager
from .slant3d import Slant3DWebhooksManager
if TYPE_CHECKING:
from ..providers import ProviderName
from .base import BaseWebhooksManager
from ._base import BaseWebhooksManager
# --8<-- [start:WEBHOOK_MANAGERS_BY_NAME]
WEBHOOK_MANAGERS_BY_NAME: dict["ProviderName", type["BaseWebhooksManager"]] = {
handler.PROVIDER_NAME: handler
for handler in [
CompassWebhookManager,
GithubWebhooksManager,
Slant3DWebhooksManager,
]

61
autogpt_platform/backend/backend/integrations/webhooks/base.py → autogpt_platform/backend/backend/integrations/webhooks/_base.py
View File

@ -1,7 +1,7 @@
import logging
import secrets
from abc import ABC, abstractmethod
from typing import ClassVar, Generic, TypeVar
from typing import ClassVar, Generic, Optional, TypeVar
from uuid import uuid4
from fastapi import Request
@ -10,6 +10,7 @@ from strenum import StrEnum
from backend.data import integrations
from backend.data.model import Credentials
from backend.integrations.providers import ProviderName
from backend.integrations.webhooks.utils import webhook_ingress_url
from backend.util.exceptions import MissingConfigError
from backend.util.settings import Config
@ -26,7 +27,7 @@ class BaseWebhooksManager(ABC, Generic[WT]):
WebhookType: WT
async def get_suitable_webhook(
async def get_suitable_auto_webhook(
self,
user_id: str,
credentials: Credentials,
@ -39,16 +40,34 @@ class BaseWebhooksManager(ABC, Generic[WT]):
"PLATFORM_BASE_URL must be set to use Webhook functionality"
)
if webhook := await integrations.find_webhook(
if webhook := await integrations.find_webhook_by_credentials_and_props(
credentials.id, webhook_type, resource, events
):
return webhook
return await self._create_webhook(
user_id, credentials, webhook_type, resource, events
user_id, webhook_type, events, resource, credentials
)
async def get_manual_webhook(
self,
user_id: str,
graph_id: str,
webhook_type: WT,
events: list[str],
):
if current_webhook := await integrations.find_webhook_by_graph_and_props(
graph_id, self.PROVIDER_NAME, webhook_type, events
):
return current_webhook
return await self._create_webhook(
user_id,
webhook_type,
events,
register=False,
)
async def prune_webhook_if_dangling(
self, webhook_id: str, credentials: Credentials
self, webhook_id: str, credentials: Optional[Credentials]
) -> bool:
webhook = await integrations.get_webhook(webhook_id)
if webhook.attached_nodes is None:
@ -57,7 +76,8 @@ class BaseWebhooksManager(ABC, Generic[WT]):
# Don't prune webhook if in use
return False
await self._deregister_webhook(webhook, credentials)
if credentials:
await self._deregister_webhook(webhook, credentials)
await integrations.delete_webhook(webhook.id)
return True
@ -135,27 +155,36 @@ class BaseWebhooksManager(ABC, Generic[WT]):
async def _create_webhook(
self,
user_id: str,
credentials: Credentials,
webhook_type: WT,
resource: str,
events: list[str],
resource: str = "",
credentials: Optional[Credentials] = None,
register: bool = True,
) -> integrations.Webhook:
if not app_config.platform_base_url:
raise MissingConfigError(
"PLATFORM_BASE_URL must be set to use Webhook functionality"
)
id = str(uuid4())
secret = secrets.token_hex(32)
provider_name = self.PROVIDER_NAME
ingress_url = (
f"{app_config.platform_base_url}/api/integrations/{provider_name.value}"
f"/webhooks/{id}/ingress"
)
provider_webhook_id, config = await self._register_webhook(
credentials, webhook_type, resource, events, ingress_url, secret
)
ingress_url = webhook_ingress_url(provider_name=provider_name, webhook_id=id)
if register:
if not credentials:
raise TypeError("credentials are required if register = True")
provider_webhook_id, config = await self._register_webhook(
credentials, webhook_type, resource, events, ingress_url, secret
)
else:
provider_webhook_id, config = "", {}
return await integrations.create_webhook(
integrations.Webhook(
id=id,
user_id=user_id,
provider=provider_name,
credentials_id=credentials.id,
credentials_id=credentials.id if credentials else "",
webhook_type=webhook_type,
resource=resource,
events=events,

30
autogpt_platform/backend/backend/integrations/webhooks/_manual_base.py Normal file
View File

@ -0,0 +1,30 @@
import logging
from backend.data import integrations
from backend.data.model import APIKeyCredentials, Credentials, OAuth2Credentials
from ._base import WT, BaseWebhooksManager
logger = logging.getLogger(__name__)
class ManualWebhookManagerBase(BaseWebhooksManager[WT]):
async def _register_webhook(
self,
credentials: Credentials,
webhook_type: WT,
resource: str,
events: list[str],
ingress_url: str,
secret: str,
) -> tuple[str, dict]:
print(ingress_url) # FIXME: pass URL to user in front end
return "", {}
async def _deregister_webhook(
self,
webhook: integrations.Webhook,
credentials: OAuth2Credentials | APIKeyCredentials,
) -> None:
pass

30
autogpt_platform/backend/backend/integrations/webhooks/compass.py Normal file
View File

@ -0,0 +1,30 @@
import logging
from fastapi import Request
from strenum import StrEnum
from backend.data import integrations
from backend.integrations.providers import ProviderName
from ._manual_base import ManualWebhookManagerBase
logger = logging.getLogger(__name__)
class CompassWebhookType(StrEnum):
TRANSCRIPTION = "transcription"
TASK = "task"
class CompassWebhookManager(ManualWebhookManagerBase):
PROVIDER_NAME = ProviderName.COMPASS
WebhookType = CompassWebhookType
@classmethod
async def validate_payload(
cls, webhook: integrations.Webhook, request: Request
) -> tuple[dict, str]:
payload = await request.json()
event_type = CompassWebhookType.TRANSCRIPTION # currently the only type
return payload, event_type

2
autogpt_platform/backend/backend/integrations/webhooks/github.py
View File

@ -10,7 +10,7 @@ from backend.data import integrations
from backend.data.model import Credentials
from backend.integrations.providers import ProviderName
from .base import BaseWebhooksManager
from ._base import BaseWebhooksManager
logger = logging.getLogger(__name__)

103
autogpt_platform/backend/backend/integrations/webhooks/graph_lifecycle_hooks.py
View File

@ -1,7 +1,7 @@
import logging
from typing import TYPE_CHECKING, Callable, Optional, cast
from backend.data.block import get_block
from backend.data.block import BlockWebhookConfig, get_block
from backend.data.graph import set_node_webhook
from backend.data.model import CREDENTIALS_FIELD_NAME
from backend.integrations.webhooks import WEBHOOK_MANAGERS_BY_NAME
@ -10,7 +10,7 @@ if TYPE_CHECKING:
from backend.data.graph import GraphModel, NodeModel
from backend.data.model import Credentials
from .base import BaseWebhooksManager
from ._base import BaseWebhooksManager
logger = logging.getLogger(__name__)
@ -108,50 +108,79 @@ async def on_node_activate(
webhooks_manager = WEBHOOK_MANAGERS_BY_NAME[provider]()
try:
resource = block.webhook_config.resource_format.format(**node.input_default)
except KeyError:
resource = None
logger.debug(
f"Constructed resource string {resource} from input {node.input_default}"
)
if auto_setup_webhook := isinstance(block.webhook_config, BlockWebhookConfig):
try:
resource = block.webhook_config.resource_format.format(**node.input_default)
except KeyError:
resource = None
logger.debug(
f"Constructed resource string {resource} from input {node.input_default}"
)
else:
resource = "" # not relevant for manual webhooks
needs_credentials = CREDENTIALS_FIELD_NAME in block.input_schema.model_fields
credentials_meta = (
node.input_default.get(CREDENTIALS_FIELD_NAME) if needs_credentials else None
)
event_filter_input_name = block.webhook_config.event_filter_input
has_everything_for_webhook = (
resource is not None
and CREDENTIALS_FIELD_NAME in node.input_default
and event_filter_input_name in node.input_default
and any(is_on for is_on in node.input_default[event_filter_input_name].values())
and (credentials_meta or not needs_credentials)
and (
not event_filter_input_name
or (
event_filter_input_name in node.input_default
and any(
is_on
for is_on in node.input_default[event_filter_input_name].values()
)
)
)
)
if has_everything_for_webhook and resource:
if has_everything_for_webhook and resource is not None:
logger.debug(f"Node #{node} has everything for a webhook!")
if not credentials:
credentials_meta = node.input_default[CREDENTIALS_FIELD_NAME]
if credentials_meta and not credentials:
raise ValueError(
f"Cannot set up webhook for node #{node.id}: "
f"credentials #{credentials_meta['id']} not available"
)
# Shape of the event filter is enforced in Block.__init__
event_filter = cast(dict, node.input_default[event_filter_input_name])
events = [
block.webhook_config.event_format.format(event=event)
for event, enabled in event_filter.items()
if enabled is True
]
logger.debug(f"Webhook events to subscribe to: {', '.join(events)}")
if event_filter_input_name:
# Shape of the event filter is enforced in Block.__init__
event_filter = cast(dict, node.input_default[event_filter_input_name])
events = [
block.webhook_config.event_format.format(event=event)
for event, enabled in event_filter.items()
if enabled is True
]
logger.debug(f"Webhook events to subscribe to: {', '.join(events)}")
else:
events = []
# Find/make and attach a suitable webhook to the node
new_webhook = await webhooks_manager.get_suitable_webhook(
user_id,
credentials,
block.webhook_config.webhook_type,
resource,
events,
)
if auto_setup_webhook:
assert credentials is not None
new_webhook = await webhooks_manager.get_suitable_auto_webhook(
user_id,
credentials,
block.webhook_config.webhook_type,
resource,
events,
)
else:
# Manual webhook -> no credentials -> don't register but do create
new_webhook = await webhooks_manager.get_manual_webhook(
user_id,
node.graph_id,
block.webhook_config.webhook_type,
events,
)
logger.debug(f"Acquired webhook: {new_webhook}")
return await set_node_webhook(node.id, new_webhook.id)
else:
logger.debug(f"Node #{node.id} does not have everything for a webhook")
return node
@ -194,12 +223,16 @@ async def on_node_deactivate(
updated_node = await set_node_webhook(node.id, None)
# Prune and deregister the webhook if it is no longer used anywhere
logger.debug("Pruning and deregistering webhook if dangling")
webhook = node.webhook
if credentials:
logger.debug(f"Pruning webhook #{webhook.id} with credentials")
await webhooks_manager.prune_webhook_if_dangling(webhook.id, credentials)
else:
logger.debug(
f"Pruning{' and deregistering' if credentials else ''} "
f"webhook #{webhook.id}"
)
await webhooks_manager.prune_webhook_if_dangling(webhook.id, credentials)
if (
CREDENTIALS_FIELD_NAME in block.input_schema.model_fields
and not credentials
):
logger.warning(
f"Cannot deregister webhook #{webhook.id}: credentials "
f"#{webhook.credentials_id} not available "

2
autogpt_platform/backend/backend/integrations/webhooks/slant3d.py
View File

@ -6,7 +6,7 @@ from fastapi import Request
from backend.data import integrations
from backend.data.model import APIKeyCredentials, Credentials
from backend.integrations.providers import ProviderName
from backend.integrations.webhooks.base import BaseWebhooksManager
from backend.integrations.webhooks._base import BaseWebhooksManager
logger = logging.getLogger(__name__)

11
autogpt_platform/backend/backend/integrations/webhooks/utils.py Normal file
View File

@ -0,0 +1,11 @@
from backend.util.settings import Config
app_config = Config()
# TODO: add test to assert this matches the actual API route
def webhook_ingress_url(provider_name: str, webhook_id: str) -> str:
return (
f"{app_config.platform_base_url}/api/integrations/{provider_name}"
f"/webhooks/{webhook_id}/ingress"
)

4
autogpt_platform/backend/backend/server/integrations/router.py
View File

@ -7,7 +7,7 @@ from pydantic import BaseModel, Field, SecretStr
from backend.data.graph import set_node_webhook
from backend.data.integrations import (
WebhookEvent,
get_all_webhooks,
get_all_webhooks_by_creds,
get_webhook,
publish_webhook_event,
wait_for_webhook_event,
@ -363,7 +363,7 @@ async def remove_all_webhooks_for_credentials(
Raises:
NeedConfirmation: If any of the webhooks are still in use and `force` is `False`
"""
webhooks = await get_all_webhooks(credentials.id)
webhooks = await get_all_webhooks_by_creds(credentials.id)
if credentials.provider not in WEBHOOK_MANAGERS_BY_NAME:
if webhooks:
logger.error(

22
autogpt_platform/backend/backend/server/routers/v1.py
View File

@ -149,7 +149,7 @@ class DeleteGraphResponse(TypedDict):
@v1_router.get(path="/graphs", tags=["graphs"], dependencies=[Depends(auth_middleware)])
async def get_graphs(
user_id: Annotated[str, Depends(get_user_id)]
) -> Sequence[graph_db.Graph]:
) -> Sequence[graph_db.GraphModel]:
return await graph_db.get_graphs(filter_by="active", user_id=user_id)
@ -166,9 +166,9 @@ async def get_graph(
user_id: Annotated[str, Depends(get_user_id)],
version: int | None = None,
hide_credentials: bool = False,
) -> graph_db.Graph:
) -> graph_db.GraphModel:
graph = await graph_db.get_graph(
graph_id, version, user_id=user_id, hide_credentials=hide_credentials
graph_id, version, user_id=user_id, for_export=hide_credentials
)
if not graph:
raise HTTPException(status_code=404, detail=f"Graph #{graph_id} not found.")
@ -187,7 +187,7 @@ async def get_graph(
)
async def get_graph_all_versions(
graph_id: str, user_id: Annotated[str, Depends(get_user_id)]
) -> Sequence[graph_db.Graph]:
) -> Sequence[graph_db.GraphModel]:
graphs = await graph_db.get_graph_all_versions(graph_id, user_id=user_id)
if not graphs:
raise HTTPException(status_code=404, detail=f"Graph #{graph_id} not found.")
@ -199,7 +199,7 @@ async def get_graph_all_versions(
)
async def create_new_graph(
create_graph: CreateGraph, user_id: Annotated[str, Depends(get_user_id)]
) -> graph_db.Graph:
) -> graph_db.GraphModel:
return await do_create_graph(create_graph, is_template=False, user_id=user_id)
@ -209,7 +209,7 @@ async def do_create_graph(
# user_id doesn't have to be annotated like on other endpoints,
# because create_graph isn't used directly as an endpoint
user_id: str,
) -> graph_db.Graph:
) -> graph_db.GraphModel:
if create_graph.graph:
graph = graph_db.make_graph_model(create_graph.graph, user_id)
elif create_graph.template_id:
@ -270,7 +270,7 @@ async def update_graph(
graph_id: str,
graph: graph_db.Graph,
user_id: Annotated[str, Depends(get_user_id)],
) -> graph_db.Graph:
) -> graph_db.GraphModel:
# Sanity check
if graph.id and graph.id != graph_id:
raise HTTPException(400, detail="Graph ID does not match ID in URI")
@ -440,7 +440,7 @@ async def get_graph_run_node_execution_results(
)
async def get_templates(
user_id: Annotated[str, Depends(get_user_id)]
) -> Sequence[graph_db.Graph]:
) -> Sequence[graph_db.GraphModel]:
return await graph_db.get_graphs(filter_by="template", user_id=user_id)
@ -449,7 +449,9 @@ async def get_templates(
tags=["templates", "graphs"],
dependencies=[Depends(auth_middleware)],
)
async def get_template(graph_id: str, version: int | None = None) -> graph_db.Graph:
async def get_template(
graph_id: str, version: int | None = None
) -> graph_db.GraphModel:
graph = await graph_db.get_graph(graph_id, version, template=True)
if not graph:
raise HTTPException(status_code=404, detail=f"Template #{graph_id} not found.")
@ -463,7 +465,7 @@ async def get_template(graph_id: str, version: int | None = None) -> graph_db.Gr
)
async def create_new_template(
create_graph: CreateGraph, user_id: Annotated[str, Depends(get_user_id)]
) -> graph_db.Graph:
) -> graph_db.GraphModel:
return await do_create_graph(create_graph, is_template=True, user_id=user_id)

51
autogpt_platform/frontend/src/components/CustomNode.tsx
View File

@ -6,7 +6,7 @@ import React, {
useContext,
useMemo,
} from "react";
import { NodeProps, useReactFlow, Node, Edge } from "@xyflow/react";
import { NodeProps, useReactFlow, Node as XYNode, Edge } from "@xyflow/react";
import "@xyflow/react/dist/style.css";
import "./customnode.css";
import InputModalComponent from "./InputModalComponent";
@ -16,6 +16,7 @@ import {
BlockIOSubSchema,
BlockIOStringSubSchema,
Category,
Node,
NodeExecutionResult,
BlockUIType,
BlockCost,
@ -71,7 +72,7 @@ export type CustomNodeData = {
outputSchema: BlockIORootSchema;
hardcodedValues: { [key: string]: any };
connections: ConnectionData;
webhookId?: string;
webhook?: Node["webhook"];
isOutputOpen: boolean;
status?: NodeExecutionResult["status"];
/** executionResults contains outputs across multiple executions
@ -87,7 +88,7 @@ export type CustomNodeData = {
uiType: BlockUIType;
};
export type CustomNode = Node<CustomNodeData, "custom">;
export type CustomNode = XYNode<CustomNodeData, "custom">;
export function CustomNode({
data,
@ -237,7 +238,11 @@ export function CustomNode({
const isHidden = propSchema.hidden;
const isConnectable =
// No input connection handles on INPUT and WEBHOOK blocks
![BlockUIType.INPUT, BlockUIType.WEBHOOK].includes(nodeType) &&
![
BlockUIType.INPUT,
BlockUIType.WEBHOOK,
BlockUIType.WEBHOOK_MANUAL,
].includes(nodeType) &&
// No input connection handles for credentials
propKey !== "credentials" &&
// For OUTPUT blocks, only show the 'value' (hides 'name') input connection handle
@ -549,22 +554,25 @@ export function CustomNode({
>(null);
useEffect(() => {
if (data.uiType != BlockUIType.WEBHOOK) return;
if (!data.webhookId) {
if (
![BlockUIType.WEBHOOK, BlockUIType.WEBHOOK_MANUAL].includes(data.uiType)
)
return;
if (!data.webhook) {
setWebhookStatus("none");
return;
}
setWebhookStatus("pending");
api
.pingWebhook(data.webhookId)
.pingWebhook(data.webhook.id)
.then((pinged) => setWebhookStatus(pinged ? "works" : "exists"))
.catch((error: Error) =>
error.message.includes("ping timed out")
? setWebhookStatus("broken")
: setWebhookStatus("none"),
);
}, [data.uiType, data.webhookId, api, setWebhookStatus]);
}, [data.uiType, data.webhook, api, setWebhookStatus]);
const webhookStatusDot = useMemo(
() =>
@ -726,6 +734,33 @@ export function CustomNode({
data-id="input-handles"
>
<div>
{data.uiType === BlockUIType.WEBHOOK_MANUAL &&
(data.webhook ? (
<div className="nodrag mr-5 flex flex-col gap-1">
Webhook URL:
<div className="flex gap-2 rounded-md bg-gray-50 p-2">
<code className="select-all text-sm">
{data.webhook.url}
</code>
<Button
variant="outline"
size="icon"
className="size-7 flex-none"
onClick={() =>
data.webhook &&
navigator.clipboard.writeText(data.webhook.url)
}
title="Copy webhook URL"
>
<CopyIcon className="size-4" />
</Button>
</div>
</div>
) : (
<p className="italic text-gray-500">
(A Webhook URL will be generated when you save the agent)
</p>
))}
{data.inputSchema &&
generateInputHandles(data.inputSchema, data.uiType)}
</div>

4
autogpt_platform/frontend/src/hooks/useAgentGraph.ts
View File

@ -169,7 +169,7 @@ export default function useAgentGraph(
inputSchema: block.inputSchema,
outputSchema: block.outputSchema,
hardcodedValues: node.input_default,
webhookId: node.webhook_id,
webhook: node.webhook,
uiType: block.uiType,
connections: graph.links
.filter((l) => [l.source_id, l.sink_id].includes(node.id))
@ -815,7 +815,7 @@ export default function useAgentGraph(
),
status: undefined,
backend_id: backendNode.id,
webhookId: backendNode.webhook_id,
webhook: backendNode.webhook,
executionResults: [],
},
}

17
autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts
View File

@ -172,7 +172,7 @@ export type Node = {
position: { x: number; y: number };
[key: string]: any;
};
webhook_id?: string;
webhook?: Webhook;
};
/* Mirror of backend/data/graph.py:Link */
@ -314,6 +314,20 @@ export type APIKeyCredentials = BaseCredentials & {
expires_at?: number;
};
/* Mirror of backend/data/integrations.py:Webhook */
type Webhook = {
id: string;
url: string;
provider: CredentialsProviderName;
credentials_id: string;
webhook_type: string;
resource?: string;
events: string[];
secret: string;
config: Record<string, any>;
provider_webhook_id?: string;
};
export type User = {
id: string;
email: string;
@ -325,6 +339,7 @@ export enum BlockUIType {
OUTPUT = "Output",
NOTE = "Note",
WEBHOOK = "Webhook",
WEBHOOK_MANUAL = "Webhook (manual)",
AGENT = "Agent",
}