From 16c115d5cb264dbb38ca3688dc6732cd862a9044 Mon Sep 17 00:00:00 2001
From: Dom Dwyer <dom@itsallbroken.com>
Date: Thu, 3 Aug 2023 17:09:12 +0200
Subject: [PATCH] docs(router): gossip subsystem types / topology

Describes the router's schema gossiping types and how they fit together.
---
 router/src/gossip/mod.rs             | 30 ++++++++++++++++-------
 router/src/gossip/namespace_cache.rs | 36 +++++++++++++++++++++++++++-
 2 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/router/src/gossip/mod.rs b/router/src/gossip/mod.rs
index 3ea36f5127..157cf87682 100644
--- a/router/src/gossip/mod.rs
+++ b/router/src/gossip/mod.rs
@@ -5,17 +5,29 @@
 //! * [`gossip`] crate: provides the gossip transport, the [`GossipHandle`], and
 //!   the [`Dispatcher`]. This crate operates on raw bytes.
 //!
-//! * The [`SchemaChangeObserver`]: a router-specific wrapper over the underlying
-//!   [`GossipHandle`]. This type translates the application calls into protobuf
-//!   [`Msg`], and serialises them into bytes for the underlying [`gossip`]
-//!   impl.
+//! * The outgoing [`SchemaChangeObserver`]: a router-specific wrapper over the
+//!   underlying [`GossipHandle`]. This type translates the application calls
+//!   into protobuf [`Msg`], and serialises them into bytes, sending them over
+//!   the underlying [`gossip`] impl.
 //!
-//! * The [`GossipMessageDispatcher`]: deserialises the incoming bytes from the
-//!   gossip [`Dispatcher`] into [`Msg`] and passes them off to the
-//!   [`GossipMessageHandler`] implementation for processing.
+//! * The incoming [`GossipMessageDispatcher`]: deserialises the incoming bytes
+//!   from the gossip [`Dispatcher`] into [`Msg`] and passes them off to the
+//!   [`NamespaceSchemaGossip`] implementation for processing.
+//!
+//! * The incoming [`NamespaceSchemaGossip`]: processes [`Msg`] received from
+//!   peers, applying them to the local cache state if necessary.
 //!
 //! ```text
-//!                   event                      handler
+//!         ┌────────────────────────────────────────────────────┐
+//!         │                   NamespaceCache                   │
+//!         └────────────────────────────────────────────────────┘
+//!                     │                           ▲
+//!                     │                           │
+//!                   diff                        diff
+//!                     │                           │
+//!                     │              ┌─────────────────────────┐
+//!                     │              │  NamespaceSchemaGossip  │
+//!                     │              └─────────────────────────┘
 //!                     │                           ▲
 //!                     │                           │
 //!                     │     Application types     │
@@ -43,7 +55,7 @@
 //! [`SchemaChangeObserver`]: schema_change_observer::SchemaChangeObserver
 //! [`Msg`]: generated_types::influxdata::iox::gossip::v1::gossip_message::Msg
 //! [`GossipMessageDispatcher`]: dispatcher::GossipMessageDispatcher
-//! [`GossipMessageHandler`]: dispatcher::GossipMessageHandler
+//! [`NamespaceSchemaGossip`]: namespace_cache::NamespaceSchemaGossip
 
 pub mod dispatcher;
 pub mod namespace_cache;
diff --git a/router/src/gossip/namespace_cache.rs b/router/src/gossip/namespace_cache.rs
index 5b7105a056..11e7b7170c 100644
--- a/router/src/gossip/namespace_cache.rs
+++ b/router/src/gossip/namespace_cache.rs
@@ -45,7 +45,41 @@ enum Error {
     TableNotFound(String),
 }
 
-/// A [`NamespaceCache`] implementation for gossip.
+/// A [`NamespaceCache`] decorator applying incoming schema change notifications
+/// via the [`gossip`] subsystem.
+///
+/// Any schema additions received from peers are applied to the decorated
+/// [`NamespaceCache`], helping to keep the peers approximately in-sync on a
+/// best-effort basis.
+///
+/// # Applying Peer Changes
+///
+/// Other peers participating in schema gossiping send changes made to their
+/// local state - this allows this local node (and all the other peers) to
+/// populate their cache before a write request is received by the local node
+/// that would cause a cache miss resulting in a catalog query, and the
+/// associated latency penalty and catalog load that comes with it.
+///
+/// This type implements the [`GossipMessageHandler`] which is invoked with the
+/// [`Msg`] received from an opaque peer by the [`gossip`] subsystem (off of the
+/// hot path), which when processed causes the cache contents to be updated if
+/// appropriate through the usual [`NamespaceCache::get_schema()`] and
+/// [`NamespaceCache::put_schema()`] abstraction.
+///
+/// # Peer Trust, Immutability, and Panic
+///
+/// Certain values are immutable for the lifetime of the associated entity; for
+/// example, the data type of a column must never change.
+///
+/// If a peer gossips an event that contradicts the local state w.r.t an
+/// immutable value, the handler will panic. This is designed to bring down the
+/// local node and cause it to rebuild the cache state from the source of truth
+/// (the catalog) at startup to converge any differences.
+///
+/// This requires trusted peers within the network this node is operating
+/// within. A malicious peer can trivially panic peers by gossiping malicious
+/// schema updates. This lays outside the threat model of the gossip system
+/// which explicitly trusts all gossip peers by design.
 #[derive(Debug)]
 pub struct NamespaceSchemaGossip<C> {
     inner: C,