From 995c3ef81bfe8bd1a86d129f11e48fde2acb084e Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Mon, 24 Mar 2025 19:33:05 -0300 Subject: [PATCH] feat(snapshots): avoid parsing raw snapshots when possible BE-11724 (#560) --- api/dataservices/interface.go | 1 + api/dataservices/snapshot/snapshot.go | 13 +++++++++++ api/dataservices/snapshot/tx.go | 23 +++++++++++++++++++ .../handler/endpoints/endpoint_inspect.go | 19 +++++++-------- api/http/handler/endpoints/endpoint_list.go | 4 +++- api/http/handler/endpoints/endpoint_update.go | 2 +- api/http/handler/system/nodes_count.go | 2 +- api/http/proxy/factory/docker/volumes.go | 2 +- api/internal/snapshot/snapshot.go | 16 +++++++++---- api/portainer.go | 2 +- app/react/hooks/useCurrentEnvironment.ts | 2 +- .../EnvironmentItem/EngineVersion.tsx | 4 ++-- .../EnvironmentList/EnvironmentList.tsx | 1 + .../environments/environment.service/index.ts | 11 +++++++-- .../environments/queries/useEnvironment.ts | 13 +++++++++-- .../common/useEnvironments.ts | 2 +- 16 files changed, 89 insertions(+), 28 deletions(-) diff --git a/api/dataservices/interface.go b/api/dataservices/interface.go index 2bc2df7f3..8ba55531c 100644 --- a/api/dataservices/interface.go +++ b/api/dataservices/interface.go @@ -159,6 +159,7 @@ type ( SnapshotService interface { BaseCRUD[portainer.Snapshot, portainer.EndpointID] + ReadWithoutSnapshotRaw(ID portainer.EndpointID) (*portainer.Snapshot, error) } // SSLSettingsService represents a service for managing application settings diff --git a/api/dataservices/snapshot/snapshot.go b/api/dataservices/snapshot/snapshot.go index 1f9cd5f9f..155077677 100644 --- a/api/dataservices/snapshot/snapshot.go +++ b/api/dataservices/snapshot/snapshot.go @@ -38,3 +38,16 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx { func (service *Service) Create(snapshot *portainer.Snapshot) error { return service.Connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot) } + +func (service *Service) ReadWithoutSnapshotRaw(ID portainer.EndpointID) (*portainer.Snapshot, error) { + var snapshot *portainer.Snapshot + + err := service.Connection.ViewTx(func(tx portainer.Transaction) error { + var err error + snapshot, err = service.Tx(tx).ReadWithoutSnapshotRaw(ID) + + return err + }) + + return snapshot, err +} diff --git a/api/dataservices/snapshot/tx.go b/api/dataservices/snapshot/tx.go index c93a747d3..8a8dcc1c2 100644 --- a/api/dataservices/snapshot/tx.go +++ b/api/dataservices/snapshot/tx.go @@ -12,3 +12,26 @@ type ServiceTx struct { func (service ServiceTx) Create(snapshot *portainer.Snapshot) error { return service.Tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot) } + +func (service ServiceTx) ReadWithoutSnapshotRaw(ID portainer.EndpointID) (*portainer.Snapshot, error) { + var snapshot struct { + Docker *struct { + X struct{} `json:"DockerSnapshotRaw"` + *portainer.DockerSnapshot + } `json:"Docker"` + + portainer.Snapshot + } + + identifier := service.Connection.ConvertToKey(int(ID)) + + if err := service.Tx.GetObject(service.Bucket, identifier, &snapshot); err != nil { + return nil, err + } + + if snapshot.Docker != nil { + snapshot.Snapshot.Docker = snapshot.Docker.DockerSnapshot + } + + return &snapshot.Snapshot, nil +} diff --git a/api/http/handler/endpoints/endpoint_inspect.go b/api/http/handler/endpoints/endpoint_inspect.go index e25e79720..132ab6350 100644 --- a/api/http/handler/endpoints/endpoint_inspect.go +++ b/api/http/handler/endpoints/endpoint_inspect.go @@ -19,6 +19,8 @@ import ( // @security jwt // @produce json // @param id path int true "Environment(Endpoint) identifier" +// @param excludeSnapshot query bool false "if true, the snapshot data won't be retrieved" +// @param excludeSnapshotRaw query bool false "if true, the SnapshotRaw field won't be retrieved" // @success 200 {object} portainer.Endpoint "Success" // @failure 400 "Invalid request" // @failure 404 "Environment(Endpoint) not found" @@ -37,8 +39,7 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request) return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err) } - err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) - if err != nil { + if err := handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint); err != nil { return httperror.Forbidden("Permission denied to access environment", err) } @@ -51,9 +52,11 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request) endpointutils.UpdateEdgeEndpointHeartbeat(endpoint, settings) endpoint.ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion() - if !excludeSnapshot(r) { - err = handler.SnapshotService.FillSnapshotData(endpoint) - if err != nil { + excludeSnapshot, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshot", true) + excludeRaw, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshotRaw", true) + + if !excludeSnapshot { + if err := handler.SnapshotService.FillSnapshotData(endpoint, !excludeRaw); err != nil { return httperror.InternalServerError("Unable to add snapshot data", err) } } @@ -83,9 +86,3 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request) return response.JSON(w, endpoint) } - -func excludeSnapshot(r *http.Request) bool { - excludeSnapshot, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshot", true) - - return excludeSnapshot -} diff --git a/api/http/handler/endpoints/endpoint_list.go b/api/http/handler/endpoints/endpoint_list.go index 7ea557910..6be1c7ff2 100644 --- a/api/http/handler/endpoints/endpoint_list.go +++ b/api/http/handler/endpoints/endpoint_list.go @@ -44,6 +44,7 @@ const ( // @param edgeDeviceUntrusted query bool false "if true, show only untrusted edge agents, if false show only trusted edge agents (relevant only for edge agents)" // @param edgeCheckInPassedSeconds query number false "if bigger then zero, show only edge agents that checked-in in the last provided seconds (relevant only for edge agents)" // @param excludeSnapshots query bool false "if true, the snapshot data won't be retrieved" +// @param excludeSnapshotRaw query bool false "if true, the SnapshotRaw field won't be retrieved" // @param name query string false "will return only environments(endpoints) with this name" // @param edgeStackId query portainer.EdgeStackID false "will return the environements of the specified edge stack" // @param edgeStackStatus query string false "only applied when edgeStackId exists. Filter the returned environments based on their deployment status in the stack (not the environment status!)" Enum("Pending", "Ok", "Error", "Acknowledged", "Remove", "RemoteUpdateSuccess", "ImagesPulled") @@ -59,6 +60,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht limit, _ := request.RetrieveNumericQueryParameter(r, "limit", true) sortField, _ := request.RetrieveQueryParameter(r, "sort", true) sortOrder, _ := request.RetrieveQueryParameter(r, "order", true) + excludeRaw, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshotRaw", true) endpointGroups, err := handler.DataStore.EndpointGroup().ReadAll() if err != nil { @@ -114,7 +116,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht endpointutils.UpdateEdgeEndpointHeartbeat(&paginatedEndpoints[idx], settings) if !query.excludeSnapshots { - if err := handler.SnapshotService.FillSnapshotData(&paginatedEndpoints[idx]); err != nil { + if err := handler.SnapshotService.FillSnapshotData(&paginatedEndpoints[idx], !excludeRaw); err != nil { return httperror.InternalServerError("Unable to add snapshot data", err) } } diff --git a/api/http/handler/endpoints/endpoint_update.go b/api/http/handler/endpoints/endpoint_update.go index 0f57136df..71c9ef702 100644 --- a/api/http/handler/endpoints/endpoint_update.go +++ b/api/http/handler/endpoints/endpoint_update.go @@ -272,7 +272,7 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) * } } - if err := handler.SnapshotService.FillSnapshotData(endpoint); err != nil { + if err := handler.SnapshotService.FillSnapshotData(endpoint, true); err != nil { return httperror.InternalServerError("Unable to add snapshot data", err) } diff --git a/api/http/handler/system/nodes_count.go b/api/http/handler/system/nodes_count.go index 94ab320fa..7d150d911 100644 --- a/api/http/handler/system/nodes_count.go +++ b/api/http/handler/system/nodes_count.go @@ -33,7 +33,7 @@ func (handler *Handler) systemNodesCount(w http.ResponseWriter, r *http.Request) var nodes int for _, endpoint := range endpoints { - if err := snapshot.FillSnapshotData(handler.dataStore, &endpoint); err != nil { + if err := snapshot.FillSnapshotData(handler.dataStore, &endpoint, false); err != nil { return httperror.InternalServerError("Unable to add snapshot data", err) } diff --git a/api/http/proxy/factory/docker/volumes.go b/api/http/proxy/factory/docker/volumes.go index aae0a4602..858a940d8 100644 --- a/api/http/proxy/factory/docker/volumes.go +++ b/api/http/proxy/factory/docker/volumes.go @@ -224,7 +224,7 @@ func (transport *Transport) getDockerID() (string, error) { if transport.snapshotService != nil { endpoint := portainer.Endpoint{ID: transport.endpoint.ID} - if err := transport.snapshotService.FillSnapshotData(&endpoint); err == nil && len(endpoint.Snapshots) > 0 { + if err := transport.snapshotService.FillSnapshotData(&endpoint, true); err == nil && len(endpoint.Snapshots) > 0 { if dockerID, err := snapshot.FetchDockerID(endpoint.Snapshots[0]); err == nil { transport.dockerID = dockerID return dockerID, nil diff --git a/api/internal/snapshot/snapshot.go b/api/internal/snapshot/snapshot.go index 019dec359..7bc109459 100644 --- a/api/internal/snapshot/snapshot.go +++ b/api/internal/snapshot/snapshot.go @@ -170,8 +170,8 @@ func (service *Service) Create(snapshot portainer.Snapshot) error { return service.dataStore.Snapshot().Create(&snapshot) } -func (service *Service) FillSnapshotData(endpoint *portainer.Endpoint) error { - return FillSnapshotData(service.dataStore, endpoint) +func (service *Service) FillSnapshotData(endpoint *portainer.Endpoint, includeRaw bool) error { + return FillSnapshotData(service.dataStore, endpoint, includeRaw) } func (service *Service) snapshotKubernetesEndpoint(endpoint *portainer.Endpoint) error { @@ -328,8 +328,16 @@ func FetchDockerID(snapshot portainer.DockerSnapshot) (string, error) { return info.Swarm.Cluster.ID, nil } -func FillSnapshotData(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint) error { - snapshot, err := tx.Snapshot().Read(endpoint.ID) +func FillSnapshotData(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint, includeRaw bool) error { + var snapshot *portainer.Snapshot + var err error + + if includeRaw { + snapshot, err = tx.Snapshot().Read(endpoint.ID) + } else { + snapshot, err = tx.Snapshot().ReadWithoutSnapshotRaw(endpoint.ID) + } + if tx.IsErrObjectNotFound(err) { endpoint.Snapshots = []portainer.DockerSnapshot{} endpoint.Kubernetes.Snapshots = []portainer.KubernetesSnapshot{} diff --git a/api/portainer.go b/api/portainer.go index b90e89b4e..1f57b0ada 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1622,7 +1622,7 @@ type ( Start() SetSnapshotInterval(snapshotInterval string) error SnapshotEndpoint(endpoint *Endpoint) error - FillSnapshotData(endpoint *Endpoint) error + FillSnapshotData(endpoint *Endpoint, includeRaw bool) error } // SwarmStackManager represents a service to manage Swarm stacks diff --git a/app/react/hooks/useCurrentEnvironment.ts b/app/react/hooks/useCurrentEnvironment.ts index 7beaf6fb6..c65ca077b 100644 --- a/app/react/hooks/useCurrentEnvironment.ts +++ b/app/react/hooks/useCurrentEnvironment.ts @@ -4,5 +4,5 @@ import { useEnvironmentId } from './useEnvironmentId'; export function useCurrentEnvironment(force = true) { const id = useEnvironmentId(force); - return useEnvironment(id); + return useEnvironment(id, undefined, { excludeSnapshot: false }); } diff --git a/app/react/portainer/HomeView/EnvironmentList/EnvironmentItem/EngineVersion.tsx b/app/react/portainer/HomeView/EnvironmentList/EnvironmentItem/EngineVersion.tsx index 83e337686..30338d656 100644 --- a/app/react/portainer/HomeView/EnvironmentList/EnvironmentItem/EngineVersion.tsx +++ b/app/react/portainer/HomeView/EnvironmentList/EnvironmentItem/EngineVersion.tsx @@ -1,16 +1,16 @@ import { DockerSnapshot } from '@/react/docker/snapshots/types'; -import { useIsPodman } from '@/react/portainer/environments/queries/useIsPodman'; import { Environment, PlatformType, KubernetesSnapshot, + ContainerEngine, } from '@/react/portainer/environments/types'; import { getPlatformType } from '@/react/portainer/environments/utils'; import { getDockerEnvironmentType } from '@/react/portainer/environments/utils/getDockerEnvironmentType'; export function EngineVersion({ environment }: { environment: Environment }) { const platform = getPlatformType(environment.Type); - const isPodman = useIsPodman(environment.Id); + const isPodman = environment.ContainerEngine === ContainerEngine.Podman; switch (platform) { case PlatformType.Docker: diff --git a/app/react/portainer/HomeView/EnvironmentList/EnvironmentList.tsx b/app/react/portainer/HomeView/EnvironmentList/EnvironmentList.tsx index 9884048b3..160e68684 100644 --- a/app/react/portainer/HomeView/EnvironmentList/EnvironmentList.tsx +++ b/app/react/portainer/HomeView/EnvironmentList/EnvironmentList.tsx @@ -110,6 +110,7 @@ export function EnvironmentList({ onClickBrowse, onRefresh }: Props) { updateInformation: isBE, edgeAsync: getEdgeAsyncValue(connectionTypes), platformTypes, + excludeSnapshotRaw: true, }; const queryWithSort = { diff --git a/app/react/portainer/environments/environment.service/index.ts b/app/react/portainer/environments/environment.service/index.ts index f96d57d69..c6d770e74 100644 --- a/app/react/portainer/environments/environment.service/index.ts +++ b/app/react/portainer/environments/environment.service/index.ts @@ -42,6 +42,7 @@ export interface BaseEnvironmentsQueryParams { edgeAsync?: boolean; edgeDeviceUntrusted?: boolean; excludeSnapshots?: boolean; + excludeSnapshotRaw?: boolean; provisioned?: boolean; name?: string; agentVersions?: string[]; @@ -119,9 +120,15 @@ export async function getAgentVersions() { } } -export async function getEndpoint(id: EnvironmentId) { +export async function getEndpoint( + id: EnvironmentId, + excludeSnapshot = true, + excludeSnapshotRaw = true +) { try { - const { data: endpoint } = await axios.get(buildUrl(id)); + const { data: endpoint } = await axios.get(buildUrl(id), { + params: { excludeSnapshot, excludeSnapshotRaw }, + }); return endpoint; } catch (e) { throw parseAxiosError(e as Error); diff --git a/app/react/portainer/environments/queries/useEnvironment.ts b/app/react/portainer/environments/queries/useEnvironment.ts index 7a0b4961c..b2b8fd6bd 100644 --- a/app/react/portainer/environments/queries/useEnvironment.ts +++ b/app/react/portainer/environments/queries/useEnvironment.ts @@ -10,11 +10,20 @@ import { environmentQueryKeys } from './query-keys'; export function useEnvironment( environmentId?: EnvironmentId, select?: (environment: Environment) => T, - options?: { autoRefreshRate?: number } + options?: { + autoRefreshRate?: number; + excludeSnapshot?: boolean; + excludeSnapshotRaw?: boolean; + } ) { return useQuery( environmentQueryKeys.item(environmentId!), - () => getEndpoint(environmentId!), + () => + getEndpoint( + environmentId!, + options?.excludeSnapshot ?? undefined, + options?.excludeSnapshotRaw ?? undefined + ), { select, ...withError('Failed loading environment'), diff --git a/app/react/portainer/environments/update-schedules/common/useEnvironments.ts b/app/react/portainer/environments/update-schedules/common/useEnvironments.ts index bfe9a83b4..7843cf27b 100644 --- a/app/react/portainer/environments/update-schedules/common/useEnvironments.ts +++ b/app/react/portainer/environments/update-schedules/common/useEnvironments.ts @@ -3,7 +3,7 @@ import { EdgeGroupId, EdgeTypes } from '@/react/portainer/environments/types'; export function useEnvironments(edgeGroupIds: Array) { const environmentsQuery = useEnvironmentList( - { edgeGroupIds, types: EdgeTypes, pageLimit: 0 }, + { edgeGroupIds, types: EdgeTypes, pageLimit: 0, excludeSnapshots: true }, { enabled: edgeGroupIds.length > 0, }