feat(snapshots): avoid parsing raw snapshots when possible BE-11724 (#560)

api/dataservices/interface.go

@@ -159,6 +159,7 @@ type (
 
 	SnapshotService interface {
 		BaseCRUD[portainer.Snapshot, portainer.EndpointID]
+		ReadWithoutSnapshotRaw(ID portainer.EndpointID) (*portainer.Snapshot, error)
 	}
 
 	// SSLSettingsService represents a service for managing application settings

api/dataservices/snapshot/snapshot.go

@@ -38,3 +38,16 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) Create(snapshot *portainer.Snapshot) error {
 	return service.Connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }
+
+func (service *Service) ReadWithoutSnapshotRaw(ID portainer.EndpointID) (*portainer.Snapshot, error) {
+	var snapshot *portainer.Snapshot
+
+	err := service.Connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		snapshot, err = service.Tx(tx).ReadWithoutSnapshotRaw(ID)
+
+		return err
+	})
+
+	return snapshot, err
+}

api/dataservices/snapshot/tx.go

@@ -12,3 +12,26 @@ type ServiceTx struct {
 func (service ServiceTx) Create(snapshot *portainer.Snapshot) error {
 	return service.Tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }
+
+func (service ServiceTx) ReadWithoutSnapshotRaw(ID portainer.EndpointID) (*portainer.Snapshot, error) {
+	var snapshot struct {
+		Docker *struct {
+			X struct{} `json:"DockerSnapshotRaw"`
+			*portainer.DockerSnapshot
+		} `json:"Docker"`
+
+		portainer.Snapshot
+	}
+
+	identifier := service.Connection.ConvertToKey(int(ID))
+
+	if err := service.Tx.GetObject(service.Bucket, identifier, &snapshot); err != nil {
+		return nil, err
+	}
+
+	if snapshot.Docker != nil {
+		snapshot.Snapshot.Docker = snapshot.Docker.DockerSnapshot
+	}
+
+	return &snapshot.Snapshot, nil
+}

api/http/handler/endpoints/endpoint_inspect.go

@@ -19,6 +19,8 @@ import (
 // @security jwt
 // @produce json
 // @param id path int true "Environment(Endpoint) identifier"
+// @param excludeSnapshot query bool false "if true, the snapshot data won't be retrieved"
+// @param excludeSnapshotRaw query bool false "if true, the SnapshotRaw field won't be retrieved"
 // @success 200 {object} portainer.Endpoint "Success"
 // @failure 400 "Invalid request"
 // @failure 404 "Environment(Endpoint) not found"
@@ -37,8 +39,7 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 		return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
 	}
 
-	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
-	if err != nil {
+	if err := handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint); err != nil {
 		return httperror.Forbidden("Permission denied to access environment", err)
 	}
 
@@ -51,9 +52,11 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 	endpointutils.UpdateEdgeEndpointHeartbeat(endpoint, settings)
 	endpoint.ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
 
-	if !excludeSnapshot(r) {
-		err = handler.SnapshotService.FillSnapshotData(endpoint)
-		if err != nil {
+	excludeSnapshot, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshot", true)
+	excludeRaw, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshotRaw", true)
+
+	if !excludeSnapshot {
+		if err := handler.SnapshotService.FillSnapshotData(endpoint, !excludeRaw); err != nil {
 			return httperror.InternalServerError("Unable to add snapshot data", err)
 		}
 	}
@@ -83,9 +86,3 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 
 	return response.JSON(w, endpoint)
 }
-
-func excludeSnapshot(r *http.Request) bool {
-	excludeSnapshot, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshot", true)
-
-	return excludeSnapshot
-}

api/http/handler/endpoints/endpoint_list.go

@@ -44,6 +44,7 @@ const (
 // @param edgeDeviceUntrusted query bool false "if true, show only untrusted edge agents, if false show only trusted edge agents (relevant only for edge agents)"
 // @param edgeCheckInPassedSeconds query number false "if bigger then zero, show only edge agents that checked-in in the last provided seconds (relevant only for edge agents)"
 // @param excludeSnapshots query bool false "if true, the snapshot data won't be retrieved"
+// @param excludeSnapshotRaw query bool false "if true, the SnapshotRaw field won't be retrieved"
 // @param name query string false "will return only environments(endpoints) with this name"
 // @param edgeStackId query portainer.EdgeStackID false "will return the environements of the specified edge stack"
 // @param edgeStackStatus query string false "only applied when edgeStackId exists. Filter the returned environments based on their deployment status in the stack (not the environment status!)" Enum("Pending", "Ok", "Error", "Acknowledged", "Remove", "RemoteUpdateSuccess", "ImagesPulled")
@@ -59,6 +60,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 	limit, _ := request.RetrieveNumericQueryParameter(r, "limit", true)
 	sortField, _ := request.RetrieveQueryParameter(r, "sort", true)
 	sortOrder, _ := request.RetrieveQueryParameter(r, "order", true)
+	excludeRaw, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshotRaw", true)
 
 	endpointGroups, err := handler.DataStore.EndpointGroup().ReadAll()
 	if err != nil {
@@ -114,7 +116,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 		endpointutils.UpdateEdgeEndpointHeartbeat(&paginatedEndpoints[idx], settings)
 
 		if !query.excludeSnapshots {
-			if err := handler.SnapshotService.FillSnapshotData(&paginatedEndpoints[idx]); err != nil {
+			if err := handler.SnapshotService.FillSnapshotData(&paginatedEndpoints[idx], !excludeRaw); err != nil {
 				return httperror.InternalServerError("Unable to add snapshot data", err)
 			}
 		}

api/http/handler/endpoints/endpoint_update.go

@@ -272,7 +272,7 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
-	if err := handler.SnapshotService.FillSnapshotData(endpoint); err != nil {
+	if err := handler.SnapshotService.FillSnapshotData(endpoint, true); err != nil {
 		return httperror.InternalServerError("Unable to add snapshot data", err)
 	}
 

api/http/handler/system/nodes_count.go

@@ -33,7 +33,7 @@ func (handler *Handler) systemNodesCount(w http.ResponseWriter, r *http.Request)
 	var nodes int
 
 	for _, endpoint := range endpoints {
-		if err := snapshot.FillSnapshotData(handler.dataStore, &endpoint); err != nil {
+		if err := snapshot.FillSnapshotData(handler.dataStore, &endpoint, false); err != nil {
 			return httperror.InternalServerError("Unable to add snapshot data", err)
 		}
 

api/http/proxy/factory/docker/volumes.go

@@ -224,7 +224,7 @@ func (transport *Transport) getDockerID() (string, error) {
 	if transport.snapshotService != nil {
 		endpoint := portainer.Endpoint{ID: transport.endpoint.ID}
 
-		if err := transport.snapshotService.FillSnapshotData(&endpoint); err == nil && len(endpoint.Snapshots) > 0 {
+		if err := transport.snapshotService.FillSnapshotData(&endpoint, true); err == nil && len(endpoint.Snapshots) > 0 {
 			if dockerID, err := snapshot.FetchDockerID(endpoint.Snapshots[0]); err == nil {
 				transport.dockerID = dockerID
 				return dockerID, nil

api/internal/snapshot/snapshot.go

@@ -170,8 +170,8 @@ func (service *Service) Create(snapshot portainer.Snapshot) error {
 	return service.dataStore.Snapshot().Create(&snapshot)
 }
 
-func (service *Service) FillSnapshotData(endpoint *portainer.Endpoint) error {
-	return FillSnapshotData(service.dataStore, endpoint)
+func (service *Service) FillSnapshotData(endpoint *portainer.Endpoint, includeRaw bool) error {
+	return FillSnapshotData(service.dataStore, endpoint, includeRaw)
 }
 
 func (service *Service) snapshotKubernetesEndpoint(endpoint *portainer.Endpoint) error {
@@ -328,8 +328,16 @@ func FetchDockerID(snapshot portainer.DockerSnapshot) (string, error) {
 	return info.Swarm.Cluster.ID, nil
 }
 
-func FillSnapshotData(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint) error {
-	snapshot, err := tx.Snapshot().Read(endpoint.ID)
+func FillSnapshotData(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint, includeRaw bool) error {
+	var snapshot *portainer.Snapshot
+	var err error
+
+	if includeRaw {
+		snapshot, err = tx.Snapshot().Read(endpoint.ID)
+	} else {
+		snapshot, err = tx.Snapshot().ReadWithoutSnapshotRaw(endpoint.ID)
+	}
+
 	if tx.IsErrObjectNotFound(err) {
 		endpoint.Snapshots = []portainer.DockerSnapshot{}
 		endpoint.Kubernetes.Snapshots = []portainer.KubernetesSnapshot{}

api/portainer.go

@@ -1622,7 +1622,7 @@ type (
 		Start()
 		SetSnapshotInterval(snapshotInterval string) error
 		SnapshotEndpoint(endpoint *Endpoint) error
-		FillSnapshotData(endpoint *Endpoint) error
+		FillSnapshotData(endpoint *Endpoint, includeRaw bool) error
 	}
 
 	// SwarmStackManager represents a service to manage Swarm stacks

app/react/hooks/useCurrentEnvironment.ts

@@ -4,5 +4,5 @@ import { useEnvironmentId } from './useEnvironmentId';
 
 export function useCurrentEnvironment(force = true) {
   const id = useEnvironmentId(force);
-  return useEnvironment(id);
+  return useEnvironment(id, undefined, { excludeSnapshot: false });
 }

app/react/portainer/HomeView/EnvironmentList/EnvironmentItem/EngineVersion.tsx

@@ -1,16 +1,16 @@
 import { DockerSnapshot } from '@/react/docker/snapshots/types';
-import { useIsPodman } from '@/react/portainer/environments/queries/useIsPodman';
 import {
   Environment,
   PlatformType,
   KubernetesSnapshot,
+  ContainerEngine,
 } from '@/react/portainer/environments/types';
 import { getPlatformType } from '@/react/portainer/environments/utils';
 import { getDockerEnvironmentType } from '@/react/portainer/environments/utils/getDockerEnvironmentType';
 
 export function EngineVersion({ environment }: { environment: Environment }) {
   const platform = getPlatformType(environment.Type);
-  const isPodman = useIsPodman(environment.Id);
+  const isPodman = environment.ContainerEngine === ContainerEngine.Podman;
 
   switch (platform) {
     case PlatformType.Docker:

app/react/portainer/HomeView/EnvironmentList/EnvironmentList.tsx

@@ -110,6 +110,7 @@ export function EnvironmentList({ onClickBrowse, onRefresh }: Props) {
     updateInformation: isBE,
     edgeAsync: getEdgeAsyncValue(connectionTypes),
     platformTypes,
+    excludeSnapshotRaw: true,
   };
 
   const queryWithSort = {

app/react/portainer/environments/environment.service/index.ts

@@ -42,6 +42,7 @@ export interface BaseEnvironmentsQueryParams {
   edgeAsync?: boolean;
   edgeDeviceUntrusted?: boolean;
   excludeSnapshots?: boolean;
+  excludeSnapshotRaw?: boolean;
   provisioned?: boolean;
   name?: string;
   agentVersions?: string[];
@@ -119,9 +120,15 @@ export async function getAgentVersions() {
   }
 }
 
-export async function getEndpoint(id: EnvironmentId) {
+export async function getEndpoint(
+  id: EnvironmentId,
+  excludeSnapshot = true,
+  excludeSnapshotRaw = true
+) {
   try {
-    const { data: endpoint } = await axios.get<Environment>(buildUrl(id));
+    const { data: endpoint } = await axios.get<Environment>(buildUrl(id), {
+      params: { excludeSnapshot, excludeSnapshotRaw },
+    });
     return endpoint;
   } catch (e) {
     throw parseAxiosError(e as Error);

app/react/portainer/environments/queries/useEnvironment.ts

@@ -10,11 +10,20 @@ import { environmentQueryKeys } from './query-keys';
 export function useEnvironment<T = Environment>(
   environmentId?: EnvironmentId,
   select?: (environment: Environment) => T,
-  options?: { autoRefreshRate?: number }
+  options?: {
+    autoRefreshRate?: number;
+    excludeSnapshot?: boolean;
+    excludeSnapshotRaw?: boolean;
+  }
 ) {
   return useQuery(
     environmentQueryKeys.item(environmentId!),
-    () => getEndpoint(environmentId!),
+    () =>
+      getEndpoint(
+        environmentId!,
+        options?.excludeSnapshot ?? undefined,
+        options?.excludeSnapshotRaw ?? undefined
+      ),
     {
       select,
       ...withError('Failed loading environment'),

app/react/portainer/environments/update-schedules/common/useEnvironments.ts

@@ -3,7 +3,7 @@ import { EdgeGroupId, EdgeTypes } from '@/react/portainer/environments/types';
 
 export function useEnvironments(edgeGroupIds: Array<EdgeGroupId>) {
   const environmentsQuery = useEnvironmentList(
-    { edgeGroupIds, types: EdgeTypes, pageLimit: 0 },
+    { edgeGroupIds, types: EdgeTypes, pageLimit: 0, excludeSnapshots: true },
     {
       enabled: edgeGroupIds.length > 0,
     }