Fixed more issues found while testing changes for large file download. #3369

pull/8786/head
Aditya Toshniwal 2025-05-24 15:25:07 +05:30
parent dfd896db10
commit c6183c9d03
14 changed files with 163 additions and 149 deletions

View File

@ -82,7 +82,7 @@ async function fileDownloadPath(callerWindow, options, prompt=true) {
export function setupDownloader() {
// Listen for the renderer's request to show the open dialog
ipcMain.handle('get-download-path', async (event, options, prompt=true) => {
ipcMain.handle('get-download-stream-path', async (event, options, prompt=true) => {
try {
const callerWindow = BrowserWindow.fromWebContents(event.sender);
const filePath = await fileDownloadPath(callerWindow, options, prompt);
@ -97,25 +97,25 @@ export function setupDownloader() {
return filePath;
} catch (error) {
writeServerLog(`Error in get-download-path: ${error}`);
writeServerLog(`Error in get-download-stream-path: ${error}`);
}
});
ipcMain.on('download-data-save-total', (event, filePath, total) => {
ipcMain.on('download-stream-save-total', (event, filePath, total) => {
const item = downloadQueue[filePath];
if (item) {
item.setTotal(total);
}
});
ipcMain.on('download-data-save-chunk', (event, filePath, chunk) => {
ipcMain.on('download-stream-save-chunk', (event, filePath, chunk) => {
const item = downloadQueue[filePath];
if (item) {
item.write(chunk);
}
});
ipcMain.on('download-data-save-end', (event, filePath, openFile=false) => {
ipcMain.on('download-stream-save-end', (event, filePath, openFile=false) => {
const item = downloadQueue[filePath];
if (item) {
item.remove();
@ -124,11 +124,18 @@ export function setupDownloader() {
});
// non-streaming direct download
ipcMain.handle('download-base64-url', async (event, base64url, options, prompt=true, openFile=false) => {
ipcMain.handle('download-base64-url-data', async (event, base64url, options, prompt=true, openFile=false) => {
const callerWindow = BrowserWindow.fromWebContents(event.sender);
const filePath = await fileDownloadPath(callerWindow, options, prompt);
const buffer = Buffer.from(base64url.split(',')[1], 'base64');
fs.writeFileSync(filePath, buffer);
openFile && shell.openPath(filePath);
});
ipcMain.handle('download-text-data', async (event, text, options, prompt=true, openFile=false) => {
const callerWindow = BrowserWindow.fromWebContents(event.sender);
const filePath = await fileDownloadPath(callerWindow, options, prompt);
fs.writeFileSync(filePath, text);
openFile && shell.openPath(filePath);
});
}

View File

@ -10,6 +10,7 @@
const { contextBridge, ipcRenderer } = require('electron/renderer');
contextBridge.exposeInMainWorld('electronUI', {
focus: () => ipcRenderer.send('focus'),
onMenuClick: (callback) => ipcRenderer.on('menu-click', (_event, details) => callback(details)),
setMenus: (menus) => {
ipcRenderer.send('setMenus', menus);
@ -25,9 +26,10 @@ contextBridge.exposeInMainWorld('electronUI', {
log: (text)=> ipcRenderer.send('log', text),
reloadApp: ()=>{ipcRenderer.send('reloadApp');},
// Download related functions
getDownloadPath: (...args) => ipcRenderer.invoke('get-download-path', ...args),
downloadDataSaveChunk: (...args) => ipcRenderer.send('download-data-save-chunk', ...args),
downloadDataSaveTotal: (...args) => ipcRenderer.send('download-data-save-total', ...args),
downloadDataSaveEnd: (...args) => ipcRenderer.send('download-data-save-end', ...args),
downloadBase64UrlData: (...args) => ipcRenderer.invoke('download-base64-url', ...args)
getDownloadStreamPath: (...args) => ipcRenderer.invoke('get-download-stream-path', ...args),
downloadStreamSaveChunk: (...args) => ipcRenderer.send('download-stream-save-chunk', ...args),
downloadStreamSaveTotal: (...args) => ipcRenderer.send('download-stream-save-total', ...args),
downloadStreamSaveEnd: (...args) => ipcRenderer.send('download-stream-save-end', ...args),
downloadBase64UrlData: (...args) => ipcRenderer.invoke('download-base64-url-data', ...args),
downloadTextData: (...args) => ipcRenderer.invoke('download-text-data', ...args)
});

View File

@ -40,7 +40,7 @@ import Replication from './Replication';
import { getExpandCell } from '../../../static/js/components/PgReactTableStyled';
import CodeMirror from '../../../static/js/components/ReactCodeMirror';
import GetAppRoundedIcon from '@mui/icons-material/GetAppRounded';
import { downloadTextData } from '../../../static/js/download_utils';
import DownloadUtils from '../../../static/js/DownloadUtils';
import RefreshButton from './components/RefreshButtons';
function parseData(data) {
@ -451,7 +451,7 @@ function Dashboard({
let fileName = 'data-' + new Date().getTime() + extension;
try {
downloadTextData(respData, fileName, `text/${type}`);
DownloadUtils.downloadTextData(respData, fileName, `text/${type}`);
} catch {
setSsMsg(gettext('Failed to download the logs.'));
}

View File

@ -32,7 +32,7 @@ import Uploader from './Uploader';
import GridView from './GridView';
import convert from 'convert-units';
import PropTypes from 'prop-types';
import { downloadBlob } from '../../../../../static/js/download_utils';
import DownloadUtils from '../../../../../static/js/DownloadUtils';
import ErrorBoundary from '../../../../../static/js/helpers/ErrorBoundary';
import { MY_STORAGE } from './FileManagerConstants';
import _ from 'lodash';
@ -311,7 +311,7 @@ export class FileManagerUtils {
'storage_folder': ss,
},
});
downloadBlob(res.data, res.headers.filename);
DownloadUtils.downloadBlob(res.data, res.headers.filename);
}
setDialogView(view) {

View File

@ -154,6 +154,7 @@ export default function BrowserComponent({pgAdmin}) {
useBeforeUnload({
enabled: confirmOnClose,
beforeClose: (forceClose)=>{
window.electronUI?.focus();
pgAdmin.Browser.notifier.confirm(
gettext('Quit pgAdmin 4'),
gettext('Are you sure you want to quit the application?'),

View File

@ -0,0 +1,125 @@
//////////////////////////////////////////////////////////////////////////
//
// pgAdmin 4 - PostgreSQL Tools
//
// Copyright (C) 2013 - 2025, The pgAdmin Development Team
// This software is released under the PostgreSQL Licence
//
//////////////////////////////////////////////////////////////////////////
import usePreferences from '../../preferences/static/js/store';
import { callFetch, parseApiError } from './api_instance';
import { getBrowser, toPrettySize } from './utils';
const DownloadUtils = {
downloadViaLink: function (url, fileName) {
const link = document.createElement('a');
link.href = url;
link.download = fileName;
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
},
downloadBlob: function (blob, fileName) {
const urlCreator = window.URL || window.webkitURL;
const downloadUrl = urlCreator.createObjectURL(blob);
this.downloadViaLink(downloadUrl, fileName);
window.URL.revokeObjectURL(downloadUrl);
},
downloadTextData: function (textData, fileName, fileType) {
const respBlob = new Blob([textData], {type : fileType});
this.downloadBlob(respBlob, fileName);
},
downloadBase64UrlData: function (downloadUrl, fileName) {
this.downloadViaLink(downloadUrl, fileName);
},
downloadFileStream: async function (allOptions, fileName, fileType, onProgress) {
const data = [];
const response = await callFetch(allOptions.url, allOptions.options);
if(!response.ok) {
throw new Error(parseApiError(await response.json()));
}
if (!response.body) {
throw new Error(response.statusText);
}
const reader = response.body.getReader();
let done = false;
let receivedLength = 0; // received bytes
while (!done) {
const { value, done: doneReading } = await reader.read();
done = doneReading;
if (value) {
data.push(value);
receivedLength += value.length;
onProgress?.(toPrettySize(receivedLength, 'B', 2));
}
}
const blob = new Blob(data, {type: fileType});
this.downloadBlob(blob, fileName);
}
};
// If we are in Electron, we will use the Electron API to download files.
if(getBrowser().name == 'Electron') {
DownloadUtils.downloadTextData = async (textData, fileName, _fileType) =>{
const {automatically_open_downloaded_file, prompt_for_download_location} = usePreferences.getState().getPreferencesForModule('misc');
await window.electronUI.downloadTextData(textData, {
defaultPath: fileName,
}, prompt_for_download_location, automatically_open_downloaded_file);
};
DownloadUtils.downloadBase64UrlData = async (downloadUrl, fileName) => {
const {automatically_open_downloaded_file, prompt_for_download_location} = usePreferences.getState().getPreferencesForModule('misc');
await window.electronUI.downloadBase64UrlData(downloadUrl, {
defaultPath: fileName,
}, prompt_for_download_location, automatically_open_downloaded_file);
};
DownloadUtils.downloadFileStream = async (allOptions, fileName, _fileType, onProgress)=>{
const {automatically_open_downloaded_file, prompt_for_download_location} = usePreferences.getState().getPreferencesForModule('misc');
const filePath = await window.electronUI.getDownloadStreamPath({
defaultPath: fileName,
}, prompt_for_download_location);
// If the user cancels the download, we will not proceed
if(!filePath) {
return;
}
const response = await callFetch(allOptions.url, allOptions.options);
if(!response.ok) {
throw new Error(parseApiError(await response.json()));
}
if (!response.body) {
throw new Error(response.statusText);
}
const contentLength = response.headers.get('Content-Length');
window.electronUI.downloadStreamSaveTotal(filePath, contentLength ? parseInt(contentLength, 10) : null);
const reader = response.body.getReader();
let done = false;
let receivedLength = 0; // received bytes
while (!done) {
const { value, done: doneReading } = await reader.read();
done = doneReading;
if (value) {
window.electronUI.downloadStreamSaveChunk(filePath, value);
receivedLength += value.length;
onProgress?.(toPrettySize(receivedLength, 'B', 2));
}
}
window.electronUI.downloadStreamSaveEnd(filePath, automatically_open_downloaded_file);
};
}
export default DownloadUtils;

View File

@ -7,7 +7,7 @@
//
//////////////////////////////////////////////////////////////
import getApiInstance from '../api_instance';
import { downloadTextData } from '../download_utils';
import DownloadUtils from '../DownloadUtils';
function convertImageURLtoDataURI(api, image) {
return new Promise(function(resolve, reject) {
@ -43,6 +43,6 @@ export function downloadSvg(svg, svgName) {
}
Promise.all(image_promises).then(function() {
downloadTextData(svgElement.outerHTML, svgName, 'image/svg+xml');
DownloadUtils.downloadTextData(svgElement.outerHTML, svgName, 'image/svg+xml');
});
}

View File

@ -149,7 +149,7 @@ export function FormInput({ children, error, className, label, helpMessage, requ
<FormIcon type={MESSAGE_TYPE.ERROR} style={{ marginLeft: 'auto', visibility: error ? 'unset' : 'hidden' }} />
</InputLabel>;
return (
<StyledGrid container spacing={0} className={className} data-testid="form-input">
<StyledGrid container spacing={0} className={className} data-testid="form-input" width="100%">
<Grid size={{ lg: labelGridBasis, md: labelGridBasis, sm: 12, xs: 12 }}>
{
labelTooltip ?

View File

@ -1,121 +0,0 @@
//////////////////////////////////////////////////////////////////////////
//
// pgAdmin 4 - PostgreSQL Tools
//
// Copyright (C) 2013 - 2025, The pgAdmin Development Team
// This software is released under the PostgreSQL Licence
//
//////////////////////////////////////////////////////////////////////////
import usePreferences from '../../preferences/static/js/store';
import { callFetch, parseApiError } from './api_instance';
import { getBrowser, toPrettySize } from './utils';
// This function is used to download the base64 data
// and create a link to download the file.
export async function downloadBase64UrlData(downloadUrl, fileName) {
if(getBrowser().name == 'Electron') {
const {automatically_open_downloaded_file, prompt_for_download_location} = usePreferences.getState().getPreferencesForModule('misc');
// In Electron, we use the electronUI to download the file.
await window.electronUI.downloadBase64UrlData(downloadUrl, {
defaultPath: fileName,
}, prompt_for_download_location, automatically_open_downloaded_file);
return;
}
// In other browsers, we create a link to download the file.
let link = document.createElement('a');
link.setAttribute('href', downloadUrl);
link.setAttribute('download', fileName);
link.style.setProperty('visibility ', 'hidden');
document.body.appendChild(link);
link.click();
link.remove();
}
// This function is used to download the blob data
export async function downloadBlob(blob, fileName) {
const urlCreator = window.URL || window.webkitURL;
const downloadUrl = urlCreator.createObjectURL(blob);
downloadBase64UrlData(downloadUrl, fileName);
window.URL.revokeObjectURL(downloadUrl);
}
// This function is used to download the text data
export function downloadTextData(textData, fileName, fileType) {
const respBlob = new Blob([textData], {type : fileType});
downloadBlob(respBlob, fileName);
}
// This function is used to download the file from the given URL
// and use streaming to download the file in chunks where there
// is no limit on the file size.
export async function downloadFileStream(allOptions, fileName, fileType, onProgress) {
const {automatically_open_downloaded_file, prompt_for_download_location} = usePreferences.getState().getPreferencesForModule('misc');
const start = async (filePath, writer) => {
const response = await callFetch(allOptions.url, allOptions.options);
if(!response.ok) {
throw new Error(parseApiError(await response.json()));
}
if (!response.body) {
throw new Error(response.statusText);
}
const contentLength = response.headers.get('Content-Length');
writer.downloadDataSaveTotal(filePath, contentLength ? parseInt(contentLength, 10) : null);
const reader = response.body.getReader();
let done = false;
let receivedLength = 0; // received bytes
while (!done) {
const { value, done: doneReading } = await reader.read();
done = doneReading;
if (value) {
writer.downloadDataSaveChunk(filePath, value);
receivedLength += value.length;
onProgress?.(toPrettySize(receivedLength, 'B', 2));
}
}
writer.downloadDataSaveEnd(filePath, automatically_open_downloaded_file);
};
let writer;
let filePath = '';
try {
if(getBrowser().name != 'Electron') {
// In other browsers, we use the blob to download the file.
const data = [];
writer = {
downloadDataSaveChunk: (_fp, chunk) => {
data.push(chunk);
},
downloadDataSaveTotal: () => {
// This is not used in the browser
},
downloadDataSaveEnd: () => {
// This is not used in the browser
},
};
await start(filePath, writer);
const blob = new Blob(data, {type: fileType});
downloadBlob(blob, fileName);
} else {
writer = window.electronUI;
filePath = await window.electronUI.getDownloadPath({
defaultPath: fileName,
}, prompt_for_download_location);
// If the user cancels the download, we will not proceed
if(!filePath) {
return;
}
await start(filePath, writer);
}
} catch (error) {
writer.downloadDataSaveEnd(filePath);
throw new Error('Download failed: ' + error.message);
}
}

View File

@ -596,7 +596,7 @@ export function getRandomColor() {
// Using this function instead of 'btoa' directly.
// https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem
function stringToBase64(str) {
export function stringToBase64(str) {
return btoa(
Array.from(
new TextEncoder().encode(str),

View File

@ -37,7 +37,7 @@ import pgAdmin from 'sources/pgadmin';
import { styled } from '@mui/material/styles';
import BeforeUnload from './BeforeUnload';
import { isMac } from '../../../../../../static/js/keyboard_shortcuts';
import { downloadBase64UrlData } from '../../../../../../static/js/download_utils';
import DownloadUtils from '../../../../../../static/js/DownloadUtils';
/* Custom react-diagram action for keyboard events */
export class KeyboardShortcutAction extends Action {
@ -760,7 +760,7 @@ export default class ERDTool extends React.Component {
}
toPng(this.canvasEle, {width, height})
.then((dataUrl)=>{
downloadBase64UrlData(dataUrl, `${this.getCurrentProjectName()}.png`);
DownloadUtils.downloadBase64UrlData(dataUrl, `${this.getCurrentProjectName()}.png`);
}).catch((err)=>{
console.error(err);
let msg = gettext('Unknown error. Check console logs');

View File

@ -26,7 +26,7 @@ import { LineChart, BarChart, PieChart, DATA_POINT_STYLE, DATA_POINT_SIZE,
LightenDarkenColor} from 'sources/chartjs';
import { QueryToolEventsContext, QueryToolContext } from '../QueryToolComponent';
import { QUERY_TOOL_EVENTS, PANELS } from '../QueryToolConstants';
import { downloadBase64UrlData } from '../../../../../../static/js/download_utils';
import DownloadUtils from '../../../../../../static/js/DownloadUtils';
import { getChartColor } from '../../../../../../static/js/utils';
const StyledBox = styled(Box)(({theme}) => ({
@ -381,7 +381,7 @@ export function GraphVisualiser({initColumns}) {
const onDownloadGraph = async ()=> {
let downloadUrl = chartObjRef.current.toBase64Image(),
fileName = 'graph_visualiser-' + new Date().getTime() + '.png';
downloadBase64UrlData(downloadUrl, fileName);
DownloadUtils.downloadBase64UrlData(downloadUrl, fileName);
};
// This plugin is used to set the background color of the canvas. Very useful

View File

@ -22,7 +22,7 @@ import { LayoutDockerContext } from '../../../../../../static/js/helpers/Layout'
import { GeometryViewer } from './GeometryViewer';
import Explain from '../../../../../../static/js/Explain';
import { QuerySources } from './QueryHistory';
import { downloadFileStream } from '../../../../../../static/js/download_utils';
import DownloadUtils from '../../../../../../static/js/DownloadUtils';
import CopyData from '../QueryToolDataGrid/CopyData';
import moment from 'moment';
import ConfirmSaveContent from '../../../../../../static/js/Dialogs/ConfirmSaveContent';
@ -516,7 +516,7 @@ export class ResultSetUtils {
async saveResultsToFile(fileName, onProgress) {
try {
this.hasQueryCommitted = false;
await downloadFileStream({
await DownloadUtils.downloadFileStream({
url: url_for('sqleditor.query_tool_download', {
'trans_id': this.transId,
}),

View File

@ -16,7 +16,7 @@ import FileManager, { FileManagerUtils, getComparator } from '../../../pgadmin/m
import MockAdapter from 'axios-mock-adapter';
import axios from 'axios';
import getApiInstance from '../../../pgadmin/static/js/api_instance';
import * as downloadUtils from '../../../pgadmin/static/js/download_utils';
import DownloadUtils from '../../../pgadmin/static/js/DownloadUtils';
import userEvent from '@testing-library/user-event';
const files = [
@ -345,9 +345,9 @@ describe('FileManagerUtils', ()=>{
});
it('downloadFile', async ()=>{
jest.spyOn(downloadUtils, 'downloadBlob').mockImplementation(() => {});
jest.spyOn(DownloadUtils, 'downloadBlob').mockImplementation(() => {});
let row = {Filename: 'newfile1', Path: '/home/newfile1', 'storage_folder': 'my_storage'};
await fmObj.downloadFile(row);
expect(downloadUtils.downloadBlob).toHaveBeenCalledWith('blobdata', 'newfile1');
expect(DownloadUtils.downloadBlob).toHaveBeenCalledWith('blobdata', 'newfile1');
});
});