From 76eb3e9b67d70baca69bc6e4711033ffdc52274c Mon Sep 17 00:00:00 2001 From: Murtuza Zabuawala Date: Mon, 27 Apr 2020 17:30:23 +0530 Subject: [PATCH] =?UTF-8?q?Fixed=20an=20issue=20when=20the=20user=20perfor?= =?UTF-8?q?ms=20refresh=20on=20a=20large=20size=C2=A0materialized=20view.?= =?UTF-8?q?=20Fixes=20#5213?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/en_US/release_notes_4_21.rst | 1 + .../databases/schemas/views/__init__.py | 186 ++++++++++++++++-- .../schemas/views/static/js/mview.js | 108 +++++++--- .../views/tests/test_mviews_refresh.py | 149 ++++++++++++++ 4 files changed, 409 insertions(+), 35 deletions(-) create mode 100644 web/pgadmin/browser/server_groups/servers/databases/schemas/views/tests/test_mviews_refresh.py diff --git a/docs/en_US/release_notes_4_21.rst b/docs/en_US/release_notes_4_21.rst index 382b11332..400f35220 100644 --- a/docs/en_US/release_notes_4_21.rst +++ b/docs/en_US/release_notes_4_21.rst @@ -60,6 +60,7 @@ Bug fixes | `Issue #5157 `_ - Ensure that default sort order should be using the primary key in View/Edit data. | `Issue #5180 `_ - Fixed an issue where the autovacuum_enabled parameter is added automatically in the RE-SQL when the table has been created using the WITH clause. | `Issue #5210 `_ - Ensure that text larger than underlying field size should not be truncated automatically. +| `Issue #5213 `_ - Fixed an issue when the user performs refresh on a large size materialized view. | `Issue #5227 `_ - Fixed an issue where user cannot be added if many users are already exists. | `Issue #5268 `_ - Fixed generated SQL when any token in FTS Configuration or any option in FTS Dictionary is changed. | `Issue #5270 `_ - Ensure that OID should be shown in properties for Synonyms. diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/views/__init__.py b/web/pgadmin/browser/server_groups/servers/databases/schemas/views/__init__.py index a8da96e12..c64de73bd 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/views/__init__.py +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/views/__init__.py @@ -16,7 +16,7 @@ from functools import wraps import simplejson as json from flask import render_template, request, jsonify, current_app from flask_babelex import gettext - +from flask_security import current_user import pgadmin.browser.server_groups.servers.databases as databases from config import PG_DEFAULT_DRIVER from pgadmin.browser.server_groups.servers.databases.schemas.utils import \ @@ -29,6 +29,9 @@ from pgadmin.utils.ajax import make_json_response, internal_server_error, \ from pgadmin.utils.driver import get_driver from pgadmin.tools.schema_diff.node_registry import SchemaDiffRegistry from pgadmin.tools.schema_diff.compare import SchemaDiffObjectCompare +from pgadmin.utils import html, does_utility_exist +from pgadmin.model import Server +from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc """ @@ -128,6 +131,53 @@ class ViewModule(SchemaChildModule): return snippets +class Message(IProcessDesc): + def __init__(self, _sid, _data, _query): + self.sid = _sid + self.data = _data + self.query = _query + + @property + def message(self): + res = gettext("Refresh Materialized View") + opts = [] + if not self.data['is_with_data']: + opts.append(gettext("With no data")) + else: + opts.append(gettext("With data")) + if self.data['is_concurrent']: + opts.append(gettext("Concurrently")) + + return res + " ({0})".format(', '.join(str(x) for x in opts)) + + @property + def type_desc(self): + return gettext("Refresh Materialized View") + + def details(self, cmd, args): + res = gettext("Refresh Materialized View ({0})") + opts = [] + if not self.data['is_with_data']: + opts.append(gettext("WITH NO DATA")) + else: + opts.append(gettext("WITH DATA")) + + if self.data['is_concurrent']: + opts.append(gettext("CONCURRENTLY")) + + res = res.format(', '.join(str(x) for x in opts)) + + res = '
' + html.safe_str(res) + + res += '
' + res += gettext("Running Query:") + res += '
' + res += html.safe_str(self.query) + res += '
' + + return res + + class MViewModule(ViewModule): """ class MViewModule(ViewModule) @@ -1504,7 +1554,8 @@ class ViewNode(PGChildNodeView, VacuumSettings, SchemaDiffObjectCompare): # Override the operations for materialized view mview_operations = { - 'refresh_data': [{'put': 'refresh_data'}, {}] + 'refresh_data': [{'put': 'refresh_data'}, {}], + 'check_utility_exists': [{'get': 'check_utility_exists'}, {}] } mview_operations.update(ViewNode.operations) @@ -2010,7 +2061,9 @@ class MViewNode(ViewNode, VacuumSettings): is_concurrent = json.loads(data['concurrent']) with_data = json.loads(data['with_data']) - + data = dict() + data['is_concurrent'] = is_concurrent + data['is_with_data'] = with_data try: # Fetch view name by view id @@ -2019,6 +2072,10 @@ class MViewNode(ViewNode, VacuumSettings): status, res = self.conn.execute_dict(SQL) if not status: return internal_server_error(errormsg=res) + if len(res['rows']) == 0: + return gone( + gettext("""Could not find the materialized view.""") + ) # Refresh view SQL = render_template( @@ -2028,21 +2085,91 @@ class MViewNode(ViewNode, VacuumSettings): is_concurrent=is_concurrent, with_data=with_data ) - status, res_data = self.conn.execute_dict(SQL) - if not status: - return internal_server_error(errormsg=res_data) + # Fetch the server details like hostname, port, roles etc + server = Server.query.filter_by( + id=sid).first() + + if server is None: + return make_json_response( + success=0, + errormsg=gettext("Could not find the given server") + ) + + # To fetch MetaData for the server + driver = get_driver(PG_DEFAULT_DRIVER) + manager = driver.connection_manager(server.id) + conn = manager.connection() + connected = conn.connected() + + if not connected: + return make_json_response( + success=0, + errormsg=gettext("Please connect to the server first.") + ) + # Fetch the database name from connection manager + db_info = manager.db_info.get(did, None) + if db_info: + data['database'] = db_info['datname'] + else: + return make_json_response( + success=0, + errormsg=gettext( + "Could not find the database on the server.") + ) + utility = manager.utility('sql') + ret_val = does_utility_exist(utility) + if ret_val: + return make_json_response( + success=0, + errormsg=ret_val + ) + + args = [ + '--host', + manager.local_bind_host if manager.use_ssh_tunnel + else server.host, + '--port', + str(manager.local_bind_port) if manager.use_ssh_tunnel + else str(server.port), + '--username', server.username, '--dbname', + data['database'], + '--command', SQL + ] + + try: + p = BatchProcess( + desc=Message(sid, data, SQL), + cmd=utility, args=args + ) + manager.export_password_env(p.id) + # Check for connection timeout and if it is greater than 0 + # then set the environment variable PGCONNECT_TIMEOUT. + if manager.connect_timeout > 0: + env = dict() + env['PGCONNECT_TIMEOUT'] = str(manager.connect_timeout) + p.set_env_variables(server, env=env) + else: + p.set_env_variables(server) + + p.start() + jid = p.id + except Exception as e: + current_app.logger.exception(e) + return make_json_response( + status=410, + success=0, + errormsg=str(e) + ) + # Return response return make_json_response( - success=1, - info=gettext("View refreshed"), data={ - 'id': vid, - 'sid': sid, - 'gid': gid, - 'did': did + 'job_id': jid, + 'status': True, + 'info': gettext( + 'Materialized view refresh job created.') } ) - except Exception as e: current_app.logger.exception(e) return internal_server_error(errormsg=str(e)) @@ -2073,6 +2200,39 @@ class MViewNode(ViewNode, VacuumSettings): return res + @check_precondition + def check_utility_exists(self, gid, sid, did, scid, vid): + """ + This function checks the utility file exist on the given path. + + Args: + sid: Server ID + Returns: + None + """ + server = Server.query.filter_by( + id=sid, user_id=current_user.id + ).first() + + if server is None: + return make_json_response( + success=0, + errormsg=gettext("Could not find the specified server.") + ) + + driver = get_driver(PG_DEFAULT_DRIVER) + manager = driver.connection_manager(server.id) + + utility = manager.utility('sql') + ret_val = does_utility_exist(utility) + if ret_val: + return make_json_response( + success=0, + errormsg=ret_val + ) + + return make_json_response(success=1) + SchemaDiffRegistry(view_blueprint.node_type, ViewNode) ViewNode.register_node_view(view_blueprint) diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/views/static/js/mview.js b/web/pgadmin/browser/server_groups/servers/databases/schemas/views/static/js/mview.js index 94c7cc431..702569e8f 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/views/static/js/mview.js +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/views/static/js/mview.js @@ -275,34 +275,98 @@ define('pgadmin.node.mview', [ obj = this, t = pgBrowser.tree, i = input.item || t.selected(), - d = i && i.length == 1 ? t.itemData(i) : undefined; + d = i && i.length == 1 ? t.itemData(i) : undefined, + server_data = null; if (!d) return false; - // Make ajax call to refresh mview data - $.ajax({ - url: obj.generate_url(i, 'refresh_data' , d, true), - type: 'PUT', - data: {'concurrent': args.concurrent, 'with_data': args.with_data}, - dataType: 'json', - }) - .done(function(res) { - if (res.success == 1) { - Alertify.success(gettext('View refreshed successfully')); - } - else { - Alertify.alert( - gettext('Error refreshing view'), - res.data.result - ); - } - }) - .fail(function(xhr, status, error) { - Alertify.pgRespErrorNotify(xhr, error, gettext('Error refreshing view')); - }); + while (i) { + var node_data = pgBrowser.tree.itemData(i); + if (node_data._type == 'server') { + server_data = node_data; + break; + } + if (pgBrowser.tree.hasParent(i)) { + i = $(pgBrowser.tree.parent(i)); + } else { + Alertify.alert(gettext('Please select server or child node from tree.')); + break; + } + } + + if (!server_data) { + return; + } + + var module = 'paths', + preference_name = 'pg_bin_dir', + msg = gettext('Please configure the PostgreSQL Binary Path in the Preferences dialog.'); + + if ((server_data.type && server_data.type == 'ppas') || + server_data.server_type == 'ppas') { + preference_name = 'ppas_bin_dir'; + msg = gettext('Please configure the EDB Advanced Server Binary Path in the Preferences dialog.'); + } + + var preference = pgBrowser.get_preference(module, preference_name); + + if (preference) { + if (!preference.value) { + Alertify.alert(gettext('Configuration required'), msg); + return; + } + } else { + Alertify.alert(gettext('Failed to load preference %s of module %s', preference_name, module)); + return; + } + + $.ajax({ + url: obj.generate_url(i, 'check_utility_exists' , d, true), + type: 'GET', + dataType: 'json', + }).done(function(res) { + if (!res.success) { + Alertify.alert( + gettext('Utility not found'), + res.errormsg + ); + return; + } + // Make ajax call to refresh mview data + $.ajax({ + url: obj.generate_url(i, 'refresh_data' , d, true), + type: 'PUT', + data: {'concurrent': args.concurrent, 'with_data': args.with_data}, + dataType: 'json', + }) + .done(function(res) { + if (res.data && res.data.status) { + //Do nothing as we are creating the job and exiting from the main dialog + Alertify.success(res.data.info); + pgBrowser.Events.trigger('pgadmin-bgprocess:created', obj); + } else { + Alertify.alert( + gettext('Failed to create materialized view refresh job.'), + res.errormsg + ); + } + }) + .fail(function(xhr, status, error) { + Alertify.pgRespErrorNotify( + xhr, error, gettext('Failed to create materialized view refresh job.') + ); + }); + }).fail(function() { + Alertify.alert( + gettext('Utility not found'), + gettext('Failed to fetch Utility information') + ); + return; + }); }, + is_version_supported: function(data, item) { var t = pgAdmin.Browser.tree, i = item || t.selected(), diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/views/tests/test_mviews_refresh.py b/web/pgadmin/browser/server_groups/servers/databases/schemas/views/tests/test_mviews_refresh.py new file mode 100644 index 000000000..b6c0bad33 --- /dev/null +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/views/tests/test_mviews_refresh.py @@ -0,0 +1,149 @@ +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2020, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## + +import json +import uuid + +from pgadmin.browser.server_groups.servers.databases.schemas.tests import \ + utils as schema_utils +from pgadmin.browser.server_groups.servers.databases.tests import utils as \ + database_utils +from pgadmin.utils import server_utils as server_utils +from pgadmin.utils.route import BaseTestGenerator +from regression import parent_node_dict +from regression.python_test_utils import test_utils as utils +from . import utils as views_utils + +MVIEW_CHECK_UTILITY_URL = 'browser/mview/check_utility_exists/' +MVIEW_REFRESH_URL = 'browser/mview/refresh_data/' +IS_UTILITY_EXISTS = True + + +class MViewsUpdateParameterTestCase(BaseTestGenerator): + """This class will check materialized view refresh functionality.""" + scenarios = [ + ('Check utility route', + dict(type='check_utility') + ), + ('Refresh materialized view with invalid oid', + dict(type='invalid') + ), + ('Refresh materialized view with data', + dict(type='with_data') + ), + ('Refresh materialized view with no data', + dict(type='with_no_data') + ), + ('Refresh materialized view with data (concurrently)', + dict(type='with_data_concurrently') + ), + ('Refresh materialized view with no data (concurrently)', + dict(type='with_no_data_concurrently') + ), + ] + + @classmethod + def setUpClass(self): + self.db_name = parent_node_dict["database"][-1]["db_name"] + schema_info = parent_node_dict["schema"][-1] + self.server_id = schema_info["server_id"] + self.db_id = schema_info["db_id"] + server_response = server_utils.connect_server(self, self.server_id) + + if server_response["data"]["version"] < 90300 and "mview" in self.url: + message = "Materialized Views are not supported by PG9.2 " \ + "and PPAS9.2 and below." + self.skipTest(message) + + db_con = database_utils.connect_database(self, utils.SERVER_GROUP, + self.server_id, self.db_id) + if not db_con['data']["connected"]: + raise Exception("Could not connect to database to update a mview.") + self.schema_id = schema_info["schema_id"] + self.schema_name = schema_info["schema_name"] + schema_response = schema_utils.verify_schemas(self.server, + self.db_name, + self.schema_name) + if not schema_response: + raise Exception("Could not find the schema to update a mview.") + + self.m_view_name = "test_mview_put_%s" % (str(uuid.uuid4())[1:8]) + m_view_sql = "CREATE MATERIALIZED VIEW %s.%s TABLESPACE pg_default " \ + "AS SELECT 'test_pgadmin' WITH NO DATA;ALTER TABLE " \ + "%s.%s OWNER TO %s" + + self.m_view_id = views_utils.create_view(self.server, + self.db_name, + self.schema_name, + m_view_sql, + self.m_view_name) + + def runTest(self): + """This class will check materialized view refresh functionality""" + + mview_response = views_utils.verify_view(self.server, self.db_name, + self.m_view_name) + if not mview_response: + raise Exception("Could not find the mview to update.") + + data = None + is_put_request = True + + if self.type == 'check_utility': + is_put_request = False + elif self.type == 'invalid': + data = dict({'concurrent': 'false', 'with_data': 'false'}) + elif self.type == 'with_data': + data = dict({'concurrent': 'false', 'with_data': 'true'}) + elif self.type == 'with_no_data': + data = dict({'concurrent': 'false', 'with_data': 'false'}) + elif self.type == 'with_data_concurrently': + data = dict({'concurrent': 'true', 'with_data': 'true'}) + elif self.type == 'with_no_data_concurrently': + data = dict({'concurrent': 'true', 'with_data': 'false'}) + + response = self.tester.get( + MVIEW_CHECK_UTILITY_URL + str(utils.SERVER_GROUP) + '/' + + str(self.server_id) + '/' + + str(self.db_id) + '/' + + str(self.schema_id) + '/' + + str(self.m_view_id), + content_type='html/json' + ) + self.assertEquals(response.status_code, 200) + if is_put_request and response.json['success'] == 0: + self.skipTest( + "Couldn't check materialized view refresh" + " functionality because utility/binary does not exists." + ) + + if is_put_request: + mvid = self.m_view_id + if self.type == 'invalid': + mvid = 99999 + response = self.tester.put( + MVIEW_REFRESH_URL + str(utils.SERVER_GROUP) + '/' + + str(self.server_id) + '/' + + str(self.db_id) + '/' + + str(self.schema_id) + '/' + + str(mvid), + data=json.dumps(data), + follow_redirects=True + ) + if self.type == 'invalid': + self.assertEquals(response.status_code, 410) + else: + self.assertEquals(response.status_code, 200) + # On success we get job_id from server + self.assertTrue('job_id' in response.json['data']) + + @classmethod + def tearDownClass(self): + # Disconnect the database + database_utils.disconnect_database(self, self.server_id, self.db_id)