Fixed an issue when the user performs refresh on a large size materialized view. Fixes #5213

docs/en_US/release_notes_4_21.rst

@@ -60,6 +60,7 @@ Bug fixes
 | `Issue #5157 <https://redmine.postgresql.org/issues/5157>`_ -  Ensure that default sort order should be using the primary key in View/Edit data.
 | `Issue #5180 <https://redmine.postgresql.org/issues/5180>`_ -  Fixed an issue where the autovacuum_enabled parameter is added automatically in the RE-SQL when the table has been created using the WITH clause.
 | `Issue #5210 <https://redmine.postgresql.org/issues/5210>`_ -  Ensure that text larger than underlying field size should not be truncated automatically.
+| `Issue #5213 <https://redmine.postgresql.org/issues/5213>`_ -  Fixed an issue when the user performs refresh on a large size materialized view.
 | `Issue #5227 <https://redmine.postgresql.org/issues/5227>`_ -  Fixed an issue where user cannot be added if many users are already exists.
 | `Issue #5268 <https://redmine.postgresql.org/issues/5268>`_ -  Fixed generated SQL when any token in FTS Configuration or any option in FTS Dictionary is changed.
 | `Issue #5270 <https://redmine.postgresql.org/issues/5270>`_ -  Ensure that OID should be shown in properties for Synonyms.

web/pgadmin/browser/server_groups/servers/databases/schemas/views/__init__.py

@@ -16,7 +16,7 @@ from functools import wraps
 import simplejson as json
 from flask import render_template, request, jsonify, current_app
 from flask_babelex import gettext
-
+from flask_security import current_user
 import pgadmin.browser.server_groups.servers.databases as databases
 from config import PG_DEFAULT_DRIVER
 from pgadmin.browser.server_groups.servers.databases.schemas.utils import \
@@ -29,6 +29,9 @@ from pgadmin.utils.ajax import make_json_response, internal_server_error, \
 from pgadmin.utils.driver import get_driver
 from pgadmin.tools.schema_diff.node_registry import SchemaDiffRegistry
 from pgadmin.tools.schema_diff.compare import SchemaDiffObjectCompare
+from pgadmin.utils import html, does_utility_exist
+from pgadmin.model import Server
+from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
 
 
 """
@@ -128,6 +131,53 @@ class ViewModule(SchemaChildModule):
         return snippets
 
 
+class Message(IProcessDesc):
+    def __init__(self, _sid, _data, _query):
+        self.sid = _sid
+        self.data = _data
+        self.query = _query
+
+    @property
+    def message(self):
+        res = gettext("Refresh Materialized View")
+        opts = []
+        if not self.data['is_with_data']:
+            opts.append(gettext("With no data"))
+        else:
+            opts.append(gettext("With data"))
+        if self.data['is_concurrent']:
+            opts.append(gettext("Concurrently"))
+
+        return res + " ({0})".format(', '.join(str(x) for x in opts))
+
+    @property
+    def type_desc(self):
+        return gettext("Refresh Materialized View")
+
+    def details(self, cmd, args):
+        res = gettext("Refresh Materialized View ({0})")
+        opts = []
+        if not self.data['is_with_data']:
+            opts.append(gettext("WITH NO DATA"))
+        else:
+            opts.append(gettext("WITH DATA"))
+
+        if self.data['is_concurrent']:
+            opts.append(gettext("CONCURRENTLY"))
+
+        res = res.format(', '.join(str(x) for x in opts))
+
+        res = '<div>' + html.safe_str(res)
+
+        res += '</div><div class="py-1">'
+        res += gettext("Running Query:")
+        res += '<div class="pg-bg-cmd enable-selection p-1">'
+        res += html.safe_str(self.query)
+        res += '</div></div>'
+
+        return res
+
+
 class MViewModule(ViewModule):
     """
      class MViewModule(ViewModule)
@@ -1504,7 +1554,8 @@ class ViewNode(PGChildNodeView, VacuumSettings, SchemaDiffObjectCompare):
 
 # Override the operations for materialized view
 mview_operations = {
-    'refresh_data': [{'put': 'refresh_data'}, {}]
+    'refresh_data': [{'put': 'refresh_data'}, {}],
+    'check_utility_exists': [{'get': 'check_utility_exists'}, {}]
 }
 mview_operations.update(ViewNode.operations)
 
@@ -2010,7 +2061,9 @@ class MViewNode(ViewNode, VacuumSettings):
 
         is_concurrent = json.loads(data['concurrent'])
         with_data = json.loads(data['with_data'])
-
+        data = dict()
+        data['is_concurrent'] = is_concurrent
+        data['is_with_data'] = with_data
         try:
 
             # Fetch view name by view id
@@ -2019,6 +2072,10 @@ class MViewNode(ViewNode, VacuumSettings):
             status, res = self.conn.execute_dict(SQL)
             if not status:
                 return internal_server_error(errormsg=res)
+            if len(res['rows']) == 0:
+                return gone(
+                    gettext("""Could not find the materialized view.""")
+                )
 
             # Refresh view
             SQL = render_template(
@@ -2028,21 +2085,91 @@ class MViewNode(ViewNode, VacuumSettings):
                 is_concurrent=is_concurrent,
                 with_data=with_data
             )
-            status, res_data = self.conn.execute_dict(SQL)
-            if not status:
-                return internal_server_error(errormsg=res_data)
 
+            # Fetch the server details like hostname, port, roles etc
+            server = Server.query.filter_by(
+                id=sid).first()
+
+            if server is None:
+                return make_json_response(
+                    success=0,
+                    errormsg=gettext("Could not find the given server")
+                )
+
+            # To fetch MetaData for the server
+            driver = get_driver(PG_DEFAULT_DRIVER)
+            manager = driver.connection_manager(server.id)
+            conn = manager.connection()
+            connected = conn.connected()
+
+            if not connected:
+                return make_json_response(
+                    success=0,
+                    errormsg=gettext("Please connect to the server first.")
+                )
+            # Fetch the database name from connection manager
+            db_info = manager.db_info.get(did, None)
+            if db_info:
+                data['database'] = db_info['datname']
+            else:
+                return make_json_response(
+                    success=0,
+                    errormsg=gettext(
+                        "Could not find the database on the server.")
+                )
+            utility = manager.utility('sql')
+            ret_val = does_utility_exist(utility)
+            if ret_val:
+                return make_json_response(
+                    success=0,
+                    errormsg=ret_val
+                )
+
+            args = [
+                '--host',
+                manager.local_bind_host if manager.use_ssh_tunnel
+                else server.host,
+                '--port',
+                str(manager.local_bind_port) if manager.use_ssh_tunnel
+                else str(server.port),
+                '--username', server.username, '--dbname',
+                data['database'],
+                '--command', SQL
+            ]
+
+            try:
+                p = BatchProcess(
+                    desc=Message(sid, data, SQL),
+                    cmd=utility, args=args
+                )
+                manager.export_password_env(p.id)
+                # Check for connection timeout and if it is greater than 0
+                # then set the environment variable PGCONNECT_TIMEOUT.
+                if manager.connect_timeout > 0:
+                    env = dict()
+                    env['PGCONNECT_TIMEOUT'] = str(manager.connect_timeout)
+                    p.set_env_variables(server, env=env)
+                else:
+                    p.set_env_variables(server)
+
+                p.start()
+                jid = p.id
+            except Exception as e:
+                current_app.logger.exception(e)
+                return make_json_response(
+                    status=410,
+                    success=0,
+                    errormsg=str(e)
+                )
+            # Return response
             return make_json_response(
-                success=1,
-                info=gettext("View refreshed"),
                 data={
-                    'id': vid,
+                    'job_id': jid,
-                    'sid': sid,
+                    'status': True,
-                    'gid': gid,
+                    'info': gettext(
-                    'did': did
+                        'Materialized view refresh job created.')
                 }
             )
-
         except Exception as e:
             current_app.logger.exception(e)
             return internal_server_error(errormsg=str(e))
@@ -2073,6 +2200,39 @@ class MViewNode(ViewNode, VacuumSettings):
 
         return res
 
+    @check_precondition
+    def check_utility_exists(self, gid, sid, did, scid, vid):
+        """
+        This function checks the utility file exist on the given path.
+
+        Args:
+            sid: Server ID
+        Returns:
+            None
+        """
+        server = Server.query.filter_by(
+            id=sid, user_id=current_user.id
+        ).first()
+
+        if server is None:
+            return make_json_response(
+                success=0,
+                errormsg=gettext("Could not find the specified server.")
+            )
+
+        driver = get_driver(PG_DEFAULT_DRIVER)
+        manager = driver.connection_manager(server.id)
+
+        utility = manager.utility('sql')
+        ret_val = does_utility_exist(utility)
+        if ret_val:
+            return make_json_response(
+                success=0,
+                errormsg=ret_val
+            )
+
+        return make_json_response(success=1)
+
 
 SchemaDiffRegistry(view_blueprint.node_type, ViewNode)
 ViewNode.register_node_view(view_blueprint)

web/pgadmin/browser/server_groups/servers/databases/schemas/views/static/js/mview.js

@@ -275,34 +275,98 @@ define('pgadmin.node.mview', [
           obj = this,
           t = pgBrowser.tree,
           i = input.item || t.selected(),
-          d = i && i.length == 1 ? t.itemData(i) : undefined;
+          d = i && i.length == 1 ? t.itemData(i) : undefined,
+          server_data = null;
 
         if (!d)
           return false;
 
-        // Make ajax call to refresh mview data
+        while (i) {
-        $.ajax({
+          var node_data = pgBrowser.tree.itemData(i);
-          url: obj.generate_url(i, 'refresh_data' , d, true),
+          if (node_data._type == 'server') {
-          type: 'PUT',
+            server_data = node_data;
-          data: {'concurrent': args.concurrent, 'with_data': args.with_data},
+            break;
-          dataType: 'json',
+          }
-        })
-          .done(function(res) {
-            if (res.success == 1) {
-              Alertify.success(gettext('View refreshed successfully'));
-            }
-            else {
-              Alertify.alert(
-                gettext('Error refreshing view'),
-                res.data.result
-              );
-            }
-          })
-          .fail(function(xhr, status, error) {
-            Alertify.pgRespErrorNotify(xhr, error, gettext('Error refreshing view'));
-          });
 
+          if (pgBrowser.tree.hasParent(i)) {
+            i = $(pgBrowser.tree.parent(i));
+          } else {
+            Alertify.alert(gettext('Please select server or child node from tree.'));
+            break;
+          }
+        }
+
+        if (!server_data) {
+          return;
+        }
+
+        var module = 'paths',
+          preference_name = 'pg_bin_dir',
+          msg = gettext('Please configure the PostgreSQL Binary Path in the Preferences dialog.');
+
+        if ((server_data.type && server_data.type == 'ppas') ||
+          server_data.server_type == 'ppas') {
+          preference_name = 'ppas_bin_dir';
+          msg = gettext('Please configure the EDB Advanced Server Binary Path in the Preferences dialog.');
+        }
+
+        var preference = pgBrowser.get_preference(module, preference_name);
+
+        if (preference) {
+          if (!preference.value) {
+            Alertify.alert(gettext('Configuration required'), msg);
+            return;
+          }
+        } else {
+          Alertify.alert(gettext('Failed to load preference %s of module %s', preference_name, module));
+          return;
+        }
+
+        $.ajax({
+          url: obj.generate_url(i, 'check_utility_exists' , d, true),
+          type: 'GET',
+          dataType: 'json',
+        }).done(function(res) {
+          if (!res.success) {
+            Alertify.alert(
+              gettext('Utility not found'),
+              res.errormsg
+            );
+            return;
+          }
+          // Make ajax call to refresh mview data
+          $.ajax({
+            url: obj.generate_url(i, 'refresh_data' , d, true),
+            type: 'PUT',
+            data: {'concurrent': args.concurrent, 'with_data': args.with_data},
+            dataType: 'json',
+          })
+            .done(function(res) {
+              if (res.data && res.data.status) {
+              //Do nothing as we are creating the job and exiting from the main dialog
+                Alertify.success(res.data.info);
+                pgBrowser.Events.trigger('pgadmin-bgprocess:created', obj);
+              } else {
+                Alertify.alert(
+                  gettext('Failed to create materialized view refresh job.'),
+                  res.errormsg
+                );
+              }
+            })
+            .fail(function(xhr, status, error) {
+              Alertify.pgRespErrorNotify(
+                xhr, error, gettext('Failed to create materialized view refresh job.')
+              );
+            });
+        }).fail(function() {
+          Alertify.alert(
+            gettext('Utility not found'),
+            gettext('Failed to fetch Utility information')
+          );
+          return;
+        });
       },
+
       is_version_supported: function(data, item) {
         var t = pgAdmin.Browser.tree,
           i = item || t.selected(),

web/pgadmin/browser/server_groups/servers/databases/schemas/views/tests/test_mviews_refresh.py

@@ -0,0 +1,149 @@
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2020, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+import json
+import uuid
+
+from pgadmin.browser.server_groups.servers.databases.schemas.tests import \
+    utils as schema_utils
+from pgadmin.browser.server_groups.servers.databases.tests import utils as \
+    database_utils
+from pgadmin.utils import server_utils as server_utils
+from pgadmin.utils.route import BaseTestGenerator
+from regression import parent_node_dict
+from regression.python_test_utils import test_utils as utils
+from . import utils as views_utils
+
+MVIEW_CHECK_UTILITY_URL = 'browser/mview/check_utility_exists/'
+MVIEW_REFRESH_URL = 'browser/mview/refresh_data/'
+IS_UTILITY_EXISTS = True
+
+
+class MViewsUpdateParameterTestCase(BaseTestGenerator):
+    """This class will check materialized view refresh functionality."""
+    scenarios = [
+        ('Check utility route',
+         dict(type='check_utility')
+         ),
+        ('Refresh materialized view with invalid oid',
+         dict(type='invalid')
+         ),
+        ('Refresh materialized view with data',
+         dict(type='with_data')
+         ),
+        ('Refresh materialized view with no data',
+         dict(type='with_no_data')
+         ),
+        ('Refresh materialized view with data (concurrently)',
+         dict(type='with_data_concurrently')
+         ),
+        ('Refresh materialized view with no data (concurrently)',
+         dict(type='with_no_data_concurrently')
+         ),
+    ]
+
+    @classmethod
+    def setUpClass(self):
+        self.db_name = parent_node_dict["database"][-1]["db_name"]
+        schema_info = parent_node_dict["schema"][-1]
+        self.server_id = schema_info["server_id"]
+        self.db_id = schema_info["db_id"]
+        server_response = server_utils.connect_server(self, self.server_id)
+
+        if server_response["data"]["version"] < 90300 and "mview" in self.url:
+            message = "Materialized Views are not supported by PG9.2 " \
+                      "and PPAS9.2 and below."
+            self.skipTest(message)
+
+        db_con = database_utils.connect_database(self, utils.SERVER_GROUP,
+                                                 self.server_id, self.db_id)
+        if not db_con['data']["connected"]:
+            raise Exception("Could not connect to database to update a mview.")
+        self.schema_id = schema_info["schema_id"]
+        self.schema_name = schema_info["schema_name"]
+        schema_response = schema_utils.verify_schemas(self.server,
+                                                      self.db_name,
+                                                      self.schema_name)
+        if not schema_response:
+            raise Exception("Could not find the schema to update a mview.")
+
+        self.m_view_name = "test_mview_put_%s" % (str(uuid.uuid4())[1:8])
+        m_view_sql = "CREATE MATERIALIZED VIEW %s.%s TABLESPACE pg_default " \
+                     "AS SELECT 'test_pgadmin' WITH NO DATA;ALTER TABLE " \
+                     "%s.%s OWNER TO %s"
+
+        self.m_view_id = views_utils.create_view(self.server,
+                                                 self.db_name,
+                                                 self.schema_name,
+                                                 m_view_sql,
+                                                 self.m_view_name)
+
+    def runTest(self):
+        """This class will check materialized view refresh functionality"""
+
+        mview_response = views_utils.verify_view(self.server, self.db_name,
+                                                 self.m_view_name)
+        if not mview_response:
+            raise Exception("Could not find the mview to update.")
+
+        data = None
+        is_put_request = True
+
+        if self.type == 'check_utility':
+            is_put_request = False
+        elif self.type == 'invalid':
+            data = dict({'concurrent': 'false', 'with_data': 'false'})
+        elif self.type == 'with_data':
+            data = dict({'concurrent': 'false', 'with_data': 'true'})
+        elif self.type == 'with_no_data':
+            data = dict({'concurrent': 'false', 'with_data': 'false'})
+        elif self.type == 'with_data_concurrently':
+            data = dict({'concurrent': 'true', 'with_data': 'true'})
+        elif self.type == 'with_no_data_concurrently':
+            data = dict({'concurrent': 'true', 'with_data': 'false'})
+
+        response = self.tester.get(
+            MVIEW_CHECK_UTILITY_URL + str(utils.SERVER_GROUP) + '/' +
+            str(self.server_id) + '/' +
+            str(self.db_id) + '/' +
+            str(self.schema_id) + '/' +
+            str(self.m_view_id),
+            content_type='html/json'
+        )
+        self.assertEquals(response.status_code, 200)
+        if is_put_request and response.json['success'] == 0:
+            self.skipTest(
+                "Couldn't check materialized view refresh"
+                " functionality because utility/binary does not exists."
+            )
+
+        if is_put_request:
+            mvid = self.m_view_id
+            if self.type == 'invalid':
+                mvid = 99999
+            response = self.tester.put(
+                MVIEW_REFRESH_URL + str(utils.SERVER_GROUP) + '/' +
+                str(self.server_id) + '/' +
+                str(self.db_id) + '/' +
+                str(self.schema_id) + '/' +
+                str(mvid),
+                data=json.dumps(data),
+                follow_redirects=True
+            )
+            if self.type == 'invalid':
+                self.assertEquals(response.status_code, 410)
+            else:
+                self.assertEquals(response.status_code, 200)
+                # On success we get job_id from server
+                self.assertTrue('job_id' in response.json['data'])
+
+    @classmethod
+    def tearDownClass(self):
+        # Disconnect the database
+        database_utils.disconnect_database(self, self.server_id, self.db_id)