pgadmin-org
/
pgadmin4
mirror of https://github.com/pgadmin-org/pgadmin4.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed an issue when the user performs refresh on a large size materialized view. Fixes #5213

pull/32/head
Murtuza Zabuawala 2020-04-27 17:30:23 +05:30 committed by Akshay Joshi
parent 0f6abcc7fa
commit 76eb3e9b67
4 changed files with 409 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en_US/release_notes_4_21.rst
View File

@ -60,6 +60,7 @@ Bug fixes
| `Issue #5157 <https://redmine.postgresql.org/issues/5157>`_ - Ensure that default sort order should be using the primary key in View/Edit data.
| `Issue #5180 <https://redmine.postgresql.org/issues/5180>`_ - Fixed an issue where the autovacuum_enabled parameter is added automatically in the RE-SQL when the table has been created using the WITH clause.
| `Issue #5210 <https://redmine.postgresql.org/issues/5210>`_ - Ensure that text larger than underlying field size should not be truncated automatically.
| `Issue #5213 <https://redmine.postgresql.org/issues/5213>`_ - Fixed an issue when the user performs refresh on a large size materialized view.
| `Issue #5227 <https://redmine.postgresql.org/issues/5227>`_ - Fixed an issue where user cannot be added if many users are already exists.
| `Issue #5268 <https://redmine.postgresql.org/issues/5268>`_ - Fixed generated SQL when any token in FTS Configuration or any option in FTS Dictionary is changed.
| `Issue #5270 <https://redmine.postgresql.org/issues/5270>`_ - Ensure that OID should be shown in properties for Synonyms.

188
web/pgadmin/browser/server_groups/servers/databases/schemas/views/__init__.py
View File

@ -16,7 +16,7 @@ from functools import wraps
import simplejson as json
from flask import render_template, request, jsonify, current_app
from flask_babelex import gettext
from flask_security import current_user
import pgadmin.browser.server_groups.servers.databases as databases
from config import PG_DEFAULT_DRIVER
from pgadmin.browser.server_groups.servers.databases.schemas.utils import \
@ -29,6 +29,9 @@ from pgadmin.utils.ajax import make_json_response, internal_server_error, \
from pgadmin.utils.driver import get_driver
from pgadmin.tools.schema_diff.node_registry import SchemaDiffRegistry
from pgadmin.tools.schema_diff.compare import SchemaDiffObjectCompare
from pgadmin.utils import html, does_utility_exist
from pgadmin.model import Server
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
"""
@ -128,6 +131,53 @@ class ViewModule(SchemaChildModule):
return snippets
class Message(IProcessDesc):
def __init__(self, _sid, _data, _query):
self.sid = _sid
self.data = _data
self.query = _query
@property
def message(self):
res = gettext("Refresh Materialized View")
opts = []
if not self.data['is_with_data']:
opts.append(gettext("With no data"))
else:
opts.append(gettext("With data"))
if self.data['is_concurrent']:
opts.append(gettext("Concurrently"))
return res + " ({0})".format(', '.join(str(x) for x in opts))
@property
def type_desc(self):
return gettext("Refresh Materialized View")
def details(self, cmd, args):
res = gettext("Refresh Materialized View ({0})")
opts = []
if not self.data['is_with_data']:
opts.append(gettext("WITH NO DATA"))
else:
opts.append(gettext("WITH DATA"))
if self.data['is_concurrent']:
opts.append(gettext("CONCURRENTLY"))
res = res.format(', '.join(str(x) for x in opts))
res = '<div>' + html.safe_str(res)
res += '</div><div class="py-1">'
res += gettext("Running Query:")
res += '<div class="pg-bg-cmd enable-selection p-1">'
res += html.safe_str(self.query)
res += '</div></div>'
return res
class MViewModule(ViewModule):
"""
class MViewModule(ViewModule)
@ -1504,7 +1554,8 @@ class ViewNode(PGChildNodeView, VacuumSettings, SchemaDiffObjectCompare):
# Override the operations for materialized view
mview_operations = {
'refresh_data': [{'put': 'refresh_data'}, {}]
'refresh_data': [{'put': 'refresh_data'}, {}],
'check_utility_exists': [{'get': 'check_utility_exists'}, {}]
}
mview_operations.update(ViewNode.operations)
@ -2010,7 +2061,9 @@ class MViewNode(ViewNode, VacuumSettings):
is_concurrent = json.loads(data['concurrent'])
with_data = json.loads(data['with_data'])
data = dict()
data['is_concurrent'] = is_concurrent
data['is_with_data'] = with_data
try:
# Fetch view name by view id
@ -2019,6 +2072,10 @@ class MViewNode(ViewNode, VacuumSettings):
status, res = self.conn.execute_dict(SQL)
if not status:
return internal_server_error(errormsg=res)
if len(res['rows']) == 0:
return gone(
gettext("""Could not find the materialized view.""")
)
# Refresh view
SQL = render_template(
@ -2028,21 +2085,91 @@ class MViewNode(ViewNode, VacuumSettings):
is_concurrent=is_concurrent,
with_data=with_data
)
status, res_data = self.conn.execute_dict(SQL)
if not status:
return internal_server_error(errormsg=res_data)
# Fetch the server details like hostname, port, roles etc
server = Server.query.filter_by(
id=sid).first()
if server is None:
return make_json_response(
success=1,
info=gettext("View refreshed"),
data={
'id': vid,
'sid': sid,
'gid': gid,
'did': did
}
success=0,
errormsg=gettext("Could not find the given server")
)
# To fetch MetaData for the server
driver = get_driver(PG_DEFAULT_DRIVER)
manager = driver.connection_manager(server.id)
conn = manager.connection()
connected = conn.connected()
if not connected:
return make_json_response(
success=0,
errormsg=gettext("Please connect to the server first.")
)
# Fetch the database name from connection manager
db_info = manager.db_info.get(did, None)
if db_info:
data['database'] = db_info['datname']
else:
return make_json_response(
success=0,
errormsg=gettext(
"Could not find the database on the server.")
)
utility = manager.utility('sql')
ret_val = does_utility_exist(utility)
if ret_val:
return make_json_response(
success=0,
errormsg=ret_val
)
args = [
'--host',
manager.local_bind_host if manager.use_ssh_tunnel
else server.host,
'--port',
str(manager.local_bind_port) if manager.use_ssh_tunnel
else str(server.port),
'--username', server.username, '--dbname',
data['database'],
'--command', SQL
]
try:
p = BatchProcess(
desc=Message(sid, data, SQL),
cmd=utility, args=args
)
manager.export_password_env(p.id)
# Check for connection timeout and if it is greater than 0
# then set the environment variable PGCONNECT_TIMEOUT.
if manager.connect_timeout > 0:
env = dict()
env['PGCONNECT_TIMEOUT'] = str(manager.connect_timeout)
p.set_env_variables(server, env=env)
else:
p.set_env_variables(server)
p.start()
jid = p.id
except Exception as e:
current_app.logger.exception(e)
return make_json_response(
status=410,
success=0,
errormsg=str(e)
)
# Return response
return make_json_response(
data={
'job_id': jid,
'status': True,
'info': gettext(
'Materialized view refresh job created.')
}
)
except Exception as e:
current_app.logger.exception(e)
return internal_server_error(errormsg=str(e))
@ -2073,6 +2200,39 @@ class MViewNode(ViewNode, VacuumSettings):
return res
@check_precondition
def check_utility_exists(self, gid, sid, did, scid, vid):
"""
This function checks the utility file exist on the given path.
Args:
sid: Server ID
Returns:
None
"""
server = Server.query.filter_by(
id=sid, user_id=current_user.id
).first()
if server is None:
return make_json_response(
success=0,
errormsg=gettext("Could not find the specified server.")
)
driver = get_driver(PG_DEFAULT_DRIVER)
manager = driver.connection_manager(server.id)
utility = manager.utility('sql')
ret_val = does_utility_exist(utility)
if ret_val:
return make_json_response(
success=0,
errormsg=ret_val
)
return make_json_response(success=1)
SchemaDiffRegistry(view_blueprint.node_type, ViewNode)
ViewNode.register_node_view(view_blueprint)

82
web/pgadmin/browser/server_groups/servers/databases/schemas/views/static/js/mview.js
View File

@ -275,11 +275,65 @@ define('pgadmin.node.mview', [
obj = this,
t = pgBrowser.tree,
i = input.item || t.selected(),
d = i && i.length == 1 ? t.itemData(i) : undefined;
d = i && i.length == 1 ? t.itemData(i) : undefined,
server_data = null;
if (!d)
return false;
while (i) {
var node_data = pgBrowser.tree.itemData(i);
if (node_data._type == 'server') {
server_data = node_data;
break;
}
if (pgBrowser.tree.hasParent(i)) {
i = $(pgBrowser.tree.parent(i));
} else {
Alertify.alert(gettext('Please select server or child node from tree.'));
break;
}
}
if (!server_data) {
return;
}
var module = 'paths',
preference_name = 'pg_bin_dir',
msg = gettext('Please configure the PostgreSQL Binary Path in the Preferences dialog.');
if ((server_data.type && server_data.type == 'ppas') ||
server_data.server_type == 'ppas') {
preference_name = 'ppas_bin_dir';
msg = gettext('Please configure the EDB Advanced Server Binary Path in the Preferences dialog.');
}
var preference = pgBrowser.get_preference(module, preference_name);
if (preference) {
if (!preference.value) {
Alertify.alert(gettext('Configuration required'), msg);
return;
}
} else {
Alertify.alert(gettext('Failed to load preference %s of module %s', preference_name, module));
return;
}
$.ajax({
url: obj.generate_url(i, 'check_utility_exists' , d, true),
type: 'GET',
dataType: 'json',
}).done(function(res) {
if (!res.success) {
Alertify.alert(
gettext('Utility not found'),
res.errormsg
);
return;
}
// Make ajax call to refresh mview data
$.ajax({
url: obj.generate_url(i, 'refresh_data' , d, true),
@ -288,21 +342,31 @@ define('pgadmin.node.mview', [
dataType: 'json',
})
.done(function(res) {
if (res.success == 1) {
Alertify.success(gettext('View refreshed successfully'));
}
else {
if (res.data && res.data.status) {
//Do nothing as we are creating the job and exiting from the main dialog
Alertify.success(res.data.info);
pgBrowser.Events.trigger('pgadmin-bgprocess:created', obj);
} else {
Alertify.alert(
gettext('Error refreshing view'),
res.data.result
gettext('Failed to create materialized view refresh job.'),
res.errormsg
);
}
})
.fail(function(xhr, status, error) {
Alertify.pgRespErrorNotify(xhr, error, gettext('Error refreshing view'));
Alertify.pgRespErrorNotify(
xhr, error, gettext('Failed to create materialized view refresh job.')
);
});
}).fail(function() {
Alertify.alert(
gettext('Utility not found'),
gettext('Failed to fetch Utility information')
);
return;
});
},
is_version_supported: function(data, item) {
var t = pgAdmin.Browser.tree,
i = item || t.selected(),

149
web/pgadmin/browser/server_groups/servers/databases/schemas/views/tests/test_mviews_refresh.py Normal file
View File

@ -0,0 +1,149 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2020, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
import json
import uuid
from pgadmin.browser.server_groups.servers.databases.schemas.tests import \
utils as schema_utils
from pgadmin.browser.server_groups.servers.databases.tests import utils as \
database_utils
from pgadmin.utils import server_utils as server_utils
from pgadmin.utils.route import BaseTestGenerator
from regression import parent_node_dict
from regression.python_test_utils import test_utils as utils
from . import utils as views_utils
MVIEW_CHECK_UTILITY_URL = 'browser/mview/check_utility_exists/'
MVIEW_REFRESH_URL = 'browser/mview/refresh_data/'
IS_UTILITY_EXISTS = True
class MViewsUpdateParameterTestCase(BaseTestGenerator):
"""This class will check materialized view refresh functionality."""
scenarios = [
('Check utility route',
dict(type='check_utility')
),
('Refresh materialized view with invalid oid',
dict(type='invalid')
),
('Refresh materialized view with data',
dict(type='with_data')
),
('Refresh materialized view with no data',
dict(type='with_no_data')
),
('Refresh materialized view with data (concurrently)',
dict(type='with_data_concurrently')
),
('Refresh materialized view with no data (concurrently)',
dict(type='with_no_data_concurrently')
),
]
@classmethod
def setUpClass(self):
self.db_name = parent_node_dict["database"][-1]["db_name"]
schema_info = parent_node_dict["schema"][-1]
self.server_id = schema_info["server_id"]
self.db_id = schema_info["db_id"]
server_response = server_utils.connect_server(self, self.server_id)
if server_response["data"]["version"] < 90300 and "mview" in self.url:
message = "Materialized Views are not supported by PG9.2 " \
"and PPAS9.2 and below."
self.skipTest(message)
db_con = database_utils.connect_database(self, utils.SERVER_GROUP,
self.server_id, self.db_id)
if not db_con['data']["connected"]:
raise Exception("Could not connect to database to update a mview.")
self.schema_id = schema_info["schema_id"]
self.schema_name = schema_info["schema_name"]
schema_response = schema_utils.verify_schemas(self.server,
self.db_name,
self.schema_name)
if not schema_response:
raise Exception("Could not find the schema to update a mview.")
self.m_view_name = "test_mview_put_%s" % (str(uuid.uuid4())[1:8])
m_view_sql = "CREATE MATERIALIZED VIEW %s.%s TABLESPACE pg_default " \
"AS SELECT 'test_pgadmin' WITH NO DATA;ALTER TABLE " \
"%s.%s OWNER TO %s"
self.m_view_id = views_utils.create_view(self.server,
self.db_name,
self.schema_name,
m_view_sql,
self.m_view_name)
def runTest(self):
"""This class will check materialized view refresh functionality"""
mview_response = views_utils.verify_view(self.server, self.db_name,
self.m_view_name)
if not mview_response:
raise Exception("Could not find the mview to update.")
data = None
is_put_request = True
if self.type == 'check_utility':
is_put_request = False
elif self.type == 'invalid':
data = dict({'concurrent': 'false', 'with_data': 'false'})
elif self.type == 'with_data':
data = dict({'concurrent': 'false', 'with_data': 'true'})
elif self.type == 'with_no_data':
data = dict({'concurrent': 'false', 'with_data': 'false'})
elif self.type == 'with_data_concurrently':
data = dict({'concurrent': 'true', 'with_data': 'true'})
elif self.type == 'with_no_data_concurrently':
data = dict({'concurrent': 'true', 'with_data': 'false'})
response = self.tester.get(
MVIEW_CHECK_UTILITY_URL + str(utils.SERVER_GROUP) + '/' +
str(self.server_id) + '/' +
str(self.db_id) + '/' +
str(self.schema_id) + '/' +
str(self.m_view_id),
content_type='html/json'
)
self.assertEquals(response.status_code, 200)
if is_put_request and response.json['success'] == 0:
self.skipTest(
"Couldn't check materialized view refresh"
" functionality because utility/binary does not exists."
)
if is_put_request:
mvid = self.m_view_id
if self.type == 'invalid':
mvid = 99999
response = self.tester.put(
MVIEW_REFRESH_URL + str(utils.SERVER_GROUP) + '/' +
str(self.server_id) + '/' +
str(self.db_id) + '/' +
str(self.schema_id) + '/' +
str(mvid),
data=json.dumps(data),
follow_redirects=True
)
if self.type == 'invalid':
self.assertEquals(response.status_code, 410)
else:
self.assertEquals(response.status_code, 200)
# On success we get job_id from server
self.assertTrue('job_id' in response.json['data'])
@classmethod
def tearDownClass(self):
# Disconnect the database
database_utils.disconnect_database(self, self.server_id, self.db_id)