From 7e3bd8b88a21fe8ce33b29cf64abfb3aa7bae185 Mon Sep 17 00:00:00 2001 From: Khushboo Vashi Date: Tue, 4 Dec 2018 17:03:19 +0000 Subject: [PATCH] Fix an encoding issue in the query tool. Fixes #3711 --- .../sqleditor/tests/test_encoding_charset.py | 15 +++ .../utils/driver/psycopg2/connection.py | 23 ++--- web/pgadmin/utils/driver/psycopg2/encoding.py | 36 +++++++ web/pgadmin/utils/tests/test_encoding.py | 98 +++++++++++++++++++ 4 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 web/pgadmin/utils/driver/psycopg2/encoding.py create mode 100644 web/pgadmin/utils/tests/test_encoding.py diff --git a/web/pgadmin/tools/sqleditor/tests/test_encoding_charset.py b/web/pgadmin/tools/sqleditor/tests/test_encoding_charset.py index 8a8d45e3f..df410a0ef 100644 --- a/web/pgadmin/tools/sqleditor/tests/test_encoding_charset.py +++ b/web/pgadmin/tools/sqleditor/tests/test_encoding_charset.py @@ -51,6 +51,21 @@ class TestEncodingCharset(BaseTestGenerator): lc_collate='C', test_str='\\255' )), + ( + 'With Encoding LATIN1', + dict( + db_encoding='LATIN1', + lc_collate='C', + test_str='Ň' + )), + ( + 'With Encoding LATIN2', + dict( + db_encoding='LATIN2', + lc_collate='C', + test_str='§' + )), + ] def setUp(self): diff --git a/web/pgadmin/utils/driver/psycopg2/connection.py b/web/pgadmin/utils/driver/psycopg2/connection.py index 0e77de5e1..a94bc4693 100644 --- a/web/pgadmin/utils/driver/psycopg2/connection.py +++ b/web/pgadmin/utils/driver/psycopg2/connection.py @@ -36,7 +36,7 @@ from .cursor import DictCursor from .typecast import register_global_typecasters, \ register_string_typecasters, register_binary_typecasters, \ register_array_to_string_typecasters, ALL_JSON_TYPES - +from .encoding import getEncoding if sys.version_info < (3,): # Python2 in-built csv module do not handle unicode @@ -402,20 +402,13 @@ class Connection(BaseConnection): if self.use_binary_placeholder: register_binary_typecasters(self.conn) - if self.conn.encoding in ('SQL_ASCII', 'SQLASCII', - 'MULE_INTERNAL', 'MULEINTERNAL'): - status = _execute(cur, "SET DateStyle=ISO;" - "SET client_min_messages=notice;" - "SET bytea_output=escape;" - "SET client_encoding='{0}';" - .format(self.conn.encoding)) - self.python_encoding = 'raw_unicode_escape' - else: - status = _execute(cur, "SET DateStyle=ISO;" - "SET client_min_messages=notice;" - "SET bytea_output=escape;" - "SET client_encoding='UNICODE';") - self.python_encoding = 'utf-8' + postgres_encoding, self.python_encoding = getEncoding(self.conn.encoding) + + status = _execute(cur, "SET DateStyle=ISO; " + "SET client_min_messages=notice;" + "SET bytea_output=escape;" + "SET client_encoding='{0}';" + .format(postgres_encoding)) # Replace the python encoding for original name and renamed encodings # psycopg2 removes the underscore in conn.encoding diff --git a/web/pgadmin/utils/driver/psycopg2/encoding.py b/web/pgadmin/utils/driver/psycopg2/encoding.py new file mode 100644 index 000000000..4448edd68 --- /dev/null +++ b/web/pgadmin/utils/driver/psycopg2/encoding.py @@ -0,0 +1,36 @@ +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2018, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## + +# Get Postgres and Python encoding + + +def getEncoding(key): + """ + :param key: Database Encoding + :return: + [Postgres_encoding, Python_encoding] - Postgres and Python encoding + """ + encode_dict = { + 'SQL_ASCII': ['SQL_ASCII', 'raw_unicode_escape'], + 'SQLASCII': ['SQL_ASCII', 'raw_unicode_escape'], + 'MULE_INTERNAL': ['MULE_INTERNAL', 'raw_unicode_escape'], + 'MULEINTERNAL': ['MULEINTERNAL', 'raw_unicode_escape'], + 'LATIN1': ['LATIN1', 'latin1'], + 'LATIN2': ['LATIN2', 'latin2'], + 'LATIN3': ['LATIN3', 'latin3'], + 'LATIN4': ['LATIN4', 'latin4'], + 'LATIN5': ['LATIN5', 'latin5'], + 'LATIN6': ['LATIN6', 'latin6'], + 'LATIN7': ['LATIN7', 'latin7'], + 'LATIN8': ['LATIN8', 'latin8'], + 'LATIN9': ['LATIN9', 'latin9'], + 'LATIN10': ['LATIN10', 'latin10'] + } + + return encode_dict.get(key, ['UNICODE', 'utf-8']) diff --git a/web/pgadmin/utils/tests/test_encoding.py b/web/pgadmin/utils/tests/test_encoding.py new file mode 100644 index 000000000..28a4b9ee0 --- /dev/null +++ b/web/pgadmin/utils/tests/test_encoding.py @@ -0,0 +1,98 @@ +####################################################################### +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2018, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## +from pgadmin.utils.driver.psycopg2.encoding import getEncoding +from pgadmin.utils.route import BaseTestGenerator + + +class TestEncoding(BaseTestGenerator): + scenarios = [ + ( + 'When the database encoding is SQL_ASCII', + dict( + db_encoding='SQL_ASCII', + expected_return_value=['SQL_ASCII', 'raw_unicode_escape'] + ) + ), ( + 'When the database encoding is MULEINTERNAL', + dict( + db_encoding='MULEINTERNAL', + expected_return_value=['MULEINTERNAL', 'raw_unicode_escape'] + ) + ), ( + 'When the database encoding is LATIN1', + dict( + db_encoding='LATIN1', + expected_return_value=['LATIN1', 'latin1'] + ) + ), ( + 'When the database encoding is LATIN2', + dict( + db_encoding='LATIN2', + expected_return_value=['LATIN2', 'latin2'] + ) + ), ( + 'When the database encoding is LATIN3', + dict( + db_encoding='LATIN3', + expected_return_value=['LATIN3', 'latin3'] + ) + ), ( + 'When the database encoding is LATIN4', + dict( + db_encoding='LATIN4', + expected_return_value=['LATIN4', 'latin4'] + ) + ), ( + 'When the database encoding is LATIN5', + dict( + db_encoding='LATIN5', + expected_return_value=['LATIN5', 'latin5'] + ) + ), ( + 'When the database encoding is LATIN6', + dict( + db_encoding='LATIN6', + expected_return_value=['LATIN6', 'latin6'] + ) + ), ( + 'When the database encoding is LATIN7', + dict( + db_encoding='LATIN7', + expected_return_value=['LATIN7', 'latin7'] + ) + ), ( + 'When the database encoding is LATIN8', + dict( + db_encoding='LATIN8', + expected_return_value=['LATIN8', 'latin8'] + ) + ), ( + 'When the database encoding is LATIN9', + dict( + db_encoding='LATIN9', + expected_return_value=['LATIN9', 'latin9'] + ) + ), ( + 'When the database encoding is LATIN10', + dict( + db_encoding='LATIN10', + expected_return_value=['LATIN10', 'latin10'] + ) + ), ( + 'When the database encoding is WIN1258', + dict( + db_encoding='WIN1258', + expected_return_value=['UNICODE', 'utf-8'] + ) + ), + ] + + def runTest(self): + result = getEncoding(self.db_encoding) + self.assertEquals(result, self.expected_return_value)