diff --git a/docs/en_US/cloud_aws_rds.rst b/docs/en_US/cloud_aws_rds.rst index 0cb6ad275..841c8d733 100644 --- a/docs/en_US/cloud_aws_rds.rst +++ b/docs/en_US/cloud_aws_rds.rst @@ -1,12 +1,12 @@ .. _cloud_aws_rds: ****************************************** -`Amazon AWS RDS Cloud Deployment`:index: +`Amazon RDS Cloud Deployment`:index: ****************************************** -To deploy a PostgreSQL server on the Amazon AWS cloud, follow the below steps. +To deploy a PostgreSQL server on the Amazon cloud, follow the below steps. -.. image:: images/cloud_aws_provider.png +.. image:: images/cloud_provider_for_postgresql.png :alt: Cloud Deployment Provider :align: center diff --git a/docs/en_US/cloud_azure_postgresql.rst b/docs/en_US/cloud_azure_database.rst similarity index 93% rename from docs/en_US/cloud_azure_postgresql.rst rename to docs/en_US/cloud_azure_database.rst index 32c7e5e42..0f06ce55f 100644 --- a/docs/en_US/cloud_azure_postgresql.rst +++ b/docs/en_US/cloud_azure_database.rst @@ -1,16 +1,16 @@ -.. _cloud_azure_postgresql: +.. cloud_azure_database: ****************************************** -`Azure PostgreSQL Cloud Deployment`:index: +`Azure Database Cloud Deployment`:index: ****************************************** -To deploy a PostgreSQL server on the Azure cloud, follow the below steps. +To deploy a PostgreSQL server on the Azure Database, follow the below steps. -.. image:: images/cloud_azure_provider.png +.. image:: images/cloud_provider_for_postgresql.png :alt: Cloud Deployment :align: center -Once you launch the tool, select the Azure PostgreSQL option. +Once you launch the tool, select the Azure Database option. Click on the *Next* button to proceed further. @@ -103,7 +103,7 @@ Click on the next button to proceed. :align: center At the end, review the instance details that you provided. Click on Finish -button to deploy the instance on Azure PostgreSQL. +button to deploy the instance on Azure Database. Once you click on the finish, one background process will start which will deploy the instance in the cloud and monitor the progress of the deployment. diff --git a/docs/en_US/cloud_deployment.rst b/docs/en_US/cloud_deployment.rst index e75f2c01d..05b9304c5 100644 --- a/docs/en_US/cloud_deployment.rst +++ b/docs/en_US/cloud_deployment.rst @@ -1,11 +1,12 @@ .. _cloud_deployment: -****************************** -`Cloud Deployment`:index: -****************************** +************************************* +`PostgreSQL Cloud Deployment`:index: +************************************* -A PostgreSQL server can be deployed on the Amazon AWS and EDB BigAnimal -cloud using this module. In future more cloud options will be available. +A PostgreSQL server can be deployed on the Amazon, EDB BigAnimal, Azure, +Google cloud using this module. In future more cloud provider options will +be available. To launch the *Cloud Deployment...* tool, right click on the *Server Group* or *Server* of the tree control, and select *Deploy a Cloud Instance* from the @@ -16,4 +17,5 @@ To launch the *Cloud Deployment...* tool, right click on the *Server Group* or cloud_aws_rds cloud_edb_biganimal - cloud_azure_postgresql \ No newline at end of file + cloud_azure_database + cloud_google_cloud_sql diff --git a/docs/en_US/cloud_edb_biganimal.rst b/docs/en_US/cloud_edb_biganimal.rst index bd0e93093..0296ed786 100644 --- a/docs/en_US/cloud_edb_biganimal.rst +++ b/docs/en_US/cloud_edb_biganimal.rst @@ -6,7 +6,7 @@ To deploy a PostgreSQL server on the EDB BigAnimal cloud, follow the below steps. -.. image:: images/cloud_biganimal_provider.png +.. image:: images/cloud_provider_for_postgresql.png :alt: Cloud Deployment Provider :align: center diff --git a/docs/en_US/cloud_google_cloud_sql.rst b/docs/en_US/cloud_google_cloud_sql.rst new file mode 100644 index 000000000..a51eeb4bb --- /dev/null +++ b/docs/en_US/cloud_google_cloud_sql.rst @@ -0,0 +1,117 @@ +.. cloud_google_cloud_sql: + +************************************************ +`Google Cloud SQL Deployment`:index: +************************************************ + +To deploy a PostgreSQL server on the Google Cloud SQL, follow the below steps. + +.. image:: images/cloud_provider_for_postgresql.png + :alt: Cloud Deployment + :align: center + +Once you launch the tool, select the Google Cloud SQL option. +Click on the *Next* button to proceed further. + + +.. image:: images/cloud_google_credentials.png + :alt: Cloud Deployment + :align: center + +In the Credentials dialog, select client secret file to authenticate +using google.You can download a client secret which is json formatted file +from google cloud console once OAuth2 client ID is created. + +.. note:: While creating client OAuth client ID, select Desktop App as application type. + Refer `this `_ link for creating client secret. + +Clicking the *Click here to authenticate yourself to Google* +button, user will be redirected to the Google authentication page in a +new browser tab. + +Once authentication is completed, click on the next button to proceed. + +.. image:: images/cloud_google_instance.png + :alt: Cloud Deployment + :align: center + +Use the fields from the Instance Specification tab to specify the Instance +details. + +* Use the *Cluster name* field to add a name for the PostgreSQL + server; the name specified will be displayed in the *Browser* tree control too. + +* Select the project from *project* dropdown under which the + PostgreSQL instance will be created. + +* Select the location to deploy PostgreSQL instance from *Location* + options. + +* Select the availability zone in specified region to deploy PostgreSQL + instance from *Availability zone* options. + +* Use *Database version* options to specify PostgreSQL database version. + +* Use the *Instance class* field to allocate the computational and + memory capacity required by planned workload of this DB instance. + +* Use the *Instance type* field to select the instance type. + +* Specify storage type by selecting option from *Storage type*. + +* Use the *Storage capacity* option to specify the storage capacity. + +.. image:: images/cloud_google_network.png + :alt: Cloud Deployment + :align: center + +* Use the *Public IP* field to specify the list of IP address range + for allowed inbound traffic, for example: 127.0.0.1/32. Add multiple + IP addresses/ranges separated with commas. + +* Use the *High Availability* option to specify High Availability + option. This option creates a standby in a select Secondary + Availability Zone. + +* Select the secondary availability zone for high availability + from *Secondary Availability zone* options. + +.. image:: images/cloud_google_database.png + :alt: Cloud Deployment + :align: center + +Use the fields from the Database Details tab to specify the PostgreSQL database details. + +* Use the drop-down list in the *pgAdmin server group* field to select the parent + node for the server; the server will be displayed in the *Browser* tree + control within the specified group. + +* Admin username field will be default to postgres. + server. + +* Use the *Password* field to provide a password that will be supplied when + authenticating with the server. + +* Use the *Confirm password* field to repeat the password. + +Click on the next button to proceed. + +.. image:: images/cloud_google_review.png + :alt: Cloud Deployment + :align: center + +At the end, review the instance details that you provided. Click on Finish +button to deploy the instance on Azure PostgreSQL. + +Once you click on the finish, one background process will start which will +deploy the instance in the cloud and monitor the progress of the deployment. +You can view all the background process with there running status and logs +on the :ref:`Processes ` tab + + +The Server will be added to the tree with the cloud deployment icon. Once the +deployment is done, the server details will be updated. + +.. image:: images/cloud_google_deployment_tree.png + :alt: Cloud Deployment Provider + :align: center diff --git a/docs/en_US/images/cloud_aws_provider.png b/docs/en_US/images/cloud_aws_provider.png deleted file mode 100644 index 79d3eefe4..000000000 Binary files a/docs/en_US/images/cloud_aws_provider.png and /dev/null differ diff --git a/docs/en_US/images/cloud_azure_provider.png b/docs/en_US/images/cloud_azure_provider.png deleted file mode 100644 index 9e1b91e05..000000000 Binary files a/docs/en_US/images/cloud_azure_provider.png and /dev/null differ diff --git a/docs/en_US/images/cloud_biganimal_provider.png b/docs/en_US/images/cloud_biganimal_provider.png deleted file mode 100644 index 6ad09e8ca..000000000 Binary files a/docs/en_US/images/cloud_biganimal_provider.png and /dev/null differ diff --git a/docs/en_US/images/cloud_google_credentials.png b/docs/en_US/images/cloud_google_credentials.png new file mode 100644 index 000000000..c31b8b071 Binary files /dev/null and b/docs/en_US/images/cloud_google_credentials.png differ diff --git a/docs/en_US/images/cloud_google_database.png b/docs/en_US/images/cloud_google_database.png new file mode 100644 index 000000000..9f26d0bc8 Binary files /dev/null and b/docs/en_US/images/cloud_google_database.png differ diff --git a/docs/en_US/images/cloud_google_deployment_tree.png b/docs/en_US/images/cloud_google_deployment_tree.png new file mode 100644 index 000000000..201af6404 Binary files /dev/null and b/docs/en_US/images/cloud_google_deployment_tree.png differ diff --git a/docs/en_US/images/cloud_google_instance.png b/docs/en_US/images/cloud_google_instance.png new file mode 100644 index 000000000..e26064f18 Binary files /dev/null and b/docs/en_US/images/cloud_google_instance.png differ diff --git a/docs/en_US/images/cloud_google_network.png b/docs/en_US/images/cloud_google_network.png new file mode 100644 index 000000000..d9cc57afa Binary files /dev/null and b/docs/en_US/images/cloud_google_network.png differ diff --git a/docs/en_US/images/cloud_google_review.png b/docs/en_US/images/cloud_google_review.png new file mode 100644 index 000000000..0e46e374f Binary files /dev/null and b/docs/en_US/images/cloud_google_review.png differ diff --git a/docs/en_US/images/cloud_provider_for_postgresql.png b/docs/en_US/images/cloud_provider_for_postgresql.png new file mode 100644 index 000000000..0f245db27 Binary files /dev/null and b/docs/en_US/images/cloud_provider_for_postgresql.png differ diff --git a/requirements.txt b/requirements.txt index 1e5fe8295..2357482d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -62,3 +62,5 @@ azure-mgmt-rdbms==10.1.0 azure-mgmt-resource==21.0.0 azure-mgmt-subscription==3.0.0 azure-identity==1.9.0 +google-api-python-client==2.* +google-auth-oauthlib==1.0.0 diff --git a/web/pgacloud/providers/google.py b/web/pgacloud/providers/google.py new file mode 100644 index 000000000..349dde8fc --- /dev/null +++ b/web/pgacloud/providers/google.py @@ -0,0 +1,223 @@ +# ########################################################################## +# # +# # pgAdmin 4 - PostgreSQL Tools +# # +# # Copyright (C) 2013 - 2023, The pgAdmin Development Team +# # This software is released under the PostgreSQL Licence +# # +# ########################################################################## +import json +import os +import time + +from utils.io import debug, error, output +from utils.misc import get_my_ip, get_random_id +from providers._abstract import AbsProvider + +from googleapiclient import discovery +from googleapiclient.errors import HttpError +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials + + +class GoogleProvider(AbsProvider): + def __init__(self): + self._credentials_json = None + self._credentials = None + self._cloud_resource_manager_api_version = 'v1' + self._sqladmin_api_version = 'v1' + self._compute_api_version = 'v1' + self._scopes = ['https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/sqlservice.admin'] + self._database_password = None + + # Get credentials from environment + if 'GOOGLE_CREDENTIALS' in os.environ: + self._credentials_json = \ + json.loads(os.environ['GOOGLE_CREDENTIALS']) + + if 'GOOGLE_DATABASE_PASSWORD' in os.environ: + self._database_password = os.environ['GOOGLE_DATABASE_PASSWORD'] + + def init_args(self, parsers): + """ Create the command line parser for this provider """ + self.parser = parsers.add_parser('google', + help='Google Cloud PostgreSQL', + epilog='Credentials are read from ' + 'the environment.') + + # Create the command sub-parser + parsers = self.parser.add_subparsers(help='Google commands', + dest='command') + + # Create the create instance command parser + parser_create_instance = parsers.add_parser('create-instance', + help='create a new ' + 'instance') + + parser_create_instance.add_argument('--region', help='name of the ' + 'Google region') + + parser_create_instance.add_argument('--project', required=True, + help='name of the project in which' + ' instance to be created') + + parser_create_instance.add_argument('--name', required=True, + help='name of the instance') + + parser_create_instance.add_argument('--db-password', required=False, + help='password for the database') + + parser_create_instance.add_argument('--db-version', required=False, + default='POSTGRES_14', + help='version of PostgreSQL to ' + 'deploy (default: POSTGRES_14)') + + parser_create_instance.add_argument('--instance-type', required=True, + help='machine type for the ' + 'instance nodes, e.g. ' + 'db-f1-micro') + + parser_create_instance.add_argument('--storage-size', type=int, + required=True, + help='storage size in GB') + + parser_create_instance.add_argument('--storage-type', default='PD_SSD', + help='storage type for the data ' + 'database (default: SSD)') + + parser_create_instance.add_argument('--high-availability', + default=False) + + parser_create_instance.add_argument('--public-ip', default='127.0.0.1', + help='Public IP ' + '(default: 127.0.0.1)') + + parser_create_instance.add_argument('--availability-zone', + help='name of the availability ' + 'zone') + + parser_create_instance.add_argument('--secondary-availability-zone', + help='name of the secondary ' + 'availability zone') + + ########################################################################## + # Google Helper functions + ########################################################################## + def _get_credentials(self, scopes): + self._credentials = Credentials.from_authorized_user_info( + self._credentials_json, scopes) + if not self._credentials or not self._credentials.valid: + if self._credentials and self._credentials.expired and \ + self._credentials.refresh_token and \ + self._credentials.has_scopes(scopes): + self._credentials.refresh(Request()) + return self._credentials + else: + from google_auth_oauthlib.flow import InstalledAppFlow + flow = InstalledAppFlow.from_client_config(self._client_config, + scopes) + self._credentials = flow.run_local_server() + return self._credentials + + @staticmethod + def get_authorized_network_list(ip): + authorized_networks = [] + ip = ip.split(',') + for i in ip: + authorized_networks.append({ + 'value': i, + 'name': 'pgcloud client {}'.format(i), + 'kind': 'sql#aclEntry' + }) + return authorized_networks + + def _create_google_postgresql_instance(self, args): + credentials = self._get_credentials(self._scopes) + service = discovery.build('sqladmin', 'v1beta4', + credentials=credentials) + high_availability = \ + 'REGIONAL' if eval(args.high_availability) else 'ZONAL' + + db_password = self._database_password \ + if self._database_password is not None else args.db_password + + ip = args.public_ip if args.public_ip else '{}/32'.format(get_my_ip()) + authorized_networks = self.get_authorized_network_list(ip) + + database_instance_body = { + 'databaseVersion': args.db_version, + 'instanceType': 'CLOUD_SQL_INSTANCE', + 'project': args.project, + 'name': args.name, + 'region': args.region, + 'gceZone': args.availability_zone, + 'secondaryGceZone': args.secondary_availability_zone, + "rootPassword": db_password, + 'settings': { + 'tier': args.instance_type, + 'availabilityType': high_availability, + 'dataDiskType': args.storage_type, + 'dataDiskSizeGb': args.storage_size, + 'ipConfiguration': { + "authorizedNetworks": authorized_networks, + 'ipv4Enabled': True + }, + } + } + operation = None + try: + debug('Creating Google instance: {}...'.format(args.name)) + req = service.instances().insert(project=args.project, + body=database_instance_body) + res = req.execute() + operation = res['name'] + + except HttpError as err: + if err.status_code == 409: + error('Google SQL instance {} already exists.'. + format(args.name)) + else: + error(str(err)) + except Exception as e: + error(str(e)) + + # Wait for completion + instance_provisioning = True + log_msg = 10000 + while instance_provisioning: + req = service.operations().get(project=args.project, + operation=operation) + res = req.execute() + status = res['status'] + if status != 'PENDING' and status != 'RUNNING': + instance_provisioning = False + else: + time.sleep(5) + log_msg -= 1 + + if log_msg % 15 == 0: + debug('Creating Google instance: {}...'.format(args.name)) + + req = service.instances().get(project=args.project, instance=args.name) + res = req.execute() + return res + ########################################################################## + # User commands + ########################################################################## + + def cmd_create_instance(self, args): + """ Create an Google instance""" + instance_data = self._create_google_postgresql_instance(args) + data = {'instance': { + 'Hostname': instance_data['ipAddresses'][0]['ipAddress'], + 'Port': 5432, + 'Database': 'postgres', + 'Username': 'postgres', + }} + output(data) + + +def load(): + """ Loads the current provider """ + return GoogleProvider() diff --git a/web/pgadmin/misc/cloud/__init__.py b/web/pgadmin/misc/cloud/__init__.py index a33d1cf91..64d5aeed5 100644 --- a/web/pgadmin/misc/cloud/__init__.py +++ b/web/pgadmin/misc/cloud/__init__.py @@ -27,6 +27,7 @@ from pgadmin.misc.cloud.biganimal import deploy_on_biganimal,\ clear_biganimal_session from pgadmin.misc.cloud.rds import deploy_on_rds, clear_aws_session from pgadmin.misc.cloud.azure import deploy_on_azure, clear_azure_session +from pgadmin.misc.cloud.google import clear_google_session, deploy_on_google import config # set template path for sql scripts @@ -78,6 +79,9 @@ class CloudModule(PgAdminModule): from .rds import blueprint as module app.register_blueprint(module) + from .google import blueprint as module + app.register_blueprint(module) + # Create blueprint for CloudModule class blueprint = CloudModule( @@ -135,6 +139,8 @@ def deploy_on_cloud(): status, p, resp = deploy_on_biganimal(data) elif data['cloud'] == 'azure': status, p, resp = deploy_on_azure(data) + elif data['cloud'] == 'google': + status, p, resp = deploy_on_google(data) else: status = False resp = gettext('No cloud implementation.') @@ -214,6 +220,7 @@ def clear_cloud_session(pid=None): clear_aws_session() clear_biganimal_session() clear_azure_session(pid) + clear_google_session() @blueprint.route( diff --git a/web/pgadmin/misc/cloud/google/__init__.py b/web/pgadmin/misc/cloud/google/__init__.py new file mode 100644 index 000000000..162f58d46 --- /dev/null +++ b/web/pgadmin/misc/cloud/google/__init__.py @@ -0,0 +1,532 @@ +# ########################################################################## +# # +# # pgAdmin 4 - PostgreSQL Tools +# # +# # Copyright (C) 2013 - 2023, The pgAdmin Development Team +# # This software is released under the PostgreSQL Licence +# # +# ########################################################################## + +# Google Cloud Deployment Implementation +import pickle +import json +import os +from pathlib import Path + +from oauthlib.oauth2 import AccessDeniedError + +from config import root +from pgadmin.utils.csrf import pgCSRFProtect +from pgadmin import make_json_response +from pgadmin.utils.ajax import plain_text_response +from pgadmin.misc.bgprocess import BatchProcess +from pgadmin.misc.cloud.utils import _create_server, CloudProcessDesc +from pgadmin.utils import PgAdminModule + +from flask_security import login_required +from flask import session, current_app, request + +from googleapiclient import discovery +from googleapiclient.errors import HttpError +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import Request + +MODULE_NAME = 'google' +os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' # Required for Oauth2 + + +class GooglePostgresqlModule(PgAdminModule): + """Cloud module to deploy on Google Cloud""" + + def get_own_stylesheets(self): + """ + Returns: + list: the stylesheets used by this module. + """ + stylesheets = [] + return stylesheets + + def get_exposed_url_endpoints(self): + return ['google.verify_credentials', + 'google.projects', + 'google.regions', + 'google.database_versions', + 'google.instance_types', + 'google.availability_zones', + 'google.verification_ack', + 'google.callback'] + + +blueprint = GooglePostgresqlModule(MODULE_NAME, __name__, + static_url_path='/misc/cloud/google') + + +@blueprint.route('/verify_credentials/', + methods=['POST'], endpoint='verify_credentials') +@login_required +def verify_credentials(): + """ + Initiate process of authorisation for google oauth2 + """ + data = json.loads(request.data) + client_secret_path = data['secret']['client_secret_file'] if \ + 'client_secret_file' in data['secret'] else None + status = True + error = None + res_data = {} + + if client_secret_path is not None and Path(client_secret_path).exists(): + with open(client_secret_path, 'r') as json_file: + client_config = json.load(json_file) + + if 'google' not in session: + session['google'] = {} + + if 'google_obj' not in session['google'] or \ + session['google']['client_config'] != client_config: + _google = Google(client_config) + else: + _google = pickle.loads(session['google']['google_obj']) + + # get auth url + auth_url, error_msg = _google.get_auth_url(request.host_url) + if error_msg: + status = False + error = error_msg + else: + res_data = {'auth_url': auth_url} + # save google object + session['google']['client_config'] = client_config + session['google']['google_obj'] = pickle.dumps(_google, -1) + else: + status = False + error = 'Client secret path not found' + + return make_json_response(success=status, errormsg=error, data=res_data) + + +@blueprint.route('/callback', + methods=['GET'], endpoint='callback') +@pgCSRFProtect.exempt +@login_required +def callback(): + """ + Call back function on google authentication response. + :return: + """ + google_obj = pickle.loads(session['google']['google_obj']) + res = google_obj.callback(request) + session['google']['google_obj'] = pickle.dumps(google_obj, -1) + return plain_text_response(res) + + +@blueprint.route('/verification_ack', + methods=['GET'], endpoint='verification_ack') +@login_required +def verification_ack(): + """ + Checks for google oauth2 authorisation confirmation + :return: + """ + verified = False + if 'google' in session and 'google_obj' in session['google']: + google_obj = pickle.loads(session['google']['google_obj']) + verified, error = google_obj.verification_ack() + session['google']['google_obj'] = pickle.dumps(google_obj, -1) + return make_json_response(success=verified, errormsg=error) + else: + return make_json_response(success=verified, + errormsg='Authentication is failed.') + + +@blueprint.route('/projects/', + methods=['GET'], endpoint='projects') +@login_required +def get_projects(): + """ + Lists the projects for authorized user + :return: list of projects + """ + if 'google' in session and 'google_obj' in session['google']: + google_obj = pickle.loads(session['google']['google_obj']) + projects_list = google_obj.get_projects() + return make_json_response(data=projects_list) + + +@blueprint.route('/regions/', + methods=['GET'], endpoint='regions') +@login_required +def get_regions(project_id): + """ + Lists regions based on project for authorized user + :param project_id: google project id + :return: google cloud sql region list + """ + if 'google' in session and 'google_obj' in session['google'] \ + and project_id: + google_obj = pickle.loads(session['google']['google_obj']) + regions_list = google_obj.get_regions(project_id) + session['google']['google_obj'] = pickle.dumps(google_obj, -1) + return make_json_response(data=regions_list) + else: + return make_json_response(data=[]) + + +@blueprint.route('/availability_zones/', + methods=['GET'], endpoint='availability_zones') +@login_required +def get_availability_zones(region): + """ + List availability zones for specified region + :param region: google region + :return: google cloud sql availability zone list + """ + if 'google' in session and 'google_obj' in session['google'] and region: + google_obj = pickle.loads(session['google']['google_obj']) + availability_zone_list = google_obj.get_availability_zones(region) + return make_json_response(data=availability_zone_list) + else: + return make_json_response(data=[]) + + +@blueprint.route('/instance_types///', + methods=['GET'], endpoint='instance_types') +@login_required +def get_instance_types(project_id, region, instance_class): + """ + List the instances types for specified google project, region & + instance type + :param project_id: google project id + :param region: google cloud region + :param instance_class: google cloud sql instnace class + :return: + """ + if 'google' in session and 'google_obj' in session['google'] and \ + project_id and region: + google_obj = pickle.loads(session['google']['google_obj']) + instance_types_dict = google_obj.get_instance_types( + project_id, region) + instance_types_list = instance_types_dict.get(instance_class, []) + return make_json_response(data=instance_types_list) + else: + return make_json_response(data=[]) + + +@blueprint.route('/database_versions/', + methods=['GET'], endpoint='database_versions') +@login_required +def get_database_versions(): + """ + Lists the postgresql database versions. + :return: PostgreSQL version list + """ + if 'google' in session and 'google_obj' in session['google']: + google_obj = pickle.loads(session['google']['google_obj']) + db_version_list = google_obj.get_database_versions() + return make_json_response(data=db_version_list) + else: + return make_json_response(data=[]) + + +def deploy_on_google(data): + """Deploy the Postgres instance on RDS.""" + _cmd = 'python' + _cmd_script = '{0}/pgacloud/pgacloud.py'.format(root) + _label = data['instance_details']['name'] + + # Supported arguments for google cloud sql deployment + args = [_cmd_script, + data['cloud'], + 'create-instance', + + '--project', data['instance_details']['project'], + + '--region', data['instance_details']['region'], + + '--name', data['instance_details']['name'], + + '--db-version', data['instance_details']['db_version'], + + '--instance-type', data['instance_details']['instance_type'], + + '--storage-type', data['instance_details']['storage_type'], + + '--storage-size', str(data['instance_details']['storage_size']), + + '--public-ip', str(data['instance_details']['public_ips']), + + '--availability-zone', + data['instance_details']['availability_zone'], + + '--high-availability', + str(data['instance_details']['high_availability']), + + '--secondary-availability-zone', + data['instance_details']['secondary_availability_zone'], + ] + + _cmd_msg = '{0} {1} {2}'.format(_cmd, _cmd_script, ' '.join(args)) + try: + sid = _create_server({ + 'gid': data['db_details']['gid'], + 'name': data['instance_details']['name'], + 'db': 'postgres', + 'username': 'postgres', + 'port': 5432, + 'cloud_status': -1 + }) + + p = BatchProcess( + desc=CloudProcessDesc(sid, _cmd_msg, data['cloud'], + data['instance_details']['name']), + cmd=_cmd, + args=args + ) + + # Set env variables for background process of deployment + env = dict() + google_obj = pickle.loads(session['google']['google_obj']) + env['GOOGLE_CREDENTIALS'] = json.dumps(google_obj.credentials_json) + + if 'db_password' in data['db_details']: + env['GOOGLE_DATABASE_PASSWORD'] = data['db_details']['db_password'] + + p.set_env_variables(None, env=env) + p.update_server_id(p.id, sid) + p.start() + + return True, p, {'label': _label, 'sid': sid} + except Exception as e: + current_app.logger.exception(e) + return False, None, str(e) + + +def clear_google_session(): + """Clear Google Session""" + if 'google' in session: + session.pop('google') + + +class Google: + def __init__(self, client_config=None): + # Google cloud sql api versions + self._cloud_resource_manager_api_version = 'v1' + self._sqladmin_api_version = 'v1' + self._compute_api_version = 'v1' + + # Scope required for google cloud sql deployment + self._scopes = ['https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/sqlservice.admin'] + + # Instance classed + self._instance_classes = [{'label': 'Standard', 'value': 'standard'}, + {'label': 'High Memory', 'value': 'highmem'}, + {'label': 'Shared', 'value': 'shared'}] + + self._client_config = client_config + self._credentials = None + self.credentials_json = None + self._project_id = None + self._regions = [] + self._availability_zones = {} + self._verification_successful = False + self._verification_error = None + self._redirect_url = None + + def get_auth_url(self, host_url): + """ + Provides google authorisation url + :param host_url: Base url for hosting application + :return: authorisation url to complete authentication + """ + auth_url = None + error = None + # reset below variable to get latest values in fresh + # authentication call + self._verification_successful = False + self._verification_error = None + try: + self._redirect_url = host_url + 'google/callback' + flow = InstalledAppFlow.from_client_config( + client_config=self._client_config, scopes=self._scopes, + redirect_uri=self._redirect_url) + auth_url, state = flow.authorization_url( + prompt='select_account', access_type='offline', + include_granted_scopes='true') + session["state"] = state + except Exception as e: + error = str(e) + self._verification_error = error + return auth_url, error + + def callback(self, flask_request): + """ + Callback function on completion of google authorisation request + :param flask_request: + :return: Success or error message + """ + try: + authorization_response = flask_request.url + if session['state'] != flask_request.args.get('state', None): + self._verification_successful = False, + self._verification_error = 'Invalid state parameter' + flow = InstalledAppFlow.from_client_config( + client_config=self._client_config, scopes=self._scopes, + redirect_uri=self._redirect_url) + flow.fetch_token(authorization_response=authorization_response) + self._credentials = flow.credentials + self.credentials_json = \ + self._credentials_to_dict(self._credentials) + self._verification_successful = True + return 'The authentication flow has completed. ' \ + 'This window will be closed.' + except AccessDeniedError as er: + self._verification_successful = False + self._verification_error = er.error + if self._verification_error == 'access_denied': + self._verification_error = 'Access denied.' + return self._verification_error + + @staticmethod + def _credentials_to_dict(credentials): + return {'token': credentials.token, + 'refresh_token': credentials.refresh_token, + 'token_uri': credentials.token_uri, + 'client_id': credentials.client_id, + 'client_secret': credentials.client_secret, + 'scopes': credentials.scopes, + 'id_token': credentials.id_token} + + def verification_ack(self): + """Check the Verification is done or not.""" + return self._verification_successful, self._verification_error + + def _get_credentials(self, scopes): + """ + Provides google credentials for google cloud sql api calls + :param scopes: Required scope of credentials + :return: google credential object + """ + if not self._credentials or not self._credentials.valid: + if self._credentials and self._credentials.expired and \ + self._credentials.refresh_token and \ + self._credentials.has_scopes(scopes): + self._credentials.refresh(Request()) + return self._credentials + return self._credentials + + def get_projects(self): + """ + List the google projects for authorised user + :return: + """ + projects = [] + credentials = self._get_credentials(self._scopes) + service = discovery.build('cloudresourcemanager', + self._cloud_resource_manager_api_version, + credentials=credentials) + req = service.projects().list() + res = req.execute() + for project in res.get('projects', []): + projects.append({'label': project['projectId'], + 'value': project['projectId']}) + return projects + + def get_regions(self, project): + """ + List regions for specified google cloud project + :param project: google cloud project id. + :return: + """ + self._project_id = project + credentials = self._get_credentials(self._scopes) + service = discovery.build('compute', + self._compute_api_version, + credentials=credentials) + try: + req = service.regions().list(project=project) + res = req.execute() + except HttpError: + self._regions = [] + return self._regions + for item in res.get('items', []): + region_name = item['name'] + self._regions.append({'label': region_name, 'value': region_name}) + region_zones = item.get('zones', []) + region_zones = list( + map(lambda region: region.split('/')[-1], region_zones)) + self._availability_zones[region_name] = region_zones + return self._regions + + def get_availability_zones(self, region): + """ + List availability zones in given google cloud region + :param region: google cloud region + :return: + """ + az_list = [] + for az in self._availability_zones.get(region, []): + az_list.append({'label': az, 'value': az}) + return az_list + + def get_instance_types(self, project, region): + """ + Lists google cloud sql instance types. + :param project: + :param region: + :return: + """ + standard_instances = [] + shared_instances = [] + high_mem = [] + credentials = self._get_credentials(self._scopes) + service = discovery.build('sqladmin', + self._sqladmin_api_version, + credentials=credentials) + req = service.tiers().list(project=project) + res = req.execute() + for item in res.get('items', []): + if region in item.get('region', []): + if item['tier'].find('standard') != -1: + vcpu = item['tier'].split('-')[-1] + mem = round(int(item['RAM']) / (1024 * 1024)) + label = vcpu + ' vCPU, ' + str(round(mem / 1024)) + ' GB' + value = 'db-custom-' + str(vcpu) + '-' + str(mem) + standard_instances.append({'label': label, 'value': value}) + elif item['tier'].find('highmem') != -1: + vcpu = item['tier'].split('-')[-1] + mem = round(int(item['RAM']) / (1024 * 1024)) + label = vcpu + ' vCPU, ' + str(round(mem / 1024)) + ' GB' + value = 'db-custom-' + str(vcpu) + '-' + str(mem) + high_mem.append({'label': label, 'value': value}) + else: + label = '1 vCPU, ' + str( + round((int(item['RAM']) / 1073741824), 2)) + ' GB' + value = item['tier'] + shared_instances.append({'label': label, 'value': value}) + instance_types = {'standard': standard_instances, + 'highmem': high_mem, + 'shared': shared_instances} + return instance_types + + def get_database_versions(self): + """ + Lists the PostgreSQL database versions + :return: + """ + pg_database_versions = [] + database_versions = [] + credentials = self._get_credentials(self._scopes) + service = discovery.build('sqladmin', + self._sqladmin_api_version, + credentials=credentials) + req = service.flags().list() + res = req.execute() + for item in res.get('items', []): + if item.get('name', '') == 'max_parallel_workers': + pg_database_versions = item.get('appliesTo', []) + for version in pg_database_versions: + label = (version.title().split('_')[0])[0:7] \ + + 'SQL ' + version.split('_')[1] + database_versions.append({'label': label, 'value': version}) + return database_versions diff --git a/web/pgadmin/misc/cloud/static/js/CloudWizard.jsx b/web/pgadmin/misc/cloud/static/js/CloudWizard.jsx index f5ad5cd8a..452cdd984 100644 --- a/web/pgadmin/misc/cloud/static/js/CloudWizard.jsx +++ b/web/pgadmin/misc/cloud/static/js/CloudWizard.jsx @@ -24,10 +24,11 @@ import { PrimaryButton } from '../../../../static/js/components/Buttons'; import {AwsCredentials, AwsInstanceDetails, AwsDatabaseDetails, validateCloudStep1, validateCloudStep2, validateCloudStep3} from './aws'; import {BigAnimalInstance, BigAnimalDatabase, BigAnimalClusterType, getProviderOptions, validateBigAnimal, validateBigAnimalStep2, validateBigAnimalStep3, validateBigAnimalStep4} from './biganimal'; import { isEmptyString } from 'sources/validators'; -import { AWSIcon, BigAnimalIcon, AzureIcon } from '../../../../static/js/components/ExternalIcon'; +import { AWSIcon, BigAnimalIcon, AzureIcon, GoogleCloudIcon } from '../../../../static/js/components/ExternalIcon'; import {AzureCredentials, AzureInstanceDetails, AzureDatabaseDetails, checkClusternameAvailbility, validateAzureStep2, validateAzureStep3} from './azure'; +import { GoogleCredentials, GoogleInstanceDetails, GoogleDatabaseDetails, validateGoogleStep2, validateGoogleStep3 } from './google'; import EventBus from '../../../../static/js/helpers/EventBus'; -import { CLOUD_PROVIDERS } from './cloud_constants'; +import { CLOUD_PROVIDERS, CLOUD_PROVIDERS_LABELS } from './cloud_constants'; const useStyles = makeStyles(() => @@ -63,10 +64,8 @@ const useStyles = makeStyles(() => export const CloudWizardEventsContext = React.createContext(); - export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) { const classes = useStyles(); - const eventBus = React.useRef(new EventBus()); let steps = [gettext('Cloud Provider'), gettext('Credentials'), gettext('Cluster Type'), @@ -81,6 +80,7 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) const [hostIP, setHostIP] = React.useState('127.0.0.1/32'); const [cloudProvider, setCloudProvider] = React.useState(''); const [verificationIntiated, setVerificationIntiated] = React.useState(false); + const [bigAnimalInstanceData, setBigAnimalInstanceData] = React.useState({}); const [bigAnimalDatabaseData, setBigAnimalDatabaseData] = React.useState({}); const [bigAnimalClusterTypeData, setBigAnimalClusterTypeData] = React.useState({}); @@ -90,6 +90,10 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) const [azureInstanceData, setAzureInstanceData] = React.useState({}); const [azureDatabaseData, setAzureDatabaseData] = React.useState({}); + const [googleCredData, setGoogleCredData] = React.useState({}); + const [googleInstanceData, setGoogleInstanceData] = React.useState({}); + const [googleDatabaseData, setGoogleDatabaseData] = React.useState({}); + const axiosApi = getApiInstance(); const [verificationURI, setVerificationURI] = React.useState(''); @@ -128,7 +132,7 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) let _url = url_for('cloud.deploy_on_cloud'), post_data = {}; - if (cloudProvider == CLOUD_PROVIDERS.RDS) { + if (cloudProvider == CLOUD_PROVIDERS.AWS) { post_data = { gid: nodeInfo.server_group._id, cloud: cloudProvider, @@ -144,6 +148,14 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) instance_details:azureInstanceData, db_details: azureDatabaseData }; + }else if(cloudProvider == CLOUD_PROVIDERS.GOOGLE){ + post_data = { + gid: nodeInfo.server_group._id, + secret: googleCredData, + cloud: cloudProvider, + instance_details:googleInstanceData, + db_details: googleDatabaseData + }; }else { post_data = { @@ -170,10 +182,10 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) setCallRDSAPI(currentStep); let isError = (cloudProvider == ''); switch(cloudProvider) { - case CLOUD_PROVIDERS.RDS: + case CLOUD_PROVIDERS.AWS: switch (currentStep) { case 0: - setCloudSelection(CLOUD_PROVIDERS.RDS); + setCloudSelection(CLOUD_PROVIDERS.AWS); break; case 1: isError = validateCloudStep1(cloudDBCred); @@ -231,13 +243,32 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) break; } break; + case CLOUD_PROVIDERS.GOOGLE: + switch (currentStep) { + case 0: + setCloudSelection(CLOUD_PROVIDERS.GOOGLE); + break; + case 1: + isError = !verificationIntiated; + break; + case 2: + break; + case 3: + isError = validateGoogleStep2(googleInstanceData); + break; + case 4: + isError = validateGoogleStep3(googleDatabaseData, nodeInfo); + break; + default: + break; + } } return isError; }; const onBeforeBack = (activeStep) => { return new Promise((resolve)=>{ - if(activeStep == 3 && (cloudProvider == CLOUD_PROVIDERS.RDS || cloudProvider == CLOUD_PROVIDERS.AZURE)) { + if(activeStep == 3 && (cloudProvider == CLOUD_PROVIDERS.AWS || cloudProvider == CLOUD_PROVIDERS.AZURE || cloudProvider == CLOUD_PROVIDERS.GOOGLE)) { resolve(true); } resolve(); @@ -246,7 +277,7 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) const onBeforeNext = (activeStep) => { return new Promise((resolve, reject)=>{ - if(activeStep == 1 && cloudProvider == CLOUD_PROVIDERS.RDS) { + if(activeStep == 1 && cloudProvider == CLOUD_PROVIDERS.AWS) { setErrMsg([MESSAGE_TYPE.INFO, gettext('Validating credentials...')]); let _url = url_for('rds.verify_credentials'); const post_data = { @@ -298,6 +329,7 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) } else if (cloudProvider == CLOUD_PROVIDERS.AZURE) { if (activeStep == 1) { // Skip the current step + setErrMsg(['', '']); resolve(true); } else if (activeStep == 2) { setErrMsg([MESSAGE_TYPE.INFO, gettext('Checking cluster name availability...')]); @@ -316,6 +348,14 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) } else { resolve(); } + }else if (cloudProvider == CLOUD_PROVIDERS.GOOGLE) { + if (activeStep == 1) { + // Skip the current step + setErrMsg(['', '']); + resolve(true); + } else if (activeStep == 2) { resolve(true);} else { + resolve(); + } } else { setErrMsg(['', '']); @@ -375,9 +415,11 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) setErrMsg([]); }); - let cloud_providers = [{label: gettext('Amazon RDS'), value: CLOUD_PROVIDERS.RDS, icon: }, - {label: gettext('EDB BigAnimal'), value: CLOUD_PROVIDERS.BIGANIMAL, icon: }, - {'label': gettext('Azure PostgreSQL'), value: CLOUD_PROVIDERS.AZURE, icon: }]; + let cloud_providers = [ + {label: gettext(CLOUD_PROVIDERS_LABELS.AWS), value: CLOUD_PROVIDERS.AWS, icon: }, + {label: gettext(CLOUD_PROVIDERS_LABELS.BIGANIMAL), value: CLOUD_PROVIDERS.BIGANIMAL, icon: }, + {label: gettext(CLOUD_PROVIDERS_LABELS.AZURE), value: CLOUD_PROVIDERS.AZURE, icon: }, + {label: gettext(CLOUD_PROVIDERS_LABELS.GOOGLE), value: CLOUD_PROVIDERS.GOOGLE, icon: }]; return ( @@ -393,7 +435,7 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) beforeBack={onBeforeBack}> - {gettext('Select a cloud provider.')} + {gettext('Select a cloud provider for PostgreSQL database.')} } - {cloudProvider == CLOUD_PROVIDERS.RDS && } + {cloudProvider == CLOUD_PROVIDERS.AWS && } + { cloudProvider == CLOUD_PROVIDERS.AZURE && - {cloudProvider == CLOUD_PROVIDERS.AZURE && } + + } + + {cloudProvider == CLOUD_PROVIDERS.GOOGLE && } @@ -434,7 +480,7 @@ export default function CloudWizard({ nodeInfo, nodeData, onClose, cloudPanel}) - {cloudProvider == CLOUD_PROVIDERS.RDS && callRDSAPI == 3 && } + {cloudProvider == CLOUD_PROVIDERS.GOOGLE && callRDSAPI == 3 && } - {cloudProvider == CLOUD_PROVIDERS.RDS && } + {cloudProvider == CLOUD_PROVIDERS.GOOGLE && + } {gettext('Please review the details before creating the cloud instance.')} - {cloudProvider == CLOUD_PROVIDERS.RDS && callRDSAPI == 5 && } + {cloudProvider == CLOUD_PROVIDERS.GOOGLE && callRDSAPI == 5 && + } diff --git a/web/pgadmin/misc/cloud/static/js/azure.js b/web/pgadmin/misc/cloud/static/js/azure.js index c00e58303..569c2b80b 100644 --- a/web/pgadmin/misc/cloud/static/js/azure.js +++ b/web/pgadmin/misc/cloud/static/js/azure.js @@ -74,7 +74,8 @@ export function AzureCredentials(props) { .then((res)=>{ if (res.data.success){ clearInterval(interval); - window.open(res.data.data.verification_uri, 'azure_authentication'); + let params = 'scrollbars=no,resizable=no,status=no,location=no,toolbar=no,menubar=no, width=550,height=650,left=920,top=150'; + window.open(res.data.data.verification_uri, 'azure_authentication', params); resolve(res); } }) diff --git a/web/pgadmin/misc/cloud/static/js/cloud.js b/web/pgadmin/misc/cloud/static/js/cloud.js index 6042b6ce0..17a1c2032 100644 --- a/web/pgadmin/misc/cloud/static/js/cloud.js +++ b/web/pgadmin/misc/cloud/static/js/cloud.js @@ -76,7 +76,7 @@ define('pgadmin.misc.cloud', [ // Register dialog panel pgBrowser.Node.registerUtilityPanel(); - let panel = pgBrowser.Node.addUtilityPanel(920, 650), + let panel = pgBrowser.Node.addUtilityPanel(930, 650), j = panel.$container.find('.obj_properties').first(); panel.title(gettext('Deploy Cloud Instance')); diff --git a/web/pgadmin/misc/cloud/static/js/cloud_components.jsx b/web/pgadmin/misc/cloud/static/js/cloud_components.jsx index 200f69b28..7b5edab22 100644 --- a/web/pgadmin/misc/cloud/static/js/cloud_components.jsx +++ b/web/pgadmin/misc/cloud/static/js/cloud_components.jsx @@ -20,13 +20,23 @@ import { commonTableStyles } from '../../../../static/js/Theme'; import { Table, TableBody, TableCell, TableHead, TableRow } from '@material-ui/core'; import clsx from 'clsx'; import gettext from 'sources/gettext'; - +import { getGoogleSummary } from './google'; +import { CLOUD_PROVIDERS_LABELS } from './cloud_constants'; const useStyles = makeStyles(() => ({ - toggleButton: { + toggleButtonGroup: { height: '100px', + flexGrow: '1' }, + toggleButtonMargin:{ + marginTop: '0px !important', + padding: '12px' + }, + gcpiconpadding:{ + paddingLeft: '1.5rem' + } + }), ); @@ -43,13 +53,14 @@ export function ToggleButtons(props) { color="primary" value={props.cloudProvider} onChange={handleCloudProvider} - className={classes.toggleButton} + className={classes.toggleButtonGroup} + orientation="vertical" exclusive> { (props.options||[]).map((option)=>{ - return ( + return (   - {option.icon}  {option.label} + {option.icon}    {option.label} ); }) } @@ -74,6 +85,9 @@ export function FinalSummary(props) { } else if(props.cloudProvider == 'azure') { summaryHeader.push('Network Connectivity','Availability'); summary = getAzureSummary(props.cloudProvider, props.instanceData, props.databaseData); + }else if(props.cloudProvider == 'google') { + summaryHeader.push('Network Connectivity','Availability'); + summary = getGoogleSummary(props.cloudProvider, props.instanceData, props.databaseData); }else { summaryHeader.push('Availability'); summary = getAWSSummary(props.cloudProvider, props.instanceData, props.databaseData); diff --git a/web/pgadmin/misc/cloud/static/js/cloud_constants.js b/web/pgadmin/misc/cloud/static/js/cloud_constants.js index 6388c47d0..7ef39f90b 100644 --- a/web/pgadmin/misc/cloud/static/js/cloud_constants.js +++ b/web/pgadmin/misc/cloud/static/js/cloud_constants.js @@ -11,5 +11,12 @@ export const CLOUD_PROVIDERS = { AZURE: 'azure', BIGANIMAL: 'biganimal', AWS: 'aws', - RDS: 'rds', + GOOGLE: 'google', +}; + +export const CLOUD_PROVIDERS_LABELS = { + AZURE: 'Azure Database', + BIGANIMAL: 'EDB BigAnimal', + AWS: 'Amazon RDS', + GOOGLE: 'Google Cloud SQL', }; diff --git a/web/pgadmin/misc/cloud/static/js/google.js b/web/pgadmin/misc/cloud/static/js/google.js new file mode 100644 index 000000000..2a14ce610 --- /dev/null +++ b/web/pgadmin/misc/cloud/static/js/google.js @@ -0,0 +1,314 @@ +///////////////////////////////////////////////////////////// +// +// pgAdmin 4 - PostgreSQL Tools +// +// Copyright (C) 2013 - 2023, The pgAdmin Development Team +// This software is released under the PostgreSQL Licence +// +////////////////////////////////////////////////////////////// +import React from 'react'; +import {GoogleCredSchema, GoogleClusterSchema, GoogleDatabaseSchema} from './google_schema.ui'; +import pgAdmin from 'sources/pgadmin'; +import { getNodeAjaxOptions, getNodeListById } from 'pgbrowser/node_ajax'; +import SchemaView from '../../../../static/js/SchemaView'; +import url_for from 'sources/url_for'; +import { isEmptyString } from 'sources/validators'; +import PropTypes from 'prop-types'; +import getApiInstance from '../../../../static/js/api_instance'; +import { CloudWizardEventsContext } from './CloudWizard'; +import {MESSAGE_TYPE } from '../../../../static/js/components/FormComponents'; +import gettext from 'sources/gettext'; +import { makeStyles } from '@material-ui/core/styles'; + +const useStyles = makeStyles(() => + ({ + formClass: { + overflow: 'auto', + } + }), +); + + +export function GoogleCredentials(props) { + const [cloudDBCredInstance, setCloudDBCredInstance] = React.useState(); + + let _eventBus = React.useContext(CloudWizardEventsContext); + let child = null; + React.useMemo(() => { + const googleCredSchema = new GoogleCredSchema({ + authenticateGoogle:(client_secret_file) => { + let loading_icon_url = url_for( + 'static', { 'filename': 'img/loading.gif'} + ); + const axiosApi = getApiInstance(); + _eventBus.fireEvent('SET_ERROR_MESSAGE_FOR_CLOUD_WIZARD', [MESSAGE_TYPE.INFO, 'Google authentication process is in progress..' + gettext('Loading...') + '']); + let _url = url_for('google.verify_credentials'); + const post_data = { + cloud: 'google', + secret: {'client_secret_file':client_secret_file} + }; + return new Promise((resolve, reject)=>{axiosApi.post(_url, post_data) + .then((res) => { + if (res.data && res.data.success == 1 ) { + _eventBus.fireEvent('SET_CRED_VERIFICATION_INITIATED',true); + let params = 'scrollbars=no,resizable=no,status=no,location=no,toolbar=no,menubar=no, width=550,height=650,left=600,top=150'; + child = window.open(res.data.data.auth_url, 'google_authentication', params); + resolve(true); + } + else if (res.data && res.data.success == 0) { + _eventBus.fireEvent('SET_ERROR_MESSAGE_FOR_CLOUD_WIZARD',[MESSAGE_TYPE.ERROR, res.data.errormsg]); + _eventBus.fireEvent('SET_CRED_VERIFICATION_INITIATED',false); + resolve(false); + } + }) + .catch((error) => { + _eventBus.fireEvent('SET_ERROR_MESSAGE_FOR_CLOUD_WIZARD',[MESSAGE_TYPE.ERROR, gettext(`Error while authentication: ${error}`)]); + reject(false); + }); + }); + }, + verification_ack:()=>{ + let auth_url = url_for('google.verification_ack'); + let countdown = 90; + const axiosApi = getApiInstance(); + return new Promise((resolve, reject)=>{ + const interval = setInterval(()=>{ + axiosApi.get(auth_url) + .then((res)=>{ + if (res.data.success && res.data.success == 1 ){ + _eventBus.fireEvent('SET_ERROR_MESSAGE_FOR_CLOUD_WIZARD',[MESSAGE_TYPE.SUCCESS, gettext('Authentication completed successfully. Click the Next button to proceed.')]); + clearInterval(interval); + if(child){ + // close authentication window + child.close(); + } + resolve(); + } else if (res.data && res.data.success == 0 && (res.data.errormsg == 'Invalid state parameter.' || res.data.errormsg == 'Access denied.' || res.data.errormsg == 'Authentication is failed.')){ + _eventBus.fireEvent('SET_ERROR_MESSAGE_FOR_CLOUD_WIZARD',[MESSAGE_TYPE.ERROR, res.data.errormsg]); + _eventBus.fireEvent('SET_CRED_VERIFICATION_INITIATED',false); + clearInterval(interval); + resolve(false); + } else if (child && child.closed || countdown <= 0) { + _eventBus.fireEvent('SET_ERROR_MESSAGE_FOR_CLOUD_WIZARD',[MESSAGE_TYPE.ERROR, 'Authentication is aborted.']); + _eventBus.fireEvent('SET_CRED_VERIFICATION_INITIATED',false); + clearInterval(interval); + } + }) + .catch((error)=>{ + clearInterval(interval); + reject(error); + }); + countdown = countdown - 1; + }, 1000); + }); + } + }, {}, _eventBus); + setCloudDBCredInstance(googleCredSchema); + }, [props.cloudProvider]); + + return { /*This is intentional (SonarQube)*/ }} + viewHelperProps={{ mode: 'create' }} + schema={cloudDBCredInstance} + showFooter={false} + isTabView={false} + onDataChange={(isChanged, changedData) => { + props.setGoogleCredData(changedData); + }} + />; +} +GoogleCredentials.propTypes = { + nodeInfo: PropTypes.object, + nodeData: PropTypes.object, + cloudProvider: PropTypes.string, + setGoogleCredData: PropTypes.func +}; + +// Google Instance +export function GoogleInstanceDetails(props) { + const [googleInstanceSchema, setGoogleInstanceSchema] = React.useState(); + const classes = useStyles(); + + React.useMemo(() => { + const GoogleClusterSchemaObj = new GoogleClusterSchema({ + projects: () => getNodeAjaxOptions('get_projects', {}, {}, {},{ + useCache:false, + cacheNode: 'server', + customGenerateUrl: ()=>{ + return url_for('google.projects'); + } + }), + regions: (project)=>getNodeAjaxOptions('get_regions', pgAdmin.Browser.Nodes['server'], props.nodeInfo, props.nodeData,{ + useCache:false, + cacheNode: 'server', + customGenerateUrl: ()=>{ + return url_for('google.regions', {'project_id': project}); + } + }), + availabilityZones: (region)=>getNodeAjaxOptions('get_availability_zones', pgAdmin.Browser.Nodes['server'], props.nodeInfo, props.nodeData, { + useCache:false, + cacheNode: 'server', + customGenerateUrl: ()=>{ + return url_for('google.availability_zones', {'region': region}); + } + }), + dbVersions: ()=>getNodeAjaxOptions('get_db_versions', pgAdmin.Browser.Nodes['server'], props.nodeInfo, props.nodeData, { + useCache:false, + cacheNode: 'server', + customGenerateUrl: ()=>{ + return url_for('google.database_versions'); + } + }), + instanceTypes: (project, region, instanceClass)=>{ + if (isEmptyString(project) || isEmptyString(region) || isEmptyString(instanceClass)) return []; + return getNodeAjaxOptions('get_instance_types', pgAdmin.Browser.Nodes['server'], props.nodeInfo, props.nodeData, { + useCache:false, + cacheNode: 'server', + customGenerateUrl: ()=>{ + return url_for('google.instance_types', {'project_id':project, 'region': region, 'instance_class': instanceClass}); + } + });}, + }, { + nodeInfo: props.nodeInfo, + nodeData: props.nodeData, + hostIP: props.hostIP, + ...props.googleInstanceData + }); + setGoogleInstanceSchema(GoogleClusterSchemaObj); + }, [props.cloudProvider]); + + return { /*This is intentional (SonarQube)*/ }} + viewHelperProps={{ mode: 'create' }} + schema={googleInstanceSchema} + showFooter={false} + isTabView={false} + onDataChange={(isChanged, changedData) => { + props.setGoogleInstanceData(changedData); + }} + formClassName={classes.formClass} + />; +} +GoogleInstanceDetails.propTypes = { + nodeInfo: PropTypes.object, + nodeData: PropTypes.object, + cloudProvider: PropTypes.string, + setGoogleInstanceData: PropTypes.func, + hostIP: PropTypes.string, + subscriptions: PropTypes.array, + googleInstanceData: PropTypes.object +}; + + +// Google Database Details +export function GoogleDatabaseDetails(props) { + const [gooeleDBInstance, setGoogleDBInstance] = React.useState(); + const classes = useStyles(); + + React.useMemo(() => { + const googleDBSchema = new GoogleDatabaseSchema({ + server_groups: ()=>getNodeListById(pgAdmin.Browser.Nodes['server_group'], props.nodeInfo, props.nodeData), + }, + { + gid: props.nodeInfo['server_group']._id, + } + ); + setGoogleDBInstance(googleDBSchema); + + }, [props.cloudProvider]); + + return { /*This is intentional (SonarQube)*/ }} + viewHelperProps={{ mode: 'create' }} + schema={gooeleDBInstance} + showFooter={false} + isTabView={false} + onDataChange={(isChanged, changedData) => { + props.setGoogleDatabaseData(changedData); + }} + formClassName={classes.formClass} + />; +} +GoogleDatabaseDetails.propTypes = { + nodeInfo: PropTypes.object, + nodeData: PropTypes.object, + cloudProvider: PropTypes.string, + setGoogleDatabaseData: PropTypes.func, +}; + +// Validation functions +export function validateGoogleStep2(cloudInstanceDetails) { + let isError = false; + if (isEmptyString(cloudInstanceDetails.name) || + isEmptyString(cloudInstanceDetails.db_version) || isEmptyString(cloudInstanceDetails.instance_type) || + isEmptyString(cloudInstanceDetails.region)|| isEmptyString(cloudInstanceDetails.storage_size) || isEmptyString(cloudInstanceDetails.public_ips)) { + isError = true; + } + return isError; +} + +export function validateGoogleStep3(cloudDBDetails, nodeInfo) { + let isError = false; + if (isEmptyString(cloudDBDetails.db_username) || isEmptyString(cloudDBDetails.db_password)) { + isError = true; + } + + if (cloudDBDetails.db_password != cloudDBDetails.db_confirm_password) { + isError = true; + } + + if (isEmptyString(cloudDBDetails.gid)) cloudDBDetails.gid = nodeInfo['server_group']._id; + return isError; +} + +// Summary creation +function createData(name, value) { + if (typeof(value) == 'boolean') { + value = (value === true) ? 'True' : 'False'; + } + return { name, value }; +} + +// Summary section +export function getGoogleSummary(cloud, cloudInstanceDetails, cloudDBDetails) { + let dbVersion = cloudInstanceDetails.db_version; + dbVersion = dbVersion.charAt(0) + dbVersion.slice(1,7).toLowerCase() + 'SQL ' + dbVersion.split('_')[1]; + let storageType = cloudInstanceDetails.storage_type.split('_')[1]; + + const rows1 = [ + createData(gettext('Cloud'), cloud), + createData(gettext('Instance name'), cloudInstanceDetails.name), + createData(gettext('Project'), cloudInstanceDetails.project), + createData(gettext('Region'), cloudInstanceDetails.region), + createData(gettext('Availability zone'), cloudInstanceDetails.availability_zone), + ]; + + const rows2 = [ + createData(gettext('PostgreSQL version'), dbVersion), + createData(gettext('Instance type'), cloudInstanceDetails.instance_type), + ]; + + const rows3 = [ + createData(gettext('Storage type'), storageType), + createData(gettext('Allocated storage'), cloudInstanceDetails.storage_size + ' GB'), + ]; + + const rows4 = [ + createData(gettext('Username'), cloudDBDetails.db_username), + createData(gettext('Password'), 'xxxxxxx'), + ]; + + const rows5 = [ + createData(gettext('Public IP'), cloudInstanceDetails.public_ips), + ]; + + const rows6 = [ + createData(gettext('High availability'), cloudInstanceDetails.high_availability), + createData(gettext('Secondary availability zone'), cloudInstanceDetails.secondary_availability_zone), + ]; + + return [rows1, rows2, rows3, rows4, rows5, rows6]; +} diff --git a/web/pgadmin/misc/cloud/static/js/google_schema.ui.js b/web/pgadmin/misc/cloud/static/js/google_schema.ui.js new file mode 100644 index 000000000..1f9182e2d --- /dev/null +++ b/web/pgadmin/misc/cloud/static/js/google_schema.ui.js @@ -0,0 +1,524 @@ +///////////////////////////////////////////////////////////// +// +// pgAdmin 4 - PostgreSQL Tools +// +// Copyright (C) 2013 - 2023, The pgAdmin Development Team +// This software is released under the PostgreSQL Licence +// +////////////////////////////////////////////////////////////// + +import gettext from 'sources/gettext'; +import BaseUISchema from 'sources/SchemaView/base_schema.ui'; +import { isEmptyString } from 'sources/validators'; + +class GoogleCredSchema extends BaseUISchema{ + constructor(fieldOptions = {}, initValues = {}, eventBus={}) { + super({ + oid: null, + client_secret_file: undefined, + ...initValues, + }); + + this.fieldOptions = { + ...fieldOptions, + }; + + this.eventBus = eventBus; + + } + + get idAttribute() { + return 'oid'; + } + + get baseFields() { + let obj = this; + return [ + { + id: 'client_secret_file', + label: gettext('Client secret file'), + type: 'file', + helpMessage: gettext('Select a client secrets file containing the client ID, client secret, and other OAuth 2.0 parameters for google authentication. Refer link for creating client secret.'), + controlProps: { + dialogType: 'select_file', + supportedTypes: ['json'], + dialogTitle: 'Select file', + }, + }, + { + id: 'auth_btn', + mode: ['create'], + deps: ['client_secret_file'], + type: 'button', + btnName: gettext('Click here to authenticate yourself to Google'), + helpMessage: gettext('After clicking the button above you will be redirected to the Google authentication page in a new browser tab.'), + disabled: (state)=>{ + return state.client_secret_file ? false : true; + }, + depChange: ()=> { + return {is_authenticating: true}; + }, + deferredDepChange: (state, source)=>{ + return new Promise((resolve, reject)=>{ + /* button clicked */ + if(source == 'auth_btn') { + obj.fieldOptions.authenticateGoogle(state.client_secret_file) + .then(()=>{ + resolve(()=>({ + })); + }) + .catch((err)=>{ + reject(err); + }); + } + }); + } + }, + { + id: 'is_authenticating', + visible: false, + type: '', + deps:['auth_btn'], + deferredDepChange: (state, source)=>{ + return new Promise((resolve, reject)=>{ + if(source == 'auth_btn' && state.is_authenticating ) { + obj.fieldOptions.verification_ack() + .then(()=>{ + resolve(); + }) + .catch((err)=>{ + reject(err); + }); + } + }); + }, + }, + ];} + +} + +class GoogleProjectDetailsSchema extends BaseUISchema { + constructor(fieldOptions = {}, initValues = {}) { + super({ + oid: undefined, + project: '', + region: '', + availability_zone: '', + ...initValues, + }); + + this.fieldOptions = { + ...fieldOptions, + }; + + this.initValues = initValues; + } + + get idAttribute() { + return 'oid'; + } + + get baseFields() { + return [ + { + id: 'project', + label: gettext('Project'), + mode: ['create'], + allowClear: false, + noEmpty: true, + type: () => { + return { + type: 'select', + options: this.fieldOptions.projects + }; + }, + }, + { + id: 'region', + label: gettext('Location'), + mode: ['create'], + deps: ['project'], + noEmpty: true, + type: (state) => { + return { + type: 'select', + options: state.project + ? () => this.fieldOptions.regions(state.project) + : [], + optionsReloadBasis: state.project, + allowClear: false, + }; + }, + }, + { + id: 'availability_zone', + label: gettext('Availability zone'), + deps: ['region'], + allowClear: false, + noEmpty: true, + type: (state) => { + return { + type: 'select', + options: state.region + ? () => this.fieldOptions.availabilityZones(state.region) + : [], + optionsReloadBasis: state.region, + }; + }, + } + ]; + } +} + +class GoogleInstanceSchema extends BaseUISchema { + constructor(fieldOptions = {}, initValues = {}) { + super({ + db_version: '', + instance_type: '', + storage_size: '', + }); + + this.fieldOptions = { + ...fieldOptions, + }; + + this.initValues = initValues; + } + + get idAttribute() { + return 'oid'; + } + + get baseFields() { + return [ + { + id: 'db_version', + label: gettext('Database version'), + deps: ['availability_zone'], + type: 'select', + noEmpty: true, + options: this.fieldOptions.dbVersions + }, + { + id: 'instance_class', + label: gettext('Instance class'), + type: 'select', + noEmpty: true, + options: [ + { + label: gettext('Shared core'), + value: 'shared' }, + { + label: gettext('Standard'), + value: 'standard', + }, + { + label: gettext('High Memory'), + value: 'highmem', + }, + ], + }, + { + id: 'instance_type', + label: gettext('Instance type'), + deps: ['instance_class'], + noEmpty: true, + type: (state) => { + return { + type: 'select', + allowClear: false, + options: state.instance_class + ? () => this.fieldOptions.instanceTypes(state.project, state.region, state.instance_class) + : [], + optionsReloadBasis: state.instance_class + }; + }, + } + ]; + } +} + +class GoogleStorageSchema extends BaseUISchema { + constructor() { + super({ + storage_type: 'SSD', + }); + } + + get baseFields() { + return [ + { + id: 'storage_type', + label: gettext('Storage type'), + type: 'select', + mode: ['create'], + noEmpty: true, + options: [ + {'label': gettext('SSD'), 'value': 'PD_SSD'}, + {'label': gettext('HDD'), 'value': 'PD_HDD'}, + ], + }, + { + id: 'storage_size', + label: gettext('Storage capacity'), + type: 'text', + mode: ['create'], + noEmpty: true, + deps: ['storage_type'], + helpMessage: gettext('Size in GB.'), + } + ]; + } + + validate(data, setErrMsg) { + if (data.storage_size && (data.storage_size < 9 || data.storage_size > 65536)) { + setErrMsg('storage_size', gettext('Please enter value betwwen 10 and 65,536.')); + return true; + } + return false; + } +} + +class GoogleNetworkSchema extends BaseUISchema { + constructor() { + super(); + } + + get baseFields() { + return [ + { + id: 'public_ips', + label: gettext('Public IP range'), + type: 'text', + mode: ['create'], + noEmpty: true, + helpMessage: gettext('IP address range for allowed inbound traffic, for example: 127.0.0.1/32. Add multiple IP addresses/ranges separated with commas.' + ), + }, + ]; + } +} + +class GoogleHighAvailabilitySchema extends BaseUISchema { + constructor(fieldOptions = {}, initValues = {}) { + super({ + oid: undefined, + high_availability: false, + ...initValues, + }); + + this.fieldOptions = { + ...fieldOptions, + }; + this.initValues = initValues; + } + + get idAttribute() { + return 'oid'; + } + + get baseFields() { + return [ + { + id: 'high_availability', + label: gettext('High availability?'), + type: 'switch', + helpMessage: gettext(''), + }, + { + id: 'secondary_availability_zone', + label: gettext('Secondary availability zone'), + deps: ['high_availability'], + allowClear: false, + disabled:(state)=> { + if (!state.high_availability){ + state.secondary_availability_zone = ''; + } + return!state.high_availability;}, + type: (state) => { + return { + type: 'select', + options: state.region + ? () => this.fieldOptions.availabilityZones(state.region) + : [], + optionsReloadBasis: state.region, + }; + }, + helpMessage: gettext(''), + } + ]; + } + + validate(data, setErrMsg) { + if (data.high_availability && (isEmptyString(data.secondary_availability_zone))) { + setErrMsg('secondary_availability_zone', gettext('Please select Secondary availability zone.')); + return true; + } + return false; + } +} + +class GoogleDatabaseSchema extends BaseUISchema { + constructor(fieldOptions = {}, initValues = {}) { + super({ + oid: undefined, + gid: undefined, + db_username: 'postgres', + db_password: '', + db_confirm_password: '', + ...initValues, + }); + + this.fieldOptions = { + ...fieldOptions, + }; + } + + + get baseFields() { + return [ + { + id: 'gid', + label: gettext('pgAdmin server group'), + type: 'select', + options: this.fieldOptions.server_groups, + mode: ['create'], + controlProps: { allowClear: false }, + noEmpty: true, + }, + { + id: 'db_username', + label: gettext('Admin username'), + type: 'text', + mode: ['create'], + noEmpty: true, + disabled: true, + helpMessage: gettext( + 'Admin username for your Google Cloud Sql PostgreSQL instance.'), + }, + { + id: 'db_password', + label: gettext('Password'), + type: 'password', + mode: ['create'], + noEmpty: true, + helpMessage: gettext( + 'Set a password for the default admin user "postgres".' + ), + }, + { + id: 'db_confirm_password', + label: gettext('Confirm password'), + type: 'password', + mode: ['create'], + noEmpty: true, + }, + ]; + } + + validate(data, setErrMsg) { + if (!isEmptyString(data.db_password) && !isEmptyString(data.db_confirm_password) && data.db_password != data.db_confirm_password) { + setErrMsg('db_confirm_password', gettext('Passwords do not match.')); + return true; + } + return false; + } +} + +class GoogleClusterSchema extends BaseUISchema { + constructor(fieldOptions = {}, initValues = {}) { + super({ + oid: undefined, + name: '', + // Need to initilize child class init values in parent class itself + public_ips: initValues?.hostIP, + db_instance_class: undefined, + high_availability: false, + ...initValues, + }); + + this.fieldOptions = { + ...fieldOptions, + }; + this.initValues = initValues; + + this.googleProjectDetails = new GoogleProjectDetailsSchema( + { + projects: this.fieldOptions.projects, + regions: this.fieldOptions.regions, + availabilityZones: this.fieldOptions.availabilityZones, + }, + {} + ); + + this.googleInstanceDetails = new GoogleInstanceSchema( + { + dbVersions: this.fieldOptions.dbVersions, + instanceTypes: this.fieldOptions.instanceTypes, + }, + {} + ); + + this.googleStorageDetails = new GoogleStorageSchema( + {}, + {} + ); + + this.googleNetworkDetails = new GoogleNetworkSchema({}, {}); + + this.googleHighAvailabilityDetails = new GoogleHighAvailabilitySchema( + { + availabilityZones: this.fieldOptions.availabilityZones, + }, + {} + ); + } + + get idAttribute() { + return 'oid'; + } + + get baseFields() { + return [ + { + id: 'name', + label: gettext('Cluster name'), + type: 'text', + mode: ['create'], + noEmpty: true, + }, + { + type: 'nested-fieldset', + label: gettext('Project Details'), + mode: ['create'], + schema: this.googleProjectDetails, + }, + { + type: 'nested-fieldset', + label: gettext('Version & Instance'), + mode: ['create'], + schema: this.googleInstanceDetails, + }, + { + type: 'nested-fieldset', + label: gettext('Storage'), + mode: ['create'], + schema: this.googleStorageDetails, + }, + { + type: 'nested-fieldset', + label: gettext('Network Connectivity'), + mode: ['create'], + schema: this.googleNetworkDetails, + }, + { + type: 'nested-fieldset', + label: gettext('Availability'), + mode: ['create'], + schema: this.googleHighAvailabilityDetails, + }, + ]; + } +} + +export {GoogleCredSchema, GoogleClusterSchema, GoogleDatabaseSchema}; \ No newline at end of file diff --git a/web/pgadmin/misc/cloud/utils/__init__.py b/web/pgadmin/misc/cloud/utils/__init__.py index d387374d4..bfc9289c4 100644 --- a/web/pgadmin/misc/cloud/utils/__init__.py +++ b/web/pgadmin/misc/cloud/utils/__init__.py @@ -69,9 +69,11 @@ class CloudProcessDesc(IProcessDesc): if _provider == 'rds': self.provider = 'Amazon RDS' elif _provider == 'azure': - self.provider = 'Azure PostgreSQL' + self.provider = 'Azure Database' + elif _provider == 'google': + self.provider = 'Google Cloud SQL' else: - self.provider = 'EDB Big Animal' + self.provider = 'EDB BigAnimal' @property def message(self): diff --git a/web/pgadmin/static/img/azure.svg b/web/pgadmin/static/img/azure.svg index 445315a5d..a03f43e18 100644 --- a/web/pgadmin/static/img/azure.svg +++ b/web/pgadmin/static/img/azure.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/web/pgadmin/static/img/google-cloud-1.svg b/web/pgadmin/static/img/google-cloud-1.svg new file mode 100644 index 000000000..d54320443 --- /dev/null +++ b/web/pgadmin/static/img/google-cloud-1.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + diff --git a/web/pgadmin/static/js/components/ExternalIcon.jsx b/web/pgadmin/static/js/components/ExternalIcon.jsx index 5f7b94950..8db3c9df2 100644 --- a/web/pgadmin/static/js/components/ExternalIcon.jsx +++ b/web/pgadmin/static/js/components/ExternalIcon.jsx @@ -20,6 +20,7 @@ import Azure from '../../img/azure.svg?svgr'; import SQLFileSvg from '../../img/sql_file.svg?svgr'; import MagicSvg from '../../img/magic.svg?svgr'; import MsAzure from '../../img/ms_azure.svg?svgr'; +import GoogleCloud from '../../img/google-cloud-1.svg?svgr'; export default function ExternalIcon({Icon, ...props}) { return ; @@ -71,15 +72,18 @@ ExpandDialogIcon.propTypes = {style: PropTypes.object}; export const MinimizeDialogIcon = ({style})=>; MinimizeDialogIcon.propTypes = {style: PropTypes.object}; -export const AWSIcon = ({style})=>; +export const AWSIcon = ({style})=>; AWSIcon.propTypes = {style: PropTypes.object}; -export const BigAnimalIcon = ({style})=>; +export const BigAnimalIcon = ({style})=>; BigAnimalIcon.propTypes = {style: PropTypes.object}; -export const AzureIcon = ({style})=>; +export const AzureIcon = ({style})=>; AzureIcon.propTypes = {style: PropTypes.object}; +export const GoogleCloudIcon = ({style})=>; +GoogleCloudIcon.propTypes = {style: PropTypes.object}; + export const SQLFileIcon = ({style})=>; SQLFileIcon.propTypes = {style: PropTypes.object}; diff --git a/web/pgadmin/utils/ajax.py b/web/pgadmin/utils/ajax.py index a011472cd..f298bf0df 100644 --- a/web/pgadmin/utils/ajax.py +++ b/web/pgadmin/utils/ajax.py @@ -182,3 +182,8 @@ def service_unavailable(errormsg=_("Service Unavailable"), info='', result=result, data=data ) + + +def plain_text_response(message=''): + response = Response(message, status=200, mimetype="text/plain") + return response