From f682f06c944004ee2386a2d75a65ed8e16683987 Mon Sep 17 00:00:00 2001 From: Ashesh Vashi Date: Fri, 13 May 2016 08:49:48 +0530 Subject: [PATCH] Adding a background process executor, and observer. We will be using the external utilities like pg_dump, pg_dumpall, pg_restore in background. pgAdmin 4 can be run as a CGI script, hence - it is not good idea to run those utility in a controlled environment. The process executor will run them in background, and we will execute the process executor in detached mode. Now that - the process executor runs in detached mode, we need an observer, which will look at the status of the processes. It also reads output, and error logs on demand. Thanks - Surinder for helping in some of the UI changes. --- web/config.py | 2 +- .../browser/templates/browser/js/messages.js | 4 +- web/pgadmin/misc/bgprocess/__init__.py | 122 +++++ .../misc/bgprocess/process_executor.py | 375 +++++++++++++ web/pgadmin/misc/bgprocess/processes.py | 385 ++++++++++++++ .../misc/bgprocess/static/css/bgprocess.css | 153 ++++++ .../misc/bgprocess/static/js/bgprocess.js | 500 ++++++++++++++++++ web/pgadmin/misc/sql/__init__.py | 6 +- web/pgadmin/model/__init__.py | 19 + .../static/js/alertifyjs/pgadmin.defaults.js | 2 +- web/setup.py | 19 +- 11 files changed, 1578 insertions(+), 9 deletions(-) create mode 100644 web/pgadmin/misc/bgprocess/__init__.py create mode 100644 web/pgadmin/misc/bgprocess/process_executor.py create mode 100644 web/pgadmin/misc/bgprocess/processes.py create mode 100644 web/pgadmin/misc/bgprocess/static/css/bgprocess.css create mode 100644 web/pgadmin/misc/bgprocess/static/js/bgprocess.js diff --git a/web/config.py b/web/config.py index b11b7c6aa..a2da99238 100644 --- a/web/config.py +++ b/web/config.py @@ -151,7 +151,7 @@ MAX_SESSION_IDLE_TIME = 60 # The schema version number for the configuration database # DO NOT CHANGE UNLESS YOU ARE A PGADMIN DEVELOPER!! -SETTINGS_SCHEMA_VERSION = 9 +SETTINGS_SCHEMA_VERSION = 10 # The default path to the SQLite database used to store user accounts and # settings. This default places the file in the same directory as this diff --git a/web/pgadmin/browser/templates/browser/js/messages.js b/web/pgadmin/browser/templates/browser/js/messages.js index 06b4d6b18..30d420460 100644 --- a/web/pgadmin/browser/templates/browser/js/messages.js +++ b/web/pgadmin/browser/templates/browser/js/messages.js @@ -9,7 +9,7 @@ function(_, S, pgAdmin) { var messages = pgBrowser.messages = { 'SERVER_LOST': '{{ _('Connection to the server has been lost.') }}', - 'CLICK_FOR_DETAILED_MSG': '%s

' + '{{ _('Click here for details.')|safe }}', + 'CLICK_FOR_DETAILED_MSG': '{{ _('Click here for details.')|safe }}', 'GENERAL_CATEGORY': '{{ _("General")|safe }}', 'SQL_TAB': '{{ _('SQL') }}', 'SQL_INCOMPLETE': '{{ _('Incomplete definition') }}', @@ -24,7 +24,7 @@ function(_, S, pgAdmin) { 'NODE_HAS_NO_STATISTICS': "{{ _("No statistics are available for the selected object.") }}", 'TRUE': "{{ _("True") }}", 'FALSE': "{{ _("False") }}", - 'NOTE_CTRL_LABEL': "{{ _("Note") }}", + 'NOTE_CTRL_LABEL': "{{ _("Note") }}" }; {% for key in current_app.messages.keys() %} diff --git a/web/pgadmin/misc/bgprocess/__init__.py b/web/pgadmin/misc/bgprocess/__init__.py new file mode 100644 index 000000000..af8ceb3d9 --- /dev/null +++ b/web/pgadmin/misc/bgprocess/__init__.py @@ -0,0 +1,122 @@ +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2016, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## + +""" +A blueprint module providing utility functions for the notify the user about +the long running background-processes. +""" +from flask import url_for +from flask.ext.babel import gettext as _ +from flask.ext.security import login_required + +from pgadmin.utils.ajax import make_response, gone, bad_request, success_return +from pgadmin.utils import PgAdminModule + +from .processes import BatchProcess + +MODULE_NAME = 'bgprocess' + + +class BGProcessModule(PgAdminModule): + + def get_own_javascripts(self): + return [{ + 'name': 'pgadmin.browser.bgprocess', + 'path': url_for('bgprocess.static', filename='js/bgprocess'), + 'when': None + }] + + def get_own_stylesheets(self): + """ + Returns: + list: the stylesheets used by this module. + """ + stylesheets = [ + url_for('bgprocess.static', filename='css/bgprocess.css') + ] + return stylesheets + + def get_own_messages(self): + """ + Returns: + dict: the i18n messages used by this module + """ + return { + 'bgprocess.index': url_for("bgprocess.index"), + 'bgprocess.list': url_for("bgprocess.list"), + 'seconds': _('seconds'), + 'started': _('Started'), + 'START_TIME': _('Start time'), + 'STATUS': _('Status'), + 'EXECUTION_TIME': _('Execution time'), + 'running': _('Running...'), + 'successfully_finished': _("Successfully Finished!"), + 'failed_with_exit_code': _("Failed (Exit code: %%s).") + } + +# Initialise the module +blueprint = BGProcessModule( + MODULE_NAME, __name__, url_prefix='/misc/bgprocess' +) + + +@blueprint.route('/') +@login_required +def index(): + return bad_request(errormsg=_('User can not call this url directly')) + + +@blueprint.route('/status//', methods=['GET']) +@blueprint.route('/status////', methods=['GET']) +@login_required +def status(pid, out=-1, err=-1): + """ + Check the status of the process running in background. + Sends back the output of stdout/stderr + Fetches & sends STDOUT/STDERR logs for the process requested by client + + Args: + pid: Process ID + out: position of the last stdout fetched + err: position of the last stderr fetched + + Returns: + Status of the process and logs (if out, and err not equal to -1) + """ + try: + process = BatchProcess(id=pid) + + return make_response(response=process.status(out, err)) + except LookupError as lerr: + return gone(errormsg=str(lerr)) + + +@blueprint.route('/list/', methods=['GET']) +def list(): + return make_response(response=BatchProcess.list()) + + +@blueprint.route('/acknowledge//', methods=['PUT']) +@login_required +def acknowledge(pid): + """ + User has acknowledge the process + + Args: + pid: Process ID + + Returns: + Positive status + """ + try: + BatchProcess.acknowledge(pid, True) + + return success_return() + except LookupError as lerr: + return gone(errormsg=str(lerr)) diff --git a/web/pgadmin/misc/bgprocess/process_executor.py b/web/pgadmin/misc/bgprocess/process_executor.py new file mode 100644 index 000000000..23e6fdeb1 --- /dev/null +++ b/web/pgadmin/misc/bgprocess/process_executor.py @@ -0,0 +1,375 @@ +# -*- coding: utf-8 -*- + +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2016, The pgAdmin Development Team +# This software is released under the PostgreSQL License +# +########################################################################## + +""" +This python script is responsible for executing a process, and logs its output, +and error in the given output directory. + +We will create a detached process, which executes this script. + +This script will: +* Fetch the configuration from the given database. +* Run the given executable specified in the configuration with the arguments. +* Create log files for both stdout, and stdout. +* Update the start time, end time, exit code, etc in the configuration + database. + +Args: + process_id -- Process id + db_file -- Database file which holds list of processes to be executed + output_directory -- output directory +""" +from __future__ import print_function, unicode_literals + +# To make print function compatible with python2 & python3 +import sys +import os +import argparse +import sqlite3 +from datetime import datetime +from subprocess import Popen, PIPE +from threading import Thread +import csv +import pytz +import codecs + + +# SQLite3 needs all string as UTF-8 +# We need to make string for Python2/3 compatible +if sys.version_info < (3,): + from io import StringIO + + def u(x): + return codecs.unicode_escape_decode(x)[0] +else: + from cStringIO import StringIO + + def u(x): + return x + + +def usage(): + """ + This function will display usage message. + + Args: + None + + Returns: + Displays help message + """ + + help_msg = """ +Usage: + +executer.py [-h|--help] + [-p|--process] Process ID + [-d|--db_file] SQLite3 database file path +""" + print(help_msg) + + +def get_current_time(format='%Y-%m-%d %H:%M:%S.%f %z'): + return datetime.utcnow().replace( + tzinfo=pytz.utc + ).strftime(format) + + +class ProcessLogger(Thread): + """ + This class definition is responsible for capturing & logging + stdout & stderr messages from subprocess + + Methods: + -------- + * __init__(stream_type, configs) + - This method is use to initlize the ProcessLogger class object + + * logging(msg) + - This method is use to log messages in sqlite3 database + + * run() + - Reads the stdout/stderr for messages and sent them to logger + """ + + def __init__(self, stream_type, configs): + """ + This method is use to initialize the ProcessLogger class object + + Args: + stream_type: Type of STD (std) + configs: Process details dict + + Returns: + None + """ + Thread.__init__(self) + self.configs = configs + self.process = None + self.stream = None + self.logger = codecs.open( + os.path.join( + configs['output_directory'], stream_type + ), 'w', "utf-8" + ) + + def attach_process_stream(self, process, stream): + """ + This function will attach a process and its stream with this thread. + + Args: + process: Process + stream: Stream attached with the process + + Returns: + None + """ + self.process = process + self.stream = stream + + def log(self, msg): + """ + This function will update log file + + Args: + msg: message + + Returns: + None + """ + # Write into log file + if self.logger: + if msg: + self.logger.write( + str('{0},{1}').format( + get_current_time(format='%Y%m%d%H%M%S%f'), msg + ) + ) + return True + return False + + def run(self): + if self.process and self.stream: + while True: + nextline = self.stream.readline() + + if nextline: + self.log(nextline) + else: + if self.process.poll() is not None: + break + + def release(self): + if self.logger: + self.logger.close() + self.logger = None + + +def read_configs(data): + """ + This reads SQLite3 database and fetches process details + + Args: + data - configuration details + + Returns: + Process details fetched from database as a dict + """ + if data.db_file is not None and data.process_id is not None: + conn = sqlite3.connect(data.db_file) + c = conn.cursor() + t = (data.process_id,) + + c.execute('SELECT command, arguments FROM process WHERE \ + exit_code is NULL \ + AND pid=?', t) + + row = c.fetchone() + conn.close() + + if row and len(row) > 1: + configs = { + 'pid': data.process_id, + 'cmd': row[0], + 'args': row[1], + 'output_directory': data.output_directory, + 'db_file': data.db_file + } + return configs + else: + return None + else: + raise ValueError("Please verify process id and db_file arguments") + + +def update_configs(kwargs): + """ + This function will updates process stats + + Args: + kwargs - Process configuration details + + Returns: + None + """ + if 'db_file' in kwargs and 'pid' in kwargs: + conn = sqlite3.connect(kwargs['db_file']) + sql = 'UPDATE process SET ' + params = list() + + for param in ['start_time', 'end_time', 'exit_code']: + if param in kwargs: + sql += (',' if len(params) else '') + param + '=? ' + params.append(kwargs[param]) + + if len(params) == 0: + return + + sql += 'WHERE pid=?' + params.append(kwargs['pid']) + + with conn: + c = conn.cursor() + c.execute(sql, params) + conn.commit() + + # Commit & close cursor + conn.close() + else: + raise ValueError("Please verify pid and db_file arguments") + + +def execute(configs): + """ + This function will execute the background process + + Args: + configs: Process configuration details + + Returns: + None + """ + if configs is not None: + command = [configs['cmd']] + args_csv = StringIO(configs['args']) + args_reader = csv.reader(args_csv, delimiter=str(',')) + for args in args_reader: + command = command + args + args = { + 'pid': configs['pid'], + 'db_file': configs['db_file'] + } + + reload(sys) + sys.setdefaultencoding('utf8') + + # Create seprate thread for stdout and stderr + process_stdout = ProcessLogger('out', configs) + process_stderr = ProcessLogger('err', configs) + + try: + # update start_time + args.update({ + 'start_time': get_current_time(), + 'stdout': process_stdout.log, + 'stderr': process_stderr.log + }) + + # Update start time + update_configs(args) + + process = Popen( + command, stdout=PIPE, stderr=PIPE, stdin=PIPE, + shell=(os.name == 'nt'), close_fds=(os.name != 'nt') + ) + + # Attach the stream to the process logger, and start logging. + process_stdout.attach_process_stream(process, process.stdout) + process_stdout.start() + process_stderr.attach_process_stream(process, process.stderr) + process_stderr.start() + + # Join both threads together + process_stdout.join() + process_stderr.join() + + # Child process return code + exitCode = process.wait() + + if exitCode is None: + exitCode = process.poll() + + args.update({'exit_code': exitCode}) + + # Add end_time + args.update({'end_time': get_current_time()}) + + # Fetch last output, and error from process if it has missed. + data = process.communicate() + if data: + if data[0]: + process_stdout.log(data[0]) + if data[1]: + process_stderr.log(data[1]) + + # If executable not found or invalid arguments passed + except OSError as e: + if process_stderr: + process_stderr.log(e.strerror) + else: + print("WARNING: ", e.strerror, file=sys.stderr) + args.update({'end_time': get_current_time()}) + args.update({'exit_code': e.errno}) + + # Unknown errors + except Exception as e: + if process_stderr: + process_stderr.log(str(e)) + else: + print("WARNING: ", str(e), file=sys.stderr) + args.update({'end_time': get_current_time()}) + args.update({'exit_code': -1}) + finally: + # Update the execution end_time, and exit-code. + update_configs(args) + if process_stderr: + process_stderr.release() + process_stderr = None + if process_stdout: + process_stdout.release() + process_stdout = None + + else: + raise ValueError("Please verify configs") + + +if __name__ == '__main__': + # Read command line arguments + parser = argparse.ArgumentParser( + description='Process executor for pgAdmin4' + ) + parser.add_argument( + '-p', '--process_id', help='Process ID', required=True + ) + parser.add_argument( + '-d', '--db_file', help='Configuration Database', required=True + ) + parser.add_argument( + '-o', '--output_directory', + help='Location where the logs will be created', required=True + ) + args = parser.parse_args() + + # Fetch bakcground process details from SQLite3 database file + configs = read_configs(args) + + # Execute the background process + execute(configs) diff --git a/web/pgadmin/misc/bgprocess/processes.py b/web/pgadmin/misc/bgprocess/processes.py new file mode 100644 index 000000000..b995a7656 --- /dev/null +++ b/web/pgadmin/misc/bgprocess/processes.py @@ -0,0 +1,385 @@ +# -*- coding: utf-8 -*- +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2016, The pgAdmin Development Team +# This software is released under the PostgreSQL License +# +########################################################################## + +""" +Introduce a function to run the process executor in detached mode. +""" +from __future__ import print_function, unicode_literals +from abc import ABCMeta, abstractproperty +import csv +from datetime import datetime +from dateutil import parser +import os +from pickle import dumps, loads +import pytz +from subprocess import Popen, PIPE +import sys +import types + +from flask.ext.babel import gettext +from flask.ext.security import current_user + +import config +from pgadmin.model import Process, db + +if sys.version_info < (3,): + from StringIO import StringIO +else: + from cStringIO import StringIO + + +def get_current_time(format='%Y-%m-%d %H:%M:%S.%f %z'): + """ + Generate the current time string in the given format. + """ + return datetime.utcnow().replace( + tzinfo=pytz.utc + ).strftime(format) + + +class IProcessDesc(object): + __metaclass__ = ABCMeta + + @abstractproperty + def message(self): + pass + + @abstractproperty + def details(self): + pass + + +class BatchProcess(object): + + def __init__(self, **kwargs): + + self.id = self.desc = self.cmd = self.args = self.log_dir = \ + self.stdout = self.stderr = self.stime = self.etime = \ + self.ecode = None + + if 'id' in kwargs: + self._retrieve_process(kwargs['id']) + else: + self._create_process(kwargs['desc'], kwargs['cmd'], kwargs['args']) + + def _retrieve_process(self, _id): + p = Process.query.filter_by(pid=_id, user_id=current_user.id).first() + + if p is None: + raise LookupError(gettext( + "Couldn't find the process specified by the id!" + )) + + # ID + self.id = _id + # Description + self.desc = loads(p.desc) + # Status Acknowledged time + self.atime = p.acknowledge + # Command + self.cmd = p.command + # Arguments + self.args = p.arguments + # Log Directory + self.log_dir = p.logdir + # Standard ouput log file + self.stdout = os.path.join(p.logdir, 'out') + # Standard error log file + self.stderr = os.path.join(p.logdir, 'err') + # Start time + self.stime = p.start_time + # End time + self.etime = p.end_time + # Exit code + self.ecode = p.exit_code + + def _create_process(self, _desc, _cmd, _args): + ctime = get_current_time(format='%y%m%d%H%M%S%f') + log_dir = os.path.join( + config.SESSION_DB_PATH, 'process_logs' + ) + + def random_number(size): + import random + import string + + return ''.join( + random.choice( + string.ascii_uppercase + string.digits + ) for _ in range(size) + ) + + created = False + size = 0 + id = ctime + while not created: + try: + id += random_number(size) + log_dir = os.path.join(log_dir, id) + size += 1 + if not os.path.exists(log_dir): + os.makedirs(log_dir, int('700', 8)) + created = True + except OSError as oe: + import errno + if oe.errno != errno.EEXIST: + raise + + # ID + self.id = ctime + # Description + self.desc = _desc + # Status Acknowledged time + self.atime = None + # Command + self.cmd = _cmd + # Log Directory + self.log_dir = log_dir + # Standard ouput log file + self.stdout = os.path.join(log_dir, 'out') + # Standard error log file + self.stderr = os.path.join(log_dir, 'err') + # Start time + self.stime = None + # End time + self.etime = None + # Exit code + self.ecode = None + + # Arguments + args_csv_io = StringIO() + csv_writer = csv.writer( + args_csv_io, delimiter=str(','), quoting=csv.QUOTE_MINIMAL + ) + csv_writer.writerow(_args) + self.args = args_csv_io.getvalue().strip(str('\r\n')) + + j = Process( + pid=int(id), command=_cmd, arguments=self.args, logdir=log_dir, + desc=dumps(self.desc), user_id=current_user.id + ) + db.session.add(j) + db.session.commit() + + def start(self): + if self.stime is not None: + if self.etime is None: + raise Exception(gettext('Process has already been started!')) + raise Exception(gettext( + 'Process has already been finished, it can not be restared!' + )) + + executor = os.path.join( + os.path.dirname(__file__), 'process_executor.py' + ) + + p = None + cmd = [ + sys.executable or 'python', + executor, + '-p', self.id, + '-o', self.log_dir, + '-d', config.SQLITE_PATH + ] + + if os.name == 'nt': + p = Popen( + cmd, stdout=None, stderr=None, stdin=None, close_fds=True, + shell=False, creationflags=0x00000008 + ) + else: + def preexec_function(): + import signal + # Detaching from the parent process group + os.setpgrp() + # Explicitly ignoring signals in the child process + signal.signal(signal.SIGINT, signal.SIG_IGN) + + p = Popen( + cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, close_fds=True, + shell=False, preexec_fn=preexec_function + ) + + self.ecode = p.poll() + if self.ecode is not None: + # TODO:: Couldn't start execution + pass + + + def status(self, out=0, err=0): + ctime = get_current_time(format='%Y%m%d%H%M%S%f') + + stdout = [] + stderr = [] + out_completed = err_completed = False + process_output = (out != -1 and err != -1) + + def read_log(logfile, log, pos, ctime, check=True): + completed = True + lines = 0 + + if not os.path.isfile(logfile): + return 0 + + with open(logfile, 'r') as stream: + stream.seek(pos) + for line in stream: + logtime = StringIO() + idx = 0 + for c in line: + idx += 1 + if c == ',': + break + logtime.write(c) + logtime = logtime.getvalue() + + if check and logtime > ctime: + completed = False + break + if lines == 5120: + ctime = logtime + completed = False + break + + lines += 1 + pos = stream.tell() + log.append([logtime, line[idx:]]) + + return pos, completed + + if process_output: + out, out_completed = read_log( + self.stdout, stdout, out, ctime, True + ) + err, err_completed = read_log( + self.stderr, stderr, err, ctime, True + ) + + j = Process.query.filter_by( + pid=self.id, user_id=current_user.id + ).first() + + execution_time = None + + if j is not None: + self.stime = j.start_time + self.etime = j.end_time + self.ecode = j.exit_code + + if self.stime is not None: + stime = parser.parse(self.stime) + etime = parser.parse(self.etime or get_current_time()) + + execution_time = (etime - stime).total_seconds() + + if process_output and self.ecode is not None and ( + len(stdout) + len(stderr) < 3073 + ): + out, out_completed = read_log( + self.stdout, stdout, out, ctime, False + ) + err, err_completed = read_log( + self.stderr, stderr, err, ctime, False + ) + else: + out_completed = err_completed = False + + if out == -1 or err == -1: + return { + 'exit_code': self.ecode, + 'execution_time': execution_time + } + + return { + 'out': {'pos': out, 'lines': stdout, 'done': out_completed}, + 'err': {'pos': err, 'lines': stderr, 'done': err_completed}, + 'exit_code': self.ecode, + 'execution_time': execution_time + } + + @staticmethod + def list(): + processes = Process.query.filter_by(user_id=current_user.id) + + res = [] + for p in processes: + if p.start_time is None or p.acknowledge is not None: + continue + execution_time = None + + stime = parser.parse(p.start_time) + etime = parser.parse(p.end_time or get_current_time()) + + execution_time = (etime - stime).total_seconds() + desc = loads(p.desc) + details = desc + + if not isinstance(desc, types.StringTypes): + details = desc.details + desc = desc.message + + res.append({ + 'id': p.pid, + 'desc': desc, + 'details': details, + 'stime': ( + stime - datetime( + 1970, 1, 1, tzinfo=pytz.utc + ) + ).total_seconds(), + 'etime': p.end_time, + 'exit_code': p.exit_code, + 'acknowledge': p.acknowledge, + 'execution_time': execution_time + }) + + return res + + @staticmethod + def acknowledge(_pid, _release): + p = Process.query.filter_by( + user_id=current_user.id, pid=_pid + ).first() + + if p is None: + raise LookupError(gettext( + "Couldn't find the process specified by the id!" + )) + + if _release: + import shutil + shutil.rmtree(p.logdir, True) + db.session.delete(p) + else: + p.acknowledge = get_current_time() + + db.session.commit() + + @staticmethod + def release(pid=None): + import shutil + processes = None + + if pid is not None: + processes = Process.query.filter_by( + user_id=current_user.id, pid=pid + ) + else: + processes = Process.query.filter_by( + user_id=current_user.id, + acknowledge=None + ) + + if processes: + for p in processes: + shutil.rmtree(p.logdir, True) + + db.session.delete(p) + db.session.commit() diff --git a/web/pgadmin/misc/bgprocess/static/css/bgprocess.css b/web/pgadmin/misc/bgprocess/static/css/bgprocess.css new file mode 100644 index 000000000..d4e7e075a --- /dev/null +++ b/web/pgadmin/misc/bgprocess/static/css/bgprocess.css @@ -0,0 +1,153 @@ +.ajs-bg-bgprocess.ajs-visible { + padding: 0px !important; +} + +.ajs-bg-bgprocess > .pg-bg-bgprocess { + background-color: #31708F; + color: #FFFFFF; + padding: 0px; + border-radius: 5px; + text-align: left; +} + +.ajs-bg-bgprocess .col-xs-12 { + padding-right: 5px; + padding-left: 5px; +} + +.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-notify-header { + background-color: #1B4A5A; + margin-top: 0px; + margin-bottom: 5px; + font-weight: 900; + padding: 5px; + white-space: pre-wrap; + border-top-left-radius: 5px; + border-top-right-radius: 5px; +} + +.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-notify-body { + font-family: monospace; +} + +.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-status { + padding: 2px; + font-weight: 700; + margin: 5px; + width: calc(100% - 10px); + text-align: center; + border-radius: 2px; + -moz-border-radius: 2px; + -webkit-border-radius: 2px; +} + +.pg-bg-process-logs { + width: 100%; +} + +.pg-bg-etime { + width: 100%; + display: block; + font-size: 95%; + padding: 5px; + font-weight: bold; + color: #fff; + text-align: center; + white-space: nowrap; + vertical-align: baseline; + border-radius: .25em; + background-color: #777; +} + +.pg-bg-click { + color: rgb(221, 194, 174); + text-decoration: underline; + cursor: pointer; +} + +.pg-bg-click:hover { + color: darkblue; +} + +.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-status.bg-success, +.bg-process-status .bg-bgprocess-success { + color: green; +} + +.bg-process-status .bg-bgprocess-failed { + color: red; +} + +.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-status.bg-failed { + color: black; + background-color: #E99595; +} + +.pg-panel-content div.bg-process-watcher.col-xs-12 { + height: 100%; + padding: 0px; + WebkitTransition: all 1s; + transition: all 1s; +} + +ol.pg-bg-process-logs { + padding: 15px 15px 15px 60px; + height: 100%; + overflow: auto; + width: 100%; + background: #000; +} + +.pg-bg-res-out, .pg-bg-res-err { + background-color: #000; + padding-left: 10px; + white-space: pre-wrap; + font-size: 12px; +} + +.pg-bg-res-out { + color: rgb(0, 157, 207); +} + +.pg-bg-res-err { + color: rgba(212, 27, 57, 0.81); +} + +.pg-panel-content .bg-process-details { + padding: 10px 15px; + min-height: 70px; + color: #000; +} + +.pg-panel-content .bg-process-stats p{ + display: inline; + padding-left: 5px; + margin-bottom: 0; + font-size: 13px; +} + +.pg-panel-content .bg-process-footer { + border-top: 1px solid #ccc; + padding: 10px 15px; + position: absolute; + bottom: 0; + background-color: white; +} + +.pg-panel-content .bg-process-footer p { + display: inline; + padding-left: 5px; + font-size: 13px; +} + +.pg-panel-content .bg-process-footer b, .bg-process-stats span b { + color: #285173; +} + +.bg-process-footer .bg-process-status { + padding-left: 0; +} + +.bg-process-footer .bg-process-exec-time { + padding-right: 0; +} diff --git a/web/pgadmin/misc/bgprocess/static/js/bgprocess.js b/web/pgadmin/misc/bgprocess/static/js/bgprocess.js new file mode 100644 index 000000000..6caf084ef --- /dev/null +++ b/web/pgadmin/misc/bgprocess/static/js/bgprocess.js @@ -0,0 +1,500 @@ +define( + ['underscore', 'underscore.string', 'jquery', 'pgadmin.browser', 'alertify', 'pgadmin.browser.messages'], +function(_, S, $, pgBrowser, alertify, pgMessages) { + + pgBrowser.BackgroundProcessObsorver = pgBrowser.BackgroundProcessObsorver || {}; + + if (pgBrowser.BackgroundProcessObsorver.initialized) { + return pgBrowser.BackgroundProcessObsorver; + } + + var BGProcess = function(info, notify) { + var self = this; + setTimeout( + function() { + self.initialize.apply(self, [info, notify]); + }, 1 + ); + }; + + _.extend( + BGProcess.prototype, { + initialize: function(info, notify) { + _.extend(this, { + details: false, + notify: (_.isUndefined(notify) || notify), + curr_status: null, + state: 0, // 0: NOT Started, 1: Started, 2: Finished + completed: false, + + id: info['id'], + desc: null, + detailed_desc: null, + stime: null, + exit_code: null, + acknowledge: info['acknowledge'], + execution_time: null, + out: null, + err: null, + + notifier: null, + container: null, + panel: null, + logs: $('
    ', {class: 'pg-bg-process-logs'}) + }); + + if (this.notify) { + pgBrowser.Events && pgBrowser.Events.on( + 'pgadmin-bgprocess:started:' + this.id, + function(process) { + if (!process.notifier) + process.show.apply(process); + } + ); + pgBrowser.Events && pgBrowser.Events.on( + 'pgadmin-bgprocess:finished:' + this.id, + function(process) { + if (!process.notifier) + process.show.apply(process); + } + ) + } + var self = this; + + setTimeout( + function() { + self.update.apply(self, [info]); + }, 1 + ); + }, + + url: function(type) { + var base_url = pgMessages['bgprocess.index'], + url = base_url; + + switch (type) { + case 'status': + url = S('%sstatus/%s/').sprintf(base_url, this.id).value(); + if (this.details) { + url = S('%s%s/%s/').sprintf( + url, (this.out && this.out.pos) || 0, + (this.err && this.err.pos) || 0 + ).value(); + } + break; + case 'info': + case 'acknowledge': + url = S('%s%s/%s/').sprintf(base_url, type, this.id).value(); + break; + } + + return url; + }, + + update: function(data) { + var self = this, + out = [], + err = [], + idx = 0; + + if ('stime' in data) + self.stime = new Date(data.stime); + + if ('execution_time' in data) + self.execution_time = parseFloat(data.execution_time); + + if ('desc' in data) + self.desc = data.desc; + + if ('details' in data) + self.detailed_desc = data.details; + + if ('exit_code' in data) + self.exit_code = data.exit_code; + + if ('out' in data) { + self.out = data.out && data.out.pos; + self.completed = data.out.done; + + if (data.out && data.out.lines) { + data.out.lines.sort(function(a, b) { return a[0] < b[0]; }); + out = data.out.lines; + } + } + + if ('err' in data) { + self.err = data.err && data.err.pos; + self.completed = (self.completed && data.err.done); + + if (data.err && data.err.lines) { + data.err.lines.sort(function(a, b) { return a[0] < b[0]; }); + err = data.err.lines; + } + } + + var io = ie = 0; + + while (io < out.length && ie < err.length && + self.logs[0].children.length < 5120) { + if (out[io][0] < err[ie][0]){ + self.logs.append( + $('
  1. ', {class: 'pg-bg-res-out'}).text(out[io++][1]) + ); + } else { + self.logs.append( + $('
  2. ', {class: 'pg-bg-res-err'}).text(err[ie++][1]) + ); + } + } + + while (io < out.length && self.logs[0].children.length < 5120) { + self.logs.append( + $('
  3. ', {class: 'pg-bg-res-out'}).text(out[io++][1]) + ); + } + + while (ie < err.length && self.logs[0].children.length < 5120) { + self.logs.append( + $('
  4. ', {class: 'pg-bg-res-err'}).text(err[ie++][1]) + ); + } + + if (self.logs[0].children.length >= 5120) { + self.completed = true; + } + + if (self.stime) { + self.curr_status = pgMessages['started']; + + if (self.execution_time >= 2) { + self.curr_status = pgMessages['running']; + } + + if ('execution_time' in data) { + self.execution_time = self.execution_time + ' ' + + pgMessages['seconds']; + } + + if (!_.isNull(self.exit_code)) { + if (self.exit_code == 0) { + self.curr_status = pgMessages['successfully_finished']; + } else { + self.curr_status = S( + pgMessages['failed_with_exit_code'] + ).sprintf(String(self.exit_code)).value(); + } + } + + if (self.state == 0 && self.stime) { + self.state = 1; + pgBrowser.Events && pgBrowser.Events.trigger( + 'pgadmin-bgprocess:started:' + self.id, self, self + ); + } + + if (self.state == 1 && !_.isNull(self.exit_code)) { + self.state = 2; + pgBrowser.Events && pgBrowser.Events.trigger( + 'pgadmin-bgprocess:finished:' + self.id, self, self + ); + } + + setTimeout(function() {self.show.apply(self)}, 10); + } + + if (self.state != 2 || (self.details && !self.completed)) { + setTimeout( + function() { + self.status.apply(self); + }, 1000 + ); + } + }, + + status: function() { + var self = this; + + $.ajax({ + typs: 'GET', + timeout: 30000, + url: self.url('status'), + cache: false, + async: true, + contentType: "application/json", + success: function(res) { + setTimeout(function() { self.update(res); }, 10); + }, + error: function(res) { + // Try after some time + setTimeout(function() { self.update(res); }, 10000); + } + }); + }, + + show: function() { + var self = this; + + if (self.notify && !self.details) { + if (!self.notifier) { + var content = $('
    ').append( + $('
    ', { + class: "col-xs-12 h3 pg-bg-notify-header" + }).text( + self.desc + ) + ).append( + $('
    ', {class: 'pg-bg-notify-body' }).append( + $('
    ', {class: 'pg-bg-start col-xs-12' }).append( + $('
    ').text(self.stime.toString()) + ).append( + $('
    ') + ) + ) + ), + for_details = $('
    ', { + class: "col-xs-12 text-center pg-bg-click" + }).text(pgMessages.CLICK_FOR_DETAILED_MSG).appendTo(content), + status = $('
    ', { + class: "pg-bg-status col-xs-12 " + ((self.exit_code === 0) ? + 'bg-success': (self.exit_code == 1) ? + 'bg-failed' : '') + }).appendTo(content); + + self.container = content; + self.notifier = alertify.notify( + content.get(0), 'bg-bgprocess', 0, null + ); + + for_details.on('click', function(ev) { + ev = ev || window.event; + ev.cancelBubble = true; + ev.stopPropagation(); + + this.notifier.dismiss(); + this.notifier = null; + + this.show_detailed_view.apply(this); + }.bind(self)); + + // Do not close the notifier, when clicked on the container, which + // is a default behaviour. + content.on('click', function(ev) { + ev = ev || window.event; + ev.cancelBubble = true; + ev.stopPropagation(); + + return; + }); + } + // TODO:: Formatted execution time + self.container.find('.pg-bg-etime').empty().text( + String(self.execution_time) + ); + self.container.find('.pg-bg-status').empty().append( + self.curr_status + ) + } + }, + + show_detailed_view: function() { + var self = this, + panel = this.panel = + pgBrowser.BackgroundProcessObsorver.create_panel(); + + panel.title('Process Watcher - ' + self.desc); + panel.focus(); + + var container = panel.$container, + status_class = ( + (self.exit_code === 0) ? + 'bg-bgprocess-success': (self.exit_code == 1) ? + 'bg-bgprocess-failed' : '' + ), + $logs = container.find('.bg-process-watcher'), + $header = container.find('.bg-process-details'), + $footer = container.find('.bg-process-footer'); + + + // set logs + $logs.html(self.logs); + + // set bgprocess detailed description + $header.find('.bg-detailed-desc').html(self.detailed_desc); + + // set bgprocess start time + $header.find('.bg-process-stats .bgprocess-start-time').html(self.stime); + + // set status + $footer.find('.bg-process-status p').addClass( + status_class + ).html( + self.curr_status + ); + + // set bgprocess execution time + $footer.find('.bg-process-exec-time p').html(self.execution_time); + + self.details = true; + setTimeout( + function() { + self.status.apply(self); + }, 1000 + ); + + var resize_log_container = function($logs, $header, $footer) { + var h = $header.outerHeight() + $footer.outerHeight(); + $logs.css('padding-bottom', h); + }.bind(panel, $logs, $header, $footer); + + panel.on(wcDocker.EVENT.RESIZED, resize_log_container); + panel.on(wcDocker.EVENT.ATTACHED, resize_log_container); + panel.on(wcDocker.EVENT.DETACHED, resize_log_container); + + resize_log_container(); + + panel.on(wcDocker.EVENT.CLOSED, function(process) { + process.panel = null; + + process.details = false; + if (process.exit_code != null) { + process.acknowledge_server.apply(process); + } + }.bind(panel, this)); + }, + + acknowledge_server: function() { + var self = this; + $.ajax({ + type: 'PUT', + timeout: 30000, + url: self.url('acknowledge'), + cache: false, + async: true, + contentType: "application/json", + success: function(res) { + return; + }, + error: function(res) { + } + }); + } + }); + + _.extend( + pgBrowser.BackgroundProcessObsorver, { + bgprocesses: {}, + init: function() { + var self = this; + + if (self.initialized) { + return; + } + self.initialized = true; + + setTimeout( + function() { + self.update_process_list.apply(self); + }, 1000 + ); + + pgBrowser.Events.on( + 'pgadmin-bgprocess:created', + function() { + setTimeout( + function() { + pgBrowser.BackgroundProcessObsorver.update_process_list(); + }, 1000 + ); + } + ) + }, + + update_process_list: function() { + var observer = this; + + $.ajax({ + typs: 'GET', + timeout: 30000, + url: pgMessages['bgprocess.list'], + cache: false, + async: true, + contentType: "application/json", + success: function(res) { + if (!res) { + // FIXME:: + // Do you think - we should call the list agains after some + // interval? + return; + } + for (idx in res) { + var process = res[idx]; + if ('id' in process) { + if (!(process.id in observer.bgprocesses)) { + observer.bgprocesses[process.id] = new BGProcess(process); + } + } + } + }, + error: function(res) { + // FIXME:: What to do now? + } + }); + }, + + create_panel: function() { + this.register_panel(); + + return pgBrowser.docker.addPanel( + 'bg_process_watcher', + wcDocker.DOCK.FLOAT, + null, { + w: (screen.width < 700 ? + screen.width * 0.95 : screen.width * 0.5), + h: (screen.height < 500 ? + screen.height * 0.95 : screen.height * 0.5), + x: (screen.width < 700 ? '2%' : '25%'), + y: (screen.height < 500 ? '2%' : '25%') + }); + }, + + register_panel: function() { + var w = pgBrowser.docker, + panels = w.findPanels('bg_process_watcher'); + + if (panels && panels.length >= 1) + return; + + var p = new pgBrowser.Panel({ + name: 'bg_process_watcher', + showTitle: true, + isCloseable: true, + isPrivate: true, + content: '
    '+ + '

    '+ + '
    '+ + '' + pgMessages['START_TIME'] + ':'+ + '

    '+ + '
    '+ + '
    '+ + '
    '+ + '
    '+ + '
    '+ + '
    '+ + '' + pgMessages['STATUS'] + ':

    '+ + '
    '+ + '
    '+ + '
    '+ + '' + pgMessages['EXECUTION_TIME'] + ':

    '+ + '
    '+ + '
    '+ + '
    ', + onCreate: function(myPanel, $container) { + $container.addClass('pg-no-overflow'); + } + }); + p.load(pgBrowser.docker); + } + }); + + return pgBrowser.BackgroundProcessObsorver; +}); diff --git a/web/pgadmin/misc/sql/__init__.py b/web/pgadmin/misc/sql/__init__.py index 131a3669f..90609aa49 100644 --- a/web/pgadmin/misc/sql/__init__.py +++ b/web/pgadmin/misc/sql/__init__.py @@ -9,10 +9,8 @@ """A blueprint module providing utility functions for the application.""" -import datetime -from flask import session, current_app, url_for +from flask import url_for from pgadmin.utils import PgAdminModule -import pgadmin.utils.driver as driver MODULE_NAME = 'sql' @@ -23,7 +21,7 @@ class SQLModule(PgAdminModule): 'name': 'pgadmin.browser.object_sql', 'path': url_for('sql.static', filename='js/sql'), 'when': None - }] + }] # Initialise the module blueprint = SQLModule(MODULE_NAME, __name__, url_prefix='/misc/sql') diff --git a/web/pgadmin/model/__init__.py b/web/pgadmin/model/__init__.py index b79f085b2..b8bb2560c 100644 --- a/web/pgadmin/model/__init__.py +++ b/web/pgadmin/model/__init__.py @@ -168,3 +168,22 @@ class DebuggerFunctionArguments(db.Model): nullable=False) value = db.Column(db.String(), nullable=True) + + +class Process(db.Model): + """Define the Process table.""" + __tablename__ = 'process' + pid = db.Column(db.String(), nullable=False, primary_key=True) + user_id = db.Column( + db.Integer, + db.ForeignKey('user.id'), + nullable=False + ) + command = db.Column(db.String(), nullable=False) + desc = db.Column(db.String(), nullable=False) + arguments = db.Column(db.String(), nullable=True) + logdir = db.Column(db.String(), nullable=True) + start_time = db.Column(db.String(), nullable=True) + end_time = db.Column(db.String(), nullable=True) + exit_code = db.Column(db.Integer(), nullable=True) + acknowledge = db.Column(db.String(), nullable=True) diff --git a/web/pgadmin/static/js/alertifyjs/pgadmin.defaults.js b/web/pgadmin/static/js/alertifyjs/pgadmin.defaults.js index 30a34d806..591cd084f 100644 --- a/web/pgadmin/static/js/alertifyjs/pgadmin.defaults.js +++ b/web/pgadmin/static/js/alertifyjs/pgadmin.defaults.js @@ -110,7 +110,7 @@ function(alertify, S) { if (contentType.indexOf('text/html') == 0) { alertify.notify( S( - window.pgAdmin.Browser.messages.CLICK_FOR_DETAILED_MSG + '%s

    ' + window.pgAdmin.Browser.messages.CLICK_FOR_DETAILED_MSG ).sprintf(promptmsg).value(), type, 0, function() { alertify.pgIframeDialog().show().set({ frameless: false }).set('pg_msg', msg); }); diff --git a/web/setup.py b/web/setup.py index 01e69a145..185d02a13 100644 --- a/web/setup.py +++ b/web/setup.py @@ -62,7 +62,7 @@ account:\n""") # Setup Flask-Security user_datastore = SQLAlchemyUserDatastore(db, User, Role) - security = Security(app, user_datastore) + Security(app, user_datastore) with app.app_context(): password = encrypt_password(p1) @@ -230,6 +230,23 @@ CREATE TABLE IF NOT EXISTS debugger_function_arguments ( use_default INTEGER NOT NULL CHECK (use_default >= 0 AND use_default <= 1) , value TEXT, PRIMARY KEY (server_id, database_id, schema_id, function_id, arg_id) + )""") + + if int(version.value) < 10: + db.engine.execute(""" +CREATE TABLE process( + user_id INTEGER NOT NULL, + pid TEXT NOT NULL, + desc TEXT NOT NULL, + command TEXT NOT NULL, + arguments TEXT, + start_time TEXT, + end_time TEXT, + logdir TEXT, + exit_code INTEGER, + acknowledge TEXT, + PRIMARY KEY(pid), + FOREIGN KEY(user_id) REFERENCES user (id) )""") # Finally, update the schema version