From 5ff0139114d700958a677775bd1018833a8c3a27 Mon Sep 17 00:00:00 2001 From: "Gavin M. Roy" Date: Tue, 19 Aug 2014 16:53:56 -0400 Subject: [PATCH 1/8] Add importlib and fix MANIFEST.in, ignore *.pyc --- .gitignore | 2 +- MANIFEST.in | 4 ++-- requirements-2.6.txt | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 1d0ef72..ca42c40 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .DS_Store -.pyc build dist +*.pyc *.egg-info diff --git a/MANIFEST.in b/MANIFEST.in index f0f644e..c1a7121 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -LICENSE -README.md +include LICENSE +include README.md diff --git a/requirements-2.6.txt b/requirements-2.6.txt index 09dae77..52981dd 100644 --- a/requirements-2.6.txt +++ b/requirements-2.6.txt @@ -1,3 +1,4 @@ argparse +importlib logutils unittest2 From a9a039a79229287fcb374720772470c17d3d2d46 Mon Sep 17 00:00:00 2001 From: "Gavin M. Roy" Date: Tue, 19 Aug 2014 16:54:17 -0400 Subject: [PATCH 2/8] Remove tornado requirement, add importlib --- setup.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 6898445..b40220b 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,15 @@ -from setuptools import setup import os -import platform +from setuptools import setup +import sys -requirements = ['tornado'] +requirements = [] tests_require = ['coverage', 'coveralls', 'mock', 'nose'] # Requirements for Python 2.6 -(major, minor, rev) = platform.python_version_tuple() -if float('%s.%s' % (major, minor)) < 2.7: +version = sys.version_info +if (version.major, version.minor) < (2, 7): requirements.append('argparse') + requirements.append('importlib') requirements.append('logutils') tests_require.append('unittest2') @@ -18,7 +19,7 @@ setup(name='sprockets', description=('A modular, loosely coupled micro-framework built on top ' 'of Tornado simplifying the creation of web applications ' 'and RabbitMQ workers'), - entry_points={'console_scripts': ['sprockets=sprockets:main']}, + entry_points={'console_scripts': ['sprockets=sprockets.cli:main']}, author='AWeber Communications', url='https://github.com/sprockets/sprockets', install_requires=requirements, From e20765f6ad4960e4d9b3e8fa439c12fc126ad5bd Mon Sep 17 00:00:00 2001 From: "Gavin M. Roy" Date: Fri, 22 Aug 2014 14:52:52 -0400 Subject: [PATCH 3/8] Initial commit, missing tests, but functional --- setup.py | 4 +- sprockets/__init__.py | 20 +++ sprockets/cli.py | 291 ++++++++++++++++++++++++++++++++++++++++++ sprockets/daemon.py | 267 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 579 insertions(+), 3 deletions(-) create mode 100644 sprockets/__init__.py create mode 100644 sprockets/cli.py create mode 100644 sprockets/daemon.py diff --git a/setup.py b/setup.py index b40220b..853d863 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import os from setuptools import setup import sys @@ -13,7 +12,6 @@ if (version.major, version.minor) < (2, 7): requirements.append('logutils') tests_require.append('unittest2') - setup(name='sprockets', version='0.1.0', description=('A modular, loosely coupled micro-framework built on top ' @@ -47,4 +45,4 @@ setup(name='sprockets', 'Topic :: Software Development :: Libraries :: Python Modules'], test_suite='nose.collector', tests_require=tests_require, - zip_safe=True) + zip_safe=False) diff --git a/sprockets/__init__.py b/sprockets/__init__.py new file mode 100644 index 0000000..9c57ee9 --- /dev/null +++ b/sprockets/__init__.py @@ -0,0 +1,20 @@ +""" +Sprockets +========= +A loosely coupled framework built on top of Tornado. Take what you need to +build awesome applications. + +""" +version_info = (0, 0, 0) +__version__ = '.'.join(str(v) for v in version_info) + +import logging + +# Ensure there is a NullHandler for logging +try: + from logging import NullHandler +except ImportError: + # Not available in Python 2.6 + class NullHandler(logging.Handler): + def emit(self, record): + pass diff --git a/sprockets/cli.py b/sprockets/cli.py new file mode 100644 index 0000000..78a8bea --- /dev/null +++ b/sprockets/cli.py @@ -0,0 +1,291 @@ +""" +Sprockets CLI +============= +The sprockets CLI interface for running applications. Applications are meant +to be run by a controller that is managed by the sprockets CLI interface. + +The sprockets CLI interface loads controller applications that are registered +using setuptools entry points. + +Each controller is expected to expose at least a `main(application, args)` +method that would be invoked when starting the application. Additional, a +controller can implement a `add_cli_arguments(parser)` method that will be +invoked when setting up the command line parameters. This allows controllers +to inject configuration directives into the cli. + +Applications can be a python package or module and if they are registered +to a specific controller, can be referenced by an alias. + +""" +import argparse +import importlib +import logging +import string +import sys + +# import logutils for Python 2.6 or logging.config for later versions +sys_version = sys.version_info +if (sys_version.major, sys_version.minor) < (2, 7): + import logutils.dictconfig as logging_config +else: + from logging import config as logging_config + +import pkg_resources + +from sprockets import daemon +from sprockets import __version__ + +DESCRIPTION = 'Available sprockets application controllers' + +# Logging formatters +SYSLOG_FORMAT = ('%(levelname)s ' + '%(name)s.%(funcName)s(): %(message)s') + +VERBOSE_FORMAT = ('%(levelname) -10s %(asctime)s %(process)-6d ' + '%(processName) -20s %(name) -20s ' + '%(funcName) -20s L%(lineno)-6d: %(message)s') + +# Base logging configuration +LOGGING = {'disable_existing_loggers': True, + 'filters': {}, + 'formatters': {'syslog': {'format': SYSLOG_FORMAT}, + 'verbose': {'datefmt': '%Y-%m-%d %H:%M:%S', + 'format': VERBOSE_FORMAT}}, + 'handlers': {'console': {'class': 'logging.StreamHandler', + 'formatter': 'verbose'}, + 'syslog': {'class': 'logging.handlers.SysLogHandler', + 'formatter': 'syslog'}}, + 'incremental': False, + 'loggers': {'sprockets': {'handlers': ['console'], + 'level': logging.WARNING, + 'propagate': True}}, + 'root': {'handlers': [], + 'level': logging.CRITICAL, + 'propagate': True}, + 'version': 1} + +LOGGER = logging.getLogger(__name__) + + +class CLI(object): + """The core Sprockets CLI application providing argument parsing and + logic for starting a controller. + + The package or module for an application is passed into + + """ + + CONTROLLERS = 'sprockets.controller' + + def __init__(self): + self._controllers = self._get_controllers() + self._arg_parser = argparse.ArgumentParser() + self._add_cli_args() + self._args = self._arg_parser.parse_args() + + def run(self): + """Evaluate the command line arguments, performing the appropriate + actions so the application can be started. + + """ + # The list command prevents any other processing of args + if self._args.list: + self._print_installed_apps(self._args.controller) + + # Should be starting an application + else: + + # If app is not specified at this point, raise an error + if not self._args.application: + sys.stderr.write('\nerror: application not specified\n\n') + self._arg_parser.print_help() + sys.exit(-1) + + # Fork into the background as a daemon if instructed to do so + if self._args.daemonize: + try: + with daemon.Daemon(self) as obj: + obj.start() + except (OSError, ValueError) as error: + sys.stderr.write('\nerror: could not start %s (%s)\n\n' % + (sys.argv[0], error)) + sys.exit(1) + + else: + # Start the application directly, no forking involved + self.start() + + def start(self): + """Invoked once the application is ready to be started. If the cli + args indicate that the application should be a daemon, it will already + be forked at this point. + + """ + # If it's a registered app reference by name, get the module name + app_module = self._get_application_module(self._args.controller, + self._args.application) + + # Configure logging based upon the flags + self._configure_logging(app_module, + self._args.verbose, + self._args.syslog) + + # Try and run the controller + try: + self._controllers[self._args.controller].main(app_module, + self._args) + except TypeError as error: + sys.stderr.write('error: could not start the %s controller for %s' + ': %s\n\n' % (self._args.controller, + app_module, + str(error))) + sys.exit(-1) + + def _add_cli_args(self): + """Add the cli arguments to the argument parser.""" + + # Optional cli arguments + self._arg_parser.add_argument('-l', '--list', + action='store_true', + help='List installed sprockets apps') + + self._arg_parser.add_argument('-d', '--daemonize', + action='store_true', + help='Fork into a background process') + + self._arg_parser.add_argument('-s', '--syslog', + action='store_true', + help='Log to syslog') + + self._arg_parser.add_argument('-v', '--verbose', + action='count', + help='Verbose logging output') + + self._arg_parser.add_argument('--version', + action='version', + version='sprockets v%s ' % __version__) + + # Controller sub-parser + subparsers = self._arg_parser.add_subparsers(dest='controller', + help=DESCRIPTION) + + # Iterate through the controllers and add their cli arguments + for key in self._controllers: + help_text = self._get_controller_help(key) + sub_parser = subparsers.add_parser(key, help=help_text) + try: + self._controllers[key].add_cli_arguments(sub_parser) + except AttributeError: + LOGGER.debug('%s missing add_cli_arguments()', key) + + # The application argument + self._arg_parser.add_argument('application', + action="store", + help='The sprockets app to run') + + @staticmethod + def _configure_logging(application, verbosity=0, syslog=False): + """Configure logging for the application, setting the appropriate + verbosity and adding syslog if it's enabled. + + :param str application: The application module/package name + :param int verbosity: 1 == INFO, 2 == DEBUG + :param bool syslog: Enable the syslog handler + + """ + # Create a new copy of the logging config that will be modified + config = dict(LOGGING) + + # Increase the logging verbosity + if verbosity == 1: + config['loggers']['sprockets']['level'] = logging.INFO + elif verbosity == 2: + config['loggers']['sprockets']['level'] = logging.DEBUG + + # Add syslog if it's enabled + if syslog: + config['loggers']['sprockets']['handlers'].append('syslog') + + # Copy the sprockets logger to the application + config['loggers'][application] = dict(config['loggers']['sprockets']) + + # Configure logging + logging_config.dictConfig(config) + + def _get_application_module(self, controller, application): + """Return the module for an application. If it's a entry-point + registered application name, return the module name from the entry + points data. If not, the passed in application name is returned. + + :param str controller: The controller type + :param str application: The application name or module + :rtype: str + + """ + for pkg in self._get_applications(controller): + if pkg.name == application: + return pkg.module_name + return application + + @staticmethod + def _get_applications(controller): + """Return a list of application names for the given controller type + that have registered themselves as sprockets applications. + + :param str controller: The type of controller for the applications + :rtype: list + + """ + group_name = 'sprockets.%s.app' % controller + return pkg_resources.iter_entry_points(group=group_name) + + @staticmethod + def _get_argument_parser(): + """Return an instance of the + + :return: argparse.ArgumentParser + + """ + return argparse.ArgumentParser() + + def _get_controllers(self): + """Iterate through the installed controller entry points and import + the module and assign the handle to the CLI._controllers dict. + + :return: dict + + """ + controllers = dict() + for pkg in pkg_resources.iter_entry_points(group=self.CONTROLLERS): + LOGGER.debug('Loading %s controller', pkg.name) + controllers[pkg.name] = importlib.import_module(pkg.module_name) + return controllers + + def _get_controller_help(self, controller): + """Return the value of the HELP attribute for a controller that should + describe the functionality of the controller. + + :rtype: str|None + + """ + if hasattr(self._controllers[controller], 'HELP'): + return self._controllers[controller].HELP + return None + + def _print_installed_apps(self, controller): + """Print out a list of installed sprockets applications + + :param str controller: The name of the controller to get apps for + """ + print('\nInstalled Sprockets %s Apps\n' % controller.upper()) + print("{0:<25} {1:>25}".format('Name', 'Module')) + print(string.ljust('', 51, '-')) + for app in self._get_applications(controller): + print('{0:<25} {1:>25}'.format(app.name, '(%s)' % app.module_name)) + print('') + + +def main(): + """Main application runner""" + cli = CLI() + cli.run() diff --git a/sprockets/daemon.py b/sprockets/daemon.py new file mode 100644 index 0000000..95e5cfa --- /dev/null +++ b/sprockets/daemon.py @@ -0,0 +1,267 @@ +""" +Unix daemonization support + +""" +import atexit +import datetime +import grp +import logging +import os +from os import path +import platform +import pwd +import re +import subprocess +import sys +import traceback +import warnings + + +# Ignore the DeprecationWarning caused by os.popen3 in Python 2.6 +warnings.filterwarnings("ignore", category=DeprecationWarning) + +LOGGER = logging.getLogger(__name__) + + +def operating_system(): + """Return a string identifying the operating system the application + is running on. + + :rtype: str + + """ + if platform.system() == 'Darwin': + return 'OS X Version %s' % platform.mac_ver()[0] + distribution = ' '.join(platform.linux_distribution()).strip() + os_platform = platform.platform(True, True) + if distribution: + os_platform += ' (%s)' % distribution + return os_platform + + +class Daemon(object): + """Daemonize the helper application, putting it in a forked background + process. + + """ + + def __init__(self, controller): + """Daemonize the controller, optionally passing in the user and group + to run as, a pid file, if core dumps should be prevented and a path to + write out exception logs to. + + :param controller: The controller to daaemonize & run + :type controller: helper.controller.Controller + + """ + # The logger is reset by the time it gets here, fix to avoid warnings + from sprockets import NullHandler + LOGGER.addHandler(NullHandler()) + + self.controller = controller + self.pidfile_path = self._get_pidfile_path() + + def __enter__(self): + """Context manager method to return the handle to this object. + + :rtype: Daemon + + """ + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """When leaving the context, examine why the context is leaving, if + it's an exception and log the error + + """ + if exc_type and not isinstance(exc_val, SystemExit): + LOGGER.error('Daemon context manager closed on exception: %r', + exc_type) + + def start(self): + """Daemonize if the process is not already running.""" + if self._is_already_running(): + LOGGER.error('Is already running') + sys.exit(1) + try: + self._daemonize() + self.controller.start() + except Exception as error: + sys.stderr.write('\nERROR: Startup of %s Failed\n.' % + sys.argv[0].split('/')[-1]) + exception_log = self._get_exception_log_path() + if exception_log: + with open(exception_log, 'a') as handle: + timestamp = datetime.datetime.now().isoformat() + handle.write('{:->80}\n'.format(' [START]')) + handle.write('%s Exception [%s]\n' % (sys.argv[0], + timestamp)) + handle.write('{:->80}\n'.format(' [INFO]')) + handle.write('Interpreter: %s\n' % sys.executable) + handle.write('CLI arguments: %s\n' % ' '.join(sys.argv)) + handle.write('Exception: %s\n' % error) + handle.write('Traceback:\n') + output = traceback.format_exception(*sys.exc_info()) + _dev_null = [(handle.write(line), + sys.stdout.write(line)) for line in output] + handle.write('{:->80}\n'.format(' [END]')) + handle.flush() + sys.stderr.write('\nException log: %s\n\n' % exception_log) + sys.exit(1) + + @property + def gid(self): + """Return the group id that the daemon will run with + + :rtype: int + + """ + return os.getgid() + + @property + def uid(self): + """Return the user id that the process will run as + + :rtype: int + + """ + return os.getuid() + + def _daemonize(self): + """Fork into a background process and setup the process, copied in part + from http://www.jejik.com/files/examples/daemon3x.py + + """ + LOGGER.info('Forking %s into the background', sys.argv[0]) + + # Write the pidfile if current uid != final uid + if os.getuid() != self.uid: + fd = open(self.pidfile_path, 'w') + os.fchmod(fd.fileno(), 0o644) + os.fchown(fd.fileno(), self.uid, self.gid) + fd.close() + + try: + pid = os.fork() + if pid > 0: + sys.exit(0) + except OSError as error: + raise OSError('Could not fork off parent: %s', error) + + # Decouple from parent environment + os.chdir('/') + os.setsid() + os.umask(0o022) + + # Fork again + try: + pid = os.fork() + if pid > 0: + sys.exit(0) + except OSError as error: + raise OSError('Could not fork child: %s', error) + + # redirect standard file descriptors + sys.stdout.flush() + sys.stderr.flush() + si = open(os.devnull, 'r') + so = open(os.devnull, 'a+') + se = open(os.devnull, 'a+') + os.dup2(si.fileno(), sys.stdin.fileno()) + os.dup2(so.fileno(), sys.stdout.fileno()) + os.dup2(se.fileno(), sys.stderr.fileno()) + + # Automatically call self._remove_pidfile when the app exits + atexit.register(self._remove_pidfile) + self._write_pidfile() + + @staticmethod + def _get_exception_log_path(): + """Return the normalized path for the connection log, raising an + exception if it can not written to. + + :return: str + + """ + app = sys.argv[0].split('/')[-1] + for exception_log in ['/var/log/%s.errors' % app, + '/var/tmp/%s.errors' % app, + '/tmp/%s.errors' % app]: + if os.access(path.dirname(exception_log), os.W_OK): + return exception_log + return None + + def _get_pidfile_path(self): + """Return the normalized path for the pidfile, raising an + exception if it can not written to. + + :return: str + :raises: ValueError + :raises: OSError + + """ + app = sys.argv[0].split('/')[-1] + for pidfile in ['%s/pids/%s.pid' % (os.getcwd(), app), + '/var/run/%s.pid' % app, + '/var/run/%s/%s.pid' % (app, app), + '/var/tmp/%s.pid' % app, + '/tmp/%s.pid' % app, + '%s.pid' % app]: + if os.access(path.dirname(pidfile), os.W_OK): + return pidfile + raise OSError('Could not find an appropriate place for a pid file') + + def _is_already_running(self): + """Check to see if the process is running, first looking for a pidfile, + then shelling out in either case, removing a pidfile if it exists but + the process is not running. + + """ + # Look for the pidfile, if exists determine if the process is alive + pidfile = self._get_pidfile_path() + if os.path.exists(pidfile): + pid = open(pidfile).read().strip() + try: + os.kill(int(pid), 0) + sys.stderr.write('Process already running as pid # %s\n' % pid) + return True + except OSError as error: + LOGGER.debug('Found pidfile, no process # %s', error) + os.unlink(pidfile) + + # Check the os for a process that is not this one that looks the same + pattern = ' '.join(sys.argv) + pattern = '[%s]%s' % (pattern[0], pattern[1:]) + try: + output = subprocess.check_output('ps a | grep "%s"' % pattern, + shell=True) + except AttributeError: + # Python 2.6 + stdin, stdout, stderr = os.popen3('ps a | grep "%s"' % pattern) + output = stdout.read() + except subprocess.CalledProcessError: + return False + pids = [int(pid) for pid in (re.findall(r'^([0-9]+)\s', + output.decode('latin-1')))] + if os.getpid() in pids: + pids.remove(os.getpid()) + if not pids: + return False + if len(pids) == 1: + pids = pids[0] + sys.stderr.write('Process already running as pid # %s\n' % pids) + return True + + def _remove_pidfile(self): + """Remove the pid file from the filesystem""" + LOGGER.debug('Removing pidfile: %s', self.pidfile_path) + try: + os.unlink(self.pidfile_path) + except OSError: + pass + + def _write_pidfile(self): + """Write the pid file out with the process number in the pid file""" + LOGGER.debug('Writing pidfile: %s', self.pidfile_path) + with open(self.pidfile_path, "w") as handle: + handle.write(str(os.getpid())) From 3b351f56581bbad04b301c83ee3b1b6f10624878 Mon Sep 17 00:00:00 2001 From: "Gavin M. Roy" Date: Fri, 22 Aug 2014 14:56:04 -0400 Subject: [PATCH 4/8] Accidently a sentence --- sprockets/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sprockets/cli.py b/sprockets/cli.py index 78a8bea..c10eec1 100644 --- a/sprockets/cli.py +++ b/sprockets/cli.py @@ -241,7 +241,7 @@ class CLI(object): @staticmethod def _get_argument_parser(): - """Return an instance of the + """Return an instance of the argument parser. :return: argparse.ArgumentParser From 09058e1e730d3667da7364cd2c7bbc8c749bd592 Mon Sep 17 00:00:00 2001 From: "Gavin M. Roy" Date: Fri, 22 Aug 2014 14:57:20 -0400 Subject: [PATCH 5/8] Add CLI usage example in README --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index e56be88..56871a4 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,22 @@ Sprockets A loosely coupled framework built on top of Tornado. Take what you need to build awesome applications. +CLI Usage +--------- +```usage: sprockets [-h] [-l] [-d] [-s] [-v] [--version] + {http,amqp} ... application + +positional arguments: + {http,amqp} Available sprockets application controllers + http HTTP Application Controller + amqp RabbitMQ Worker Controller + application The sprockets app to run + +optional arguments: + -h, --help show this help message and exit + -l, --list List installed sprockets apps + -d, --daemonize Fork into a background process + -s, --syslog Log to syslog + -v, --verbose Verbose logging output + --version show program's version number and exit +``` \ No newline at end of file From 4d81938155ce1c12100422008db0a8383e36e638 Mon Sep 17 00:00:00 2001 From: "Gavin M. Roy" Date: Fri, 22 Aug 2014 14:57:59 -0400 Subject: [PATCH 6/8] Change the format for syntax highlighitng --- README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 56871a4..d609479 100644 --- a/README.md +++ b/README.md @@ -5,20 +5,20 @@ awesome applications. CLI Usage --------- -```usage: sprockets [-h] [-l] [-d] [-s] [-v] [--version] - {http,amqp} ... application -positional arguments: - {http,amqp} Available sprockets application controllers - http HTTP Application Controller - amqp RabbitMQ Worker Controller - application The sprockets app to run - -optional arguments: - -h, --help show this help message and exit - -l, --list List installed sprockets apps - -d, --daemonize Fork into a background process - -s, --syslog Log to syslog - -v, --verbose Verbose logging output - --version show program's version number and exit -``` \ No newline at end of file + usage: sprockets [-h] [-l] [-d] [-s] [-v] [--version] + {http,amqp} ... application + + positional arguments: + {http,amqp} Available sprockets application controllers + http HTTP Application Controller + amqp RabbitMQ Worker Controller + application The sprockets app to run + + optional arguments: + -h, --help show this help message and exit + -l, --list List installed sprockets apps + -d, --daemonize Fork into a background process + -s, --syslog Log to syslog + -v, --verbose Verbose logging output + --version show program's version number and exit From b03b3a36c84097cab361d802475381c36bc8656e Mon Sep 17 00:00:00 2001 From: "Gavin M. Roy" Date: Mon, 25 Aug 2014 10:51:36 -0400 Subject: [PATCH 7/8] Remove daemonization No need in first revision --- sprockets/cli.py | 39 +------ sprockets/daemon.py | 267 -------------------------------------------- 2 files changed, 6 insertions(+), 300 deletions(-) delete mode 100644 sprockets/daemon.py diff --git a/sprockets/cli.py b/sprockets/cli.py index c10eec1..1c7129c 100644 --- a/sprockets/cli.py +++ b/sprockets/cli.py @@ -32,7 +32,6 @@ else: import pkg_resources -from sprockets import daemon from sprockets import __version__ DESCRIPTION = 'Available sprockets application controllers' @@ -91,36 +90,14 @@ class CLI(object): # The list command prevents any other processing of args if self._args.list: self._print_installed_apps(self._args.controller) + sys.exit(0) - # Should be starting an application - else: + # If app is not specified at this point, raise an error + if not self._args.application: + sys.stderr.write('\nerror: application not specified\n\n') + self._arg_parser.print_help() + sys.exit(-1) - # If app is not specified at this point, raise an error - if not self._args.application: - sys.stderr.write('\nerror: application not specified\n\n') - self._arg_parser.print_help() - sys.exit(-1) - - # Fork into the background as a daemon if instructed to do so - if self._args.daemonize: - try: - with daemon.Daemon(self) as obj: - obj.start() - except (OSError, ValueError) as error: - sys.stderr.write('\nerror: could not start %s (%s)\n\n' % - (sys.argv[0], error)) - sys.exit(1) - - else: - # Start the application directly, no forking involved - self.start() - - def start(self): - """Invoked once the application is ready to be started. If the cli - args indicate that the application should be a daemon, it will already - be forked at this point. - - """ # If it's a registered app reference by name, get the module name app_module = self._get_application_module(self._args.controller, self._args.application) @@ -149,10 +126,6 @@ class CLI(object): action='store_true', help='List installed sprockets apps') - self._arg_parser.add_argument('-d', '--daemonize', - action='store_true', - help='Fork into a background process') - self._arg_parser.add_argument('-s', '--syslog', action='store_true', help='Log to syslog') diff --git a/sprockets/daemon.py b/sprockets/daemon.py deleted file mode 100644 index 95e5cfa..0000000 --- a/sprockets/daemon.py +++ /dev/null @@ -1,267 +0,0 @@ -""" -Unix daemonization support - -""" -import atexit -import datetime -import grp -import logging -import os -from os import path -import platform -import pwd -import re -import subprocess -import sys -import traceback -import warnings - - -# Ignore the DeprecationWarning caused by os.popen3 in Python 2.6 -warnings.filterwarnings("ignore", category=DeprecationWarning) - -LOGGER = logging.getLogger(__name__) - - -def operating_system(): - """Return a string identifying the operating system the application - is running on. - - :rtype: str - - """ - if platform.system() == 'Darwin': - return 'OS X Version %s' % platform.mac_ver()[0] - distribution = ' '.join(platform.linux_distribution()).strip() - os_platform = platform.platform(True, True) - if distribution: - os_platform += ' (%s)' % distribution - return os_platform - - -class Daemon(object): - """Daemonize the helper application, putting it in a forked background - process. - - """ - - def __init__(self, controller): - """Daemonize the controller, optionally passing in the user and group - to run as, a pid file, if core dumps should be prevented and a path to - write out exception logs to. - - :param controller: The controller to daaemonize & run - :type controller: helper.controller.Controller - - """ - # The logger is reset by the time it gets here, fix to avoid warnings - from sprockets import NullHandler - LOGGER.addHandler(NullHandler()) - - self.controller = controller - self.pidfile_path = self._get_pidfile_path() - - def __enter__(self): - """Context manager method to return the handle to this object. - - :rtype: Daemon - - """ - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """When leaving the context, examine why the context is leaving, if - it's an exception and log the error - - """ - if exc_type and not isinstance(exc_val, SystemExit): - LOGGER.error('Daemon context manager closed on exception: %r', - exc_type) - - def start(self): - """Daemonize if the process is not already running.""" - if self._is_already_running(): - LOGGER.error('Is already running') - sys.exit(1) - try: - self._daemonize() - self.controller.start() - except Exception as error: - sys.stderr.write('\nERROR: Startup of %s Failed\n.' % - sys.argv[0].split('/')[-1]) - exception_log = self._get_exception_log_path() - if exception_log: - with open(exception_log, 'a') as handle: - timestamp = datetime.datetime.now().isoformat() - handle.write('{:->80}\n'.format(' [START]')) - handle.write('%s Exception [%s]\n' % (sys.argv[0], - timestamp)) - handle.write('{:->80}\n'.format(' [INFO]')) - handle.write('Interpreter: %s\n' % sys.executable) - handle.write('CLI arguments: %s\n' % ' '.join(sys.argv)) - handle.write('Exception: %s\n' % error) - handle.write('Traceback:\n') - output = traceback.format_exception(*sys.exc_info()) - _dev_null = [(handle.write(line), - sys.stdout.write(line)) for line in output] - handle.write('{:->80}\n'.format(' [END]')) - handle.flush() - sys.stderr.write('\nException log: %s\n\n' % exception_log) - sys.exit(1) - - @property - def gid(self): - """Return the group id that the daemon will run with - - :rtype: int - - """ - return os.getgid() - - @property - def uid(self): - """Return the user id that the process will run as - - :rtype: int - - """ - return os.getuid() - - def _daemonize(self): - """Fork into a background process and setup the process, copied in part - from http://www.jejik.com/files/examples/daemon3x.py - - """ - LOGGER.info('Forking %s into the background', sys.argv[0]) - - # Write the pidfile if current uid != final uid - if os.getuid() != self.uid: - fd = open(self.pidfile_path, 'w') - os.fchmod(fd.fileno(), 0o644) - os.fchown(fd.fileno(), self.uid, self.gid) - fd.close() - - try: - pid = os.fork() - if pid > 0: - sys.exit(0) - except OSError as error: - raise OSError('Could not fork off parent: %s', error) - - # Decouple from parent environment - os.chdir('/') - os.setsid() - os.umask(0o022) - - # Fork again - try: - pid = os.fork() - if pid > 0: - sys.exit(0) - except OSError as error: - raise OSError('Could not fork child: %s', error) - - # redirect standard file descriptors - sys.stdout.flush() - sys.stderr.flush() - si = open(os.devnull, 'r') - so = open(os.devnull, 'a+') - se = open(os.devnull, 'a+') - os.dup2(si.fileno(), sys.stdin.fileno()) - os.dup2(so.fileno(), sys.stdout.fileno()) - os.dup2(se.fileno(), sys.stderr.fileno()) - - # Automatically call self._remove_pidfile when the app exits - atexit.register(self._remove_pidfile) - self._write_pidfile() - - @staticmethod - def _get_exception_log_path(): - """Return the normalized path for the connection log, raising an - exception if it can not written to. - - :return: str - - """ - app = sys.argv[0].split('/')[-1] - for exception_log in ['/var/log/%s.errors' % app, - '/var/tmp/%s.errors' % app, - '/tmp/%s.errors' % app]: - if os.access(path.dirname(exception_log), os.W_OK): - return exception_log - return None - - def _get_pidfile_path(self): - """Return the normalized path for the pidfile, raising an - exception if it can not written to. - - :return: str - :raises: ValueError - :raises: OSError - - """ - app = sys.argv[0].split('/')[-1] - for pidfile in ['%s/pids/%s.pid' % (os.getcwd(), app), - '/var/run/%s.pid' % app, - '/var/run/%s/%s.pid' % (app, app), - '/var/tmp/%s.pid' % app, - '/tmp/%s.pid' % app, - '%s.pid' % app]: - if os.access(path.dirname(pidfile), os.W_OK): - return pidfile - raise OSError('Could not find an appropriate place for a pid file') - - def _is_already_running(self): - """Check to see if the process is running, first looking for a pidfile, - then shelling out in either case, removing a pidfile if it exists but - the process is not running. - - """ - # Look for the pidfile, if exists determine if the process is alive - pidfile = self._get_pidfile_path() - if os.path.exists(pidfile): - pid = open(pidfile).read().strip() - try: - os.kill(int(pid), 0) - sys.stderr.write('Process already running as pid # %s\n' % pid) - return True - except OSError as error: - LOGGER.debug('Found pidfile, no process # %s', error) - os.unlink(pidfile) - - # Check the os for a process that is not this one that looks the same - pattern = ' '.join(sys.argv) - pattern = '[%s]%s' % (pattern[0], pattern[1:]) - try: - output = subprocess.check_output('ps a | grep "%s"' % pattern, - shell=True) - except AttributeError: - # Python 2.6 - stdin, stdout, stderr = os.popen3('ps a | grep "%s"' % pattern) - output = stdout.read() - except subprocess.CalledProcessError: - return False - pids = [int(pid) for pid in (re.findall(r'^([0-9]+)\s', - output.decode('latin-1')))] - if os.getpid() in pids: - pids.remove(os.getpid()) - if not pids: - return False - if len(pids) == 1: - pids = pids[0] - sys.stderr.write('Process already running as pid # %s\n' % pids) - return True - - def _remove_pidfile(self): - """Remove the pid file from the filesystem""" - LOGGER.debug('Removing pidfile: %s', self.pidfile_path) - try: - os.unlink(self.pidfile_path) - except OSError: - pass - - def _write_pidfile(self): - """Write the pid file out with the process number in the pid file""" - LOGGER.debug('Writing pidfile: %s', self.pidfile_path) - with open(self.pidfile_path, "w") as handle: - handle.write(str(os.getpid())) From 5f33a58c62ba571e28fbcba8dd0ef88343362011 Mon Sep 17 00:00:00 2001 From: "Gavin M. Roy" Date: Mon, 25 Aug 2014 10:58:22 -0400 Subject: [PATCH 8/8] Update the README and cli help --- MANIFEST.in | 2 +- README.md | 24 ------------------------ README.rst | 45 +++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- sprockets/cli.py | 3 ++- 5 files changed, 49 insertions(+), 27 deletions(-) delete mode 100644 README.md create mode 100644 README.rst diff --git a/MANIFEST.in b/MANIFEST.in index c1a7121..9d5d250 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ include LICENSE -include README.md +include README.rst diff --git a/README.md b/README.md deleted file mode 100644 index d609479..0000000 --- a/README.md +++ /dev/null @@ -1,24 +0,0 @@ -Sprockets -========= -A loosely coupled framework built on top of Tornado. Take what you need to build -awesome applications. - -CLI Usage ---------- - - usage: sprockets [-h] [-l] [-d] [-s] [-v] [--version] - {http,amqp} ... application - - positional arguments: - {http,amqp} Available sprockets application controllers - http HTTP Application Controller - amqp RabbitMQ Worker Controller - application The sprockets app to run - - optional arguments: - -h, --help show this help message and exit - -l, --list List installed sprockets apps - -d, --daemonize Fork into a background process - -s, --syslog Log to syslog - -v, --verbose Verbose logging output - --version show program's version number and exit diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..9028d82 --- /dev/null +++ b/README.rst @@ -0,0 +1,45 @@ +Sprockets +========= +A loosely coupled framework built on top of Tornado. Take what you need to build +awesome applications. + +The core `sprockets` packages offers only the command line application for +invoking Sprockets controllers such as the HTTP and AMQP controllers. + +|Version| |Downloads| |Status| |Coverage| |License| + +CLI Usage +--------- + + usage: sprockets [-h] [-l] [-d] [-s] [-v] [--version] + {http,amqp} ... application + + positional arguments: + {http,amqp} Available sprockets application controllers + http HTTP Application Controller + amqp RabbitMQ Worker Controller + application The sprockets app to run + + optional arguments: + -h, --help show this help message and exit + -l, --list List installed sprockets apps + -s, --syslog Log to syslog + -v, --verbose Verbose logging output, use -vv for DEBUG level logging + --version show program's version number and exit + + + +.. |Version| image:: https://badge.fury.io/py/sprockets.svg? + :target: http://badge.fury.io/py/sprockets + +.. |Status| image:: https://travis-ci.org/sprockets/sprockets.svg?branch=master + :target: https://travis-ci.org/sprockets/sprockets + +.. |Coverage| image:: https://coveralls.io/repos/sprockets/sprockets/badge.png + :target: https://coveralls.io/r/sprockets/sprockets + +.. |Downloads| image:: https://pypip.in/d/sprockets/badge.svg? + :target: https://pypi.python.org/pypi/sprockets + +.. |License| image:: https://pypip.in/license/sprockets/badge.svg? + :target: https://sprockets.readthedocs.org diff --git a/setup.py b/setup.py index 853d863..a7cce89 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup(name='sprockets', url='https://github.com/sprockets/sprockets', install_requires=requirements, license=open('LICENSE').read(), - package_data={'': ['LICENSE', 'README.md']}, + package_data={'': ['LICENSE', 'README.rst']}, packages=['sprockets'], classifiers=['Development Status :: 3 - Alpha', 'Environment :: No Input/Output (Daemon)', diff --git a/sprockets/cli.py b/sprockets/cli.py index 1c7129c..f6e2818 100644 --- a/sprockets/cli.py +++ b/sprockets/cli.py @@ -132,7 +132,8 @@ class CLI(object): self._arg_parser.add_argument('-v', '--verbose', action='count', - help='Verbose logging output') + help=('Verbose logging output, use -vv ' + 'for DEBUG level logging')) self._arg_parser.add_argument('--version', action='version',