Initial commit, missing tests, but functional

setup.py

@@ -1,4 +1,3 @@
-import os
 from setuptools import setup
 import sys
 
@@ -13,7 +12,6 @@ if (version.major, version.minor) < (2, 7):
     requirements.append('logutils')
     tests_require.append('unittest2')
 
-
 setup(name='sprockets',
       version='0.1.0',
       description=('A modular, loosely coupled micro-framework built on top '
@@ -47,4 +45,4 @@ setup(name='sprockets',
                    'Topic :: Software Development :: Libraries :: Python Modules'],
       test_suite='nose.collector',
       tests_require=tests_require,
-      zip_safe=True)
+      zip_safe=False)

sprockets/__init__.py

@@ -0,0 +1,20 @@
+"""
+Sprockets
+=========
+A loosely coupled framework built on top of Tornado. Take what you need to
+build awesome applications.
+
+"""
+version_info = (0, 0, 0)
+__version__ = '.'.join(str(v) for v in version_info)
+
+import logging
+
+# Ensure there is a NullHandler for logging
+try:
+    from logging import NullHandler
+except ImportError:
+    # Not available in Python 2.6
+    class NullHandler(logging.Handler):
+        def emit(self, record):
+            pass

sprockets/cli.py

@@ -0,0 +1,291 @@
+"""
+Sprockets CLI
+=============
+The sprockets CLI interface for running applications. Applications are meant
+to be run by a controller that is managed by the sprockets CLI interface.
+
+The sprockets CLI interface loads controller applications that are registered
+using setuptools entry points.
+
+Each controller is expected to expose at least a `main(application, args)`
+method that would be invoked when starting the application. Additional, a
+controller can implement a `add_cli_arguments(parser)` method that will be
+invoked when setting up the command line parameters. This allows controllers
+to inject configuration directives into the cli.
+
+Applications can be a python package or module and if they are registered
+to a specific controller, can be referenced by an alias.
+
+"""
+import argparse
+import importlib
+import logging
+import string
+import sys
+
+# import logutils for Python 2.6 or logging.config for later versions
+sys_version = sys.version_info
+if (sys_version.major, sys_version.minor) < (2, 7):
+    import logutils.dictconfig as logging_config
+else:
+    from logging import config as logging_config
+
+import pkg_resources
+
+from sprockets import daemon
+from sprockets import __version__
+
+DESCRIPTION = 'Available sprockets application controllers'
+
+# Logging formatters
+SYSLOG_FORMAT = ('%(levelname)s <PID %(process)d:%(processName)s> '
+                 '%(name)s.%(funcName)s(): %(message)s')
+
+VERBOSE_FORMAT = ('%(levelname) -10s %(asctime)s %(process)-6d '
+                  '%(processName) -20s %(name) -20s '
+                  '%(funcName) -20s L%(lineno)-6d: %(message)s')
+
+# Base logging configuration
+LOGGING = {'disable_existing_loggers': True,
+           'filters': {},
+           'formatters': {'syslog': {'format': SYSLOG_FORMAT},
+                          'verbose': {'datefmt': '%Y-%m-%d %H:%M:%S',
+                                      'format': VERBOSE_FORMAT}},
+           'handlers': {'console': {'class': 'logging.StreamHandler',
+                                    'formatter': 'verbose'},
+                        'syslog': {'class': 'logging.handlers.SysLogHandler',
+                                   'formatter': 'syslog'}},
+           'incremental': False,
+           'loggers': {'sprockets': {'handlers': ['console'],
+                                     'level': logging.WARNING,
+                                     'propagate': True}},
+           'root': {'handlers': [],
+                    'level': logging.CRITICAL,
+                    'propagate': True},
+           'version': 1}
+
+LOGGER = logging.getLogger(__name__)
+
+
+class CLI(object):
+    """The core Sprockets CLI application providing argument parsing and
+    logic for starting a controller.
+
+    The package or module for an application is passed into
+
+    """
+
+    CONTROLLERS = 'sprockets.controller'
+
+    def __init__(self):
+        self._controllers = self._get_controllers()
+        self._arg_parser = argparse.ArgumentParser()
+        self._add_cli_args()
+        self._args = self._arg_parser.parse_args()
+
+    def run(self):
+        """Evaluate the command line arguments, performing the appropriate
+        actions so the application can be started.
+
+        """
+        # The list command prevents any other processing of args
+        if self._args.list:
+            self._print_installed_apps(self._args.controller)
+
+        # Should be starting an application
+        else:
+
+            # If app is not specified at this point, raise an error
+            if not self._args.application:
+                sys.stderr.write('\nerror: application not specified\n\n')
+                self._arg_parser.print_help()
+                sys.exit(-1)
+
+            # Fork into the background as a daemon if instructed to do so
+            if self._args.daemonize:
+                try:
+                    with daemon.Daemon(self) as obj:
+                        obj.start()
+                except (OSError, ValueError) as error:
+                    sys.stderr.write('\nerror: could not start %s (%s)\n\n' %
+                                     (sys.argv[0], error))
+                    sys.exit(1)
+
+            else:
+                # Start the application directly, no forking involved
+                self.start()
+
+    def start(self):
+        """Invoked once the application is ready to be started. If the cli
+        args indicate that the application should be a daemon, it will already
+        be forked at this point.
+
+        """
+        # If it's a registered app reference by name, get the module name
+        app_module = self._get_application_module(self._args.controller,
+                                                  self._args.application)
+
+        # Configure logging based upon the flags
+        self._configure_logging(app_module,
+                                self._args.verbose,
+                                self._args.syslog)
+
+        # Try and run the controller
+        try:
+            self._controllers[self._args.controller].main(app_module,
+                                                          self._args)
+        except TypeError as error:
+            sys.stderr.write('error: could not start the %s controller for %s'
+                             ': %s\n\n' % (self._args.controller,
+                                           app_module,
+                                           str(error)))
+            sys.exit(-1)
+
+    def _add_cli_args(self):
+        """Add the cli arguments to the argument parser."""
+
+        # Optional cli arguments
+        self._arg_parser.add_argument('-l', '--list',
+                                      action='store_true',
+                                      help='List installed sprockets apps')
+
+        self._arg_parser.add_argument('-d', '--daemonize',
+                                      action='store_true',
+                                      help='Fork into a background process')
+
+        self._arg_parser.add_argument('-s', '--syslog',
+                                      action='store_true',
+                                      help='Log to syslog')
+
+        self._arg_parser.add_argument('-v', '--verbose',
+                                      action='count',
+                                      help='Verbose logging output')
+
+        self._arg_parser.add_argument('--version',
+                                      action='version',
+                                      version='sprockets v%s ' % __version__)
+
+        # Controller sub-parser
+        subparsers = self._arg_parser.add_subparsers(dest='controller',
+                                                          help=DESCRIPTION)
+
+        # Iterate through the controllers and add their cli arguments
+        for key in self._controllers:
+            help_text = self._get_controller_help(key)
+            sub_parser = subparsers.add_parser(key, help=help_text)
+            try:
+                self._controllers[key].add_cli_arguments(sub_parser)
+            except AttributeError:
+                LOGGER.debug('%s missing add_cli_arguments()', key)
+
+        # The application argument
+        self._arg_parser.add_argument('application',
+                                      action="store",
+                                      help='The sprockets app to run')
+
+    @staticmethod
+    def _configure_logging(application, verbosity=0, syslog=False):
+        """Configure logging for the application, setting the appropriate
+        verbosity and adding syslog if it's enabled.
+
+        :param str application: The application module/package name
+        :param int verbosity: 1 == INFO, 2 == DEBUG
+        :param bool syslog: Enable the syslog handler
+
+        """
+        # Create a new copy of the logging config that will be modified
+        config = dict(LOGGING)
+
+        # Increase the logging verbosity
+        if verbosity == 1:
+            config['loggers']['sprockets']['level'] = logging.INFO
+        elif verbosity == 2:
+            config['loggers']['sprockets']['level'] = logging.DEBUG
+
+        # Add syslog if it's enabled
+        if syslog:
+            config['loggers']['sprockets']['handlers'].append('syslog')
+
+        # Copy the sprockets logger to the application
+        config['loggers'][application] = dict(config['loggers']['sprockets'])
+
+        # Configure logging
+        logging_config.dictConfig(config)
+
+    def _get_application_module(self, controller, application):
+        """Return the module for an application. If it's a entry-point
+        registered application name, return the module name from the entry
+        points data. If not, the passed in application name is returned.
+
+        :param str controller: The controller type
+        :param str application: The application name or module
+        :rtype: str
+
+        """
+        for pkg in self._get_applications(controller):
+            if pkg.name == application:
+                return pkg.module_name
+        return application
+
+    @staticmethod
+    def _get_applications(controller):
+        """Return a list of application names for the given controller type
+        that have registered themselves as sprockets applications.
+
+        :param str controller: The type of controller for the applications
+        :rtype: list
+
+        """
+        group_name = 'sprockets.%s.app' % controller
+        return pkg_resources.iter_entry_points(group=group_name)
+
+    @staticmethod
+    def _get_argument_parser():
+        """Return an instance of the 
+
+        :return: argparse.ArgumentParser
+
+        """
+        return argparse.ArgumentParser()
+
+    def _get_controllers(self):
+        """Iterate through the installed controller entry points and import
+        the module and assign the handle to the CLI._controllers dict.
+
+        :return: dict
+
+        """
+        controllers = dict()
+        for pkg in pkg_resources.iter_entry_points(group=self.CONTROLLERS):
+            LOGGER.debug('Loading %s controller', pkg.name)
+            controllers[pkg.name] = importlib.import_module(pkg.module_name)
+        return controllers
+
+    def _get_controller_help(self, controller):
+        """Return the value of the HELP attribute for a controller that should
+        describe the functionality of the controller.
+
+        :rtype: str|None
+
+        """
+        if hasattr(self._controllers[controller], 'HELP'):
+            return self._controllers[controller].HELP
+        return None
+
+    def _print_installed_apps(self, controller):
+        """Print out a list of installed sprockets applications
+
+        :param str controller: The name of the controller to get apps for
+        """
+        print('\nInstalled Sprockets %s Apps\n' % controller.upper())
+        print("{0:<25} {1:>25}".format('Name', 'Module'))
+        print(string.ljust('', 51, '-'))
+        for app in self._get_applications(controller):
+            print('{0:<25} {1:>25}'.format(app.name, '(%s)' % app.module_name))
+        print('')
+
+
+def main():
+    """Main application runner"""
+    cli = CLI()
+    cli.run()

sprockets/daemon.py

@@ -0,0 +1,267 @@
+"""
+Unix daemonization support
+
+"""
+import atexit
+import datetime
+import grp
+import logging
+import os
+from os import path
+import platform
+import pwd
+import re
+import subprocess
+import sys
+import traceback
+import warnings
+
+
+# Ignore the DeprecationWarning caused by os.popen3 in Python 2.6
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+LOGGER = logging.getLogger(__name__)
+
+
+def operating_system():
+    """Return a string identifying the operating system the application
+    is running on.
+
+    :rtype: str
+
+    """
+    if platform.system() == 'Darwin':
+        return 'OS X Version %s' % platform.mac_ver()[0]
+    distribution = ' '.join(platform.linux_distribution()).strip()
+    os_platform = platform.platform(True, True)
+    if distribution:
+        os_platform += ' (%s)' % distribution
+    return os_platform
+
+
+class Daemon(object):
+    """Daemonize the helper application, putting it in a forked background
+    process.
+
+    """
+
+    def __init__(self, controller):
+        """Daemonize the controller, optionally passing in the user and group
+        to run as, a pid file, if core dumps should be prevented and a path to
+        write out exception logs to.
+
+        :param controller: The controller to daaemonize & run
+        :type controller: helper.controller.Controller
+
+        """
+        # The logger is reset by the time it gets here, fix to avoid warnings
+        from sprockets import NullHandler
+        LOGGER.addHandler(NullHandler())
+
+        self.controller = controller
+        self.pidfile_path = self._get_pidfile_path()
+
+    def __enter__(self):
+        """Context manager method to return the handle to this object.
+
+        :rtype: Daemon
+
+        """
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """When leaving the context, examine why the context is leaving, if
+        it's an exception and log the error
+
+        """
+        if exc_type and not isinstance(exc_val, SystemExit):
+            LOGGER.error('Daemon context manager closed on exception: %r',
+                         exc_type)
+
+    def start(self):
+        """Daemonize if the process is not already running."""
+        if self._is_already_running():
+            LOGGER.error('Is already running')
+            sys.exit(1)
+        try:
+            self._daemonize()
+            self.controller.start()
+        except Exception as error:
+            sys.stderr.write('\nERROR: Startup of %s Failed\n.' %
+                             sys.argv[0].split('/')[-1])
+            exception_log = self._get_exception_log_path()
+            if exception_log:
+                with open(exception_log, 'a') as handle:
+                    timestamp = datetime.datetime.now().isoformat()
+                    handle.write('{:->80}\n'.format(' [START]'))
+                    handle.write('%s Exception [%s]\n' % (sys.argv[0],
+                                                          timestamp))
+                    handle.write('{:->80}\n'.format(' [INFO]'))
+                    handle.write('Interpreter: %s\n' % sys.executable)
+                    handle.write('CLI arguments: %s\n' % ' '.join(sys.argv))
+                    handle.write('Exception: %s\n' % error)
+                    handle.write('Traceback:\n')
+                    output = traceback.format_exception(*sys.exc_info())
+                    _dev_null = [(handle.write(line),
+                                 sys.stdout.write(line)) for line in output]
+                    handle.write('{:->80}\n'.format(' [END]'))
+                    handle.flush()
+                sys.stderr.write('\nException log: %s\n\n' % exception_log)
+            sys.exit(1)
+
+    @property
+    def gid(self):
+        """Return the group id that the daemon will run with
+
+        :rtype: int
+
+        """
+        return os.getgid()
+
+    @property
+    def uid(self):
+        """Return the user id that the process will run as
+
+        :rtype: int
+
+        """
+        return os.getuid()
+
+    def _daemonize(self):
+        """Fork into a background process and setup the process, copied in part
+        from http://www.jejik.com/files/examples/daemon3x.py
+
+        """
+        LOGGER.info('Forking %s into the background', sys.argv[0])
+
+        # Write the pidfile if current uid != final uid
+        if os.getuid() != self.uid:
+            fd = open(self.pidfile_path, 'w')
+            os.fchmod(fd.fileno(), 0o644)
+            os.fchown(fd.fileno(), self.uid, self.gid)
+            fd.close()
+
+        try:
+            pid = os.fork()
+            if pid > 0:
+                    sys.exit(0)
+        except OSError as error:
+                raise OSError('Could not fork off parent: %s', error)
+
+        # Decouple from parent environment
+        os.chdir('/')
+        os.setsid()
+        os.umask(0o022)
+
+        # Fork again
+        try:
+            pid = os.fork()
+            if pid > 0:
+                sys.exit(0)
+        except OSError as error:
+            raise OSError('Could not fork child: %s', error)
+
+        # redirect standard file descriptors
+        sys.stdout.flush()
+        sys.stderr.flush()
+        si = open(os.devnull, 'r')
+        so = open(os.devnull, 'a+')
+        se = open(os.devnull, 'a+')
+        os.dup2(si.fileno(), sys.stdin.fileno())
+        os.dup2(so.fileno(), sys.stdout.fileno())
+        os.dup2(se.fileno(), sys.stderr.fileno())
+
+        # Automatically call self._remove_pidfile when the app exits
+        atexit.register(self._remove_pidfile)
+        self._write_pidfile()
+
+    @staticmethod
+    def _get_exception_log_path():
+        """Return the normalized path for the connection log, raising an
+        exception if it can not written to.
+
+        :return: str
+
+        """
+        app = sys.argv[0].split('/')[-1]
+        for exception_log in ['/var/log/%s.errors' % app,
+                              '/var/tmp/%s.errors' % app,
+                              '/tmp/%s.errors' % app]:
+            if os.access(path.dirname(exception_log), os.W_OK):
+                return exception_log
+        return None
+
+    def _get_pidfile_path(self):
+        """Return the normalized path for the pidfile, raising an
+        exception if it can not written to.
+
+        :return: str
+        :raises: ValueError
+        :raises: OSError
+
+        """
+        app = sys.argv[0].split('/')[-1]
+        for pidfile in ['%s/pids/%s.pid' % (os.getcwd(), app),
+                        '/var/run/%s.pid' % app,
+                        '/var/run/%s/%s.pid' % (app, app),
+                        '/var/tmp/%s.pid' % app,
+                        '/tmp/%s.pid' % app,
+                        '%s.pid' % app]:
+            if os.access(path.dirname(pidfile), os.W_OK):
+                return pidfile
+        raise OSError('Could not find an appropriate place for a pid file')
+
+    def _is_already_running(self):
+        """Check to see if the process is running, first looking for a pidfile,
+        then shelling out in either case, removing a pidfile if it exists but
+        the process is not running.
+
+        """
+        # Look for the pidfile, if exists determine if the process is alive
+        pidfile = self._get_pidfile_path()
+        if os.path.exists(pidfile):
+            pid = open(pidfile).read().strip()
+            try:
+                os.kill(int(pid), 0)
+                sys.stderr.write('Process already running as pid # %s\n' % pid)
+                return True
+            except OSError as error:
+                LOGGER.debug('Found pidfile, no process # %s', error)
+                os.unlink(pidfile)
+
+        # Check the os for a process that is not this one that looks the same
+        pattern = ' '.join(sys.argv)
+        pattern = '[%s]%s' % (pattern[0], pattern[1:])
+        try:
+            output = subprocess.check_output('ps a | grep "%s"' % pattern,
+                                             shell=True)
+        except AttributeError:
+            # Python 2.6
+            stdin, stdout, stderr = os.popen3('ps a | grep "%s"' % pattern)
+            output = stdout.read()
+        except subprocess.CalledProcessError:
+            return False
+        pids = [int(pid) for pid in (re.findall(r'^([0-9]+)\s',
+                                                output.decode('latin-1')))]
+        if os.getpid() in pids:
+            pids.remove(os.getpid())
+        if not pids:
+            return False
+        if len(pids) == 1:
+            pids = pids[0]
+        sys.stderr.write('Process already running as pid # %s\n' % pids)
+        return True
+
+    def _remove_pidfile(self):
+        """Remove the pid file from the filesystem"""
+        LOGGER.debug('Removing pidfile: %s', self.pidfile_path)
+        try:
+            os.unlink(self.pidfile_path)
+        except OSError:
+            pass
+
+    def _write_pidfile(self):
+        """Write the pid file out with the process number in the pid file"""
+        LOGGER.debug('Writing pidfile: %s', self.pidfile_path)
+        with open(self.pidfile_path, "w") as handle:
+            handle.write(str(os.getpid()))