Spawning Mutliple Scripts

Simple Script Spawner

In this script I needed to spawn scripts to utilized a limited number of ODBC connectors I had set up. I also wanted the script to process overnight and so I needed to specify a duration. This script uses those old UNIX favorites fork()/exec()/wait() to manage the sub-scripts.

#! /usr/bin/env python
#
#    $Id:$
#
#  Purpose:  Spawn multiple test scripts in a controlled way with logging
#
#-------------------------------------------------------------------------------

import sys
import re
import os
import time
import getopt
import string
import logging
import datetime

#-------------------------------------------------------------------------------

__version__        = '1.3.0'

debug_level        = 0
verbose_flg        = False

log                = None

stagger_delay      = 3
duration           = None
no_connectors      = 3
connectors         = []
procs              = {}

#===============================================================================

def INFO(msg):
   if log: log.info(' ' + msg)
   if verbose_flg: print "[test]  %s" % msg

#-------------------------------------------------------------------------------

def ERROR(msg):
   if log: log.error(msg)
   sys.stderr.write('[test]  %s\n' % msg)

#-------------------------------------------------------------------------------

def WARNING(msg):
   if log: log.warning('*****' + msg + '*****')
   if verbose_flg: print "[test]  %s" % msg

#===============================================================================

def init():
   global log

   pid      = os.getpid()

   if debug_level > 0:  print "My PID is %d" % pid

   log  = logging.getLogger('spawn')
   hdlr = logging.FileHandler('log/spawn.log')
   fmtr = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

   hdlr.setFormatter(fmtr)
"spawn.py" 197 lines, 5002 characters written
$ cat spawn.py 
#! /usr/bin/env python
#
#    $Id:$
#
#  Purpose:  Spawn multiple test scripts in a controlled way with logging
#
#-------------------------------------------------------------------------------

import sys
import re
import os
import time
import getopt
import string
import logging
import datetime

#-------------------------------------------------------------------------------

__version__        = '1.3.0'

debug_level        = 0
verbose_flg        = False

log                = None

stagger_delay      = 3
duration           = None
no_connectors      = 3
connectors         = []
procs              = {}

#===============================================================================

def INFO(msg):
   if log: log.info(' ' + msg)
   if verbose_flg: print "[test]  %s" % msg

#-------------------------------------------------------------------------------

def ERROR(msg):
   if log: log.error(msg)
   sys.stderr.write('[test]  %s\n' % msg)

#-------------------------------------------------------------------------------

def WARNING(msg):
   if log: log.warning('*****' + msg + '*****')
   if verbose_flg: print "[test]  %s" % msg

#===============================================================================

def init():
   global log

   pid      = os.getpid()

   if debug_level > 0:  print "My PID is %d" % pid

   log  = logging.getLogger('spawn')
   hdlr = logging.FileHandler('log/spawn.log')
   fmtr = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

   hdlr.setFormatter(fmtr)
   log.addHandler(hdlr) 
   log.setLevel(logging.INFO)

   INFO("===== Started processing ==================================")

#-------------------------------------------------------------------------

def do_child(connector, delay):
   time.sleep(delay)
   os.execv("./test.py", ["./test.py", "-c", "%d" % connector])

#-------------------------------------------------------------------------

def spawn():
   global stagger_delay

   loop = True

   delay = 0

   t_start = datetime.datetime.now()

   while loop:
      if len(connectors) > 0:
         connector = connectors.pop(0)

         child_pid = os.fork()

         if child_pid == 0:
            do_child(connector, delay)
         else:
            INFO("Spawned test process (PID %d) on connector %d" % (child_pid, connector))
            procs[child_pid] = connector

         delay += stagger_delay
      else:
         stagger_delay = 0

         (pid, status) = os.wait()

         connector     = procs[pid]

         INFO("cleaned up child process (PID %d) with connector %d" % (pid, connector))

         del procs[pid]

         t_now   = datetime.datetime.now()

         t_delta = t_now - t_start

         if duration:
            if t_delta < duration:
               connectors.append(connector)
               continue

         if len(procs) == 0:
            loop = False

   INFO("***** FINISHED *****")

#-------------------------------------------------------------------------------

def usage():
   USAGE = """
   
     Usage:
     
       $ spawn.py -n <no_to_spawn>
    
   """
   
   sys.stderr.write(USAGE)
   
#-------------------------------------------------------------------------------

def main(argv):
   global debug_flg, verbose_flg, no_connectors, duration, stagger_delay

   #----- Process command line arguments ----------------------------

   duration_min = None

   try:
      opts, args = getopt.getopt(argv, "dD:hn:s:vV",
              ["debug", "duration=", "help", "no=", "stagger=", "verbose", "version"])
   except getopt.GetoptError:
      usage()
      sys.exit(2)
   else:
      for opt, arg in opts:
         if opt in ("-h", "--help"):
            usage()
            sys.exit(0)
         elif opt == "-d":
            debug_level                += 1
         elif opt == "--debug":
            debug_level                 = int(arg)
         elif opt in ("-D", "--duration"):
            duration_min                = int(arg)
         elif opt in ("-s", "--stagger"):
            stagger_delay               = int(arg)
         elif opt in ("-n", "--no"):
            no_connectors               = int(arg)
            if no_connectors > 10:
               no_connectors = 10
               sys.stderr.write("Only 10 ODBC connectors available!")
               sys.stderr.flush()
         elif opt in ("-v", "--verbose"):
            verbose_flg = True
         elif opt in ("-V", "--version"):
            print "Version: %s" % __version__
            sys.exit(0)

   if duration_min:
      print "[spawn]  Running for a duration of %d minutes" % duration_min
      duration = datetime.timedelta(minutes=duration_min)

   for i in range(no_connectors):
      connectors.append(i + 1)

   print "[spawn]  Using these ODBC connectors - %s" % connectors

   init()

   spawn()

#---------------------------------------------------------------------

if __name__ == "__main__":
   main(sys.argv[1:])

#---------------------------------------------------------------------

The log file output looks like this:

2008-05-01 18:22:00,577 INFO  ===== Started processing ==================================
2008-05-01 18:22:00,608 INFO  Spawned test process (PID 11348) on connector 1
2008-05-01 18:22:00,640 INFO  Spawned test process (PID 12216) on connector 2
2008-05-01 18:22:26,983 INFO  cleaned up child process (PID 11348) with connector 1
2008-05-01 18:22:27,374 INFO  cleaned up child process (PID 12216) with connector 2
2008-05-01 18:22:27,374 INFO  ***** FINISHED *****
2008-05-01 18:23:37,359 INFO  ===== Started processing ==================================
2008-05-01 18:23:37,390 INFO  Spawned test process (PID 11068) on connector 1
2008-05-01 18:23:37,437 INFO  Spawned test process (PID 12252) on connector 2
2008-05-01 18:24:03,749 INFO  cleaned up child process (PID 11068) with connector 1
2008-05-01 18:24:03,780 INFO  cleaned up child process (PID 12252) with connector 2
2008-05-01 18:24:03,780 INFO  ***** FINISHED *****
2008-05-01 18:27:08,390 INFO  ===== Started processing ==================================
2008-05-01 18:27:08,437 INFO  Spawned test process (PID 11720) on connector 1
2008-05-01 18:27:18,453 INFO  Spawned test process (PID 10796) on connector 2
2008-05-01 18:27:28,453 INFO  cleaned up child process (PID 11720) with connector 1
2008-05-01 18:27:35,717 INFO  cleaned up child process (PID 10796) with connector 2
2008-05-01 18:27:35,717 INFO  ***** FINISHED *****

An here is the load generated: