Spawning Mutliple Scripts
Revision as of 17:36, 2 May 2008 by PeterHarding (talk | contribs)
Simple Script Spawner
In this script I needed to spawn scripts to utilized a limited number of ODBC connectors I had set up. I also wanted the script to process overnight and so I needed to specify a duration. This script uses those old UNIX favorites fork()/exec()/wait() to manage the sub-scripts.
#! /usr/bin/env python
#
# $Id:$
#
# Purpose: Spawn multiple test scripts in a controlled way with logging
#
#-------------------------------------------------------------------------------
import sys
import re
import os
import time
import getopt
import string
import logging
import datetime
#-------------------------------------------------------------------------------
__version__ = '1.3.0'
debug_level = 0
verbose_flg = False
log = None
stagger_delay = 3
duration = None
no_connectors = 3
connectors = []
procs = {}
#===============================================================================
def INFO(msg):
if log: log.info(' ' + msg)
if verbose_flg: print "[test] %s" % msg
#-------------------------------------------------------------------------------
def ERROR(msg):
if log: log.error(msg)
sys.stderr.write('[test] %s\n' % msg)
#-------------------------------------------------------------------------------
def WARNING(msg):
if log: log.warning('*****' + msg + '*****')
if verbose_flg: print "[test] %s" % msg
#===============================================================================
def init():
global log
pid = os.getpid()
if debug_level > 0: print "My PID is %d" % pid
log = logging.getLogger('spawn')
hdlr = logging.FileHandler('log/spawn.log')
fmtr = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(fmtr)
"spawn.py" 197 lines, 5002 characters written
$ cat spawn.py
#! /usr/bin/env python
#
# $Id:$
#
# Purpose: Spawn multiple test scripts in a controlled way with logging
#
#-------------------------------------------------------------------------------
import sys
import re
import os
import time
import getopt
import string
import logging
import datetime
#-------------------------------------------------------------------------------
__version__ = '1.3.0'
debug_level = 0
verbose_flg = False
log = None
stagger_delay = 3
duration = None
no_connectors = 3
connectors = []
procs = {}
#===============================================================================
def INFO(msg):
if log: log.info(' ' + msg)
if verbose_flg: print "[test] %s" % msg
#-------------------------------------------------------------------------------
def ERROR(msg):
if log: log.error(msg)
sys.stderr.write('[test] %s\n' % msg)
#-------------------------------------------------------------------------------
def WARNING(msg):
if log: log.warning('*****' + msg + '*****')
if verbose_flg: print "[test] %s" % msg
#===============================================================================
def init():
global log
pid = os.getpid()
if debug_level > 0: print "My PID is %d" % pid
log = logging.getLogger('spawn')
hdlr = logging.FileHandler('log/spawn.log')
fmtr = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(fmtr)
log.addHandler(hdlr)
log.setLevel(logging.INFO)
INFO("===== Started processing ==================================")
#-------------------------------------------------------------------------
def do_child(connector, delay):
time.sleep(delay)
os.execv("./test.py", ["./test.py", "-c", "%d" % connector])
#-------------------------------------------------------------------------
def spawn():
global stagger_delay
loop = True
delay = 0
t_start = datetime.datetime.now()
while loop:
if len(connectors) > 0:
connector = connectors.pop(0)
child_pid = os.fork()
if child_pid == 0:
do_child(connector, delay)
else:
INFO("Spawned test process (PID %d) on connector %d" % (child_pid, connector))
procs[child_pid] = connector
delay += stagger_delay
else:
stagger_delay = 0
(pid, status) = os.wait()
connector = procs[pid]
INFO("cleaned up child process (PID %d) with connector %d" % (pid, connector))
del procs[pid]
t_now = datetime.datetime.now()
t_delta = t_now - t_start
if duration:
if t_delta < duration:
connectors.append(connector)
continue
if len(procs) == 0:
loop = False
INFO("***** FINISHED *****")
#-------------------------------------------------------------------------------
def usage():
USAGE = """
Usage:
$ spawn.py -n <no_to_spawn>
"""
sys.stderr.write(USAGE)
#-------------------------------------------------------------------------------
def main(argv):
global debug_flg, verbose_flg, no_connectors, duration, stagger_delay
#----- Process command line arguments ----------------------------
duration_min = None
try:
opts, args = getopt.getopt(argv, "dD:hn:s:vV",
["debug", "duration=", "help", "no=", "stagger=", "verbose", "version"])
except getopt.GetoptError:
usage()
sys.exit(2)
else:
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
sys.exit(0)
elif opt == "-d":
debug_level += 1
elif opt == "--debug":
debug_level = int(arg)
elif opt in ("-D", "--duration"):
duration_min = int(arg)
elif opt in ("-s", "--stagger"):
stagger_delay = int(arg)
elif opt in ("-n", "--no"):
no_connectors = int(arg)
if no_connectors > 10:
no_connectors = 10
sys.stderr.write("Only 10 ODBC connectors available!")
sys.stderr.flush()
elif opt in ("-v", "--verbose"):
verbose_flg = True
elif opt in ("-V", "--version"):
print "Version: %s" % __version__
sys.exit(0)
if duration_min:
print "[spawn] Running for a duration of %d minutes" % duration_min
duration = datetime.timedelta(minutes=duration_min)
for i in range(no_connectors):
connectors.append(i + 1)
print "[spawn] Using these ODBC connectors - %s" % connectors
init()
spawn()
#---------------------------------------------------------------------
if __name__ == "__main__":
main(sys.argv[1:])
#---------------------------------------------------------------------
The log file output looks like this:
2008-05-01 18:22:00,577 INFO ===== Started processing ================================== 2008-05-01 18:22:00,608 INFO Spawned test process (PID 11348) on connector 1 2008-05-01 18:22:00,640 INFO Spawned test process (PID 12216) on connector 2 2008-05-01 18:22:26,983 INFO cleaned up child process (PID 11348) with connector 1 2008-05-01 18:22:27,374 INFO cleaned up child process (PID 12216) with connector 2 2008-05-01 18:22:27,374 INFO ***** FINISHED ***** 2008-05-01 18:23:37,359 INFO ===== Started processing ================================== 2008-05-01 18:23:37,390 INFO Spawned test process (PID 11068) on connector 1 2008-05-01 18:23:37,437 INFO Spawned test process (PID 12252) on connector 2 2008-05-01 18:24:03,749 INFO cleaned up child process (PID 11068) with connector 1 2008-05-01 18:24:03,780 INFO cleaned up child process (PID 12252) with connector 2 2008-05-01 18:24:03,780 INFO ***** FINISHED ***** 2008-05-01 18:27:08,390 INFO ===== Started processing ================================== 2008-05-01 18:27:08,437 INFO Spawned test process (PID 11720) on connector 1 2008-05-01 18:27:18,453 INFO Spawned test process (PID 10796) on connector 2 2008-05-01 18:27:28,453 INFO cleaned up child process (PID 11720) with connector 1 2008-05-01 18:27:35,717 INFO cleaned up child process (PID 10796) with connector 2 2008-05-01 18:27:35,717 INFO ***** FINISHED *****
An here is the load generated:
