Analyse SLAPD Logs - Revision history

PeterHarding at 00:50, 2 April 2009

2009-04-02T00:50:21Z

← Older revision		Revision as of 10:50, 2 April 2009
Line 192:		Line 192:
	m1 = p_SRCH.search(rest)		m1 = p_SRCH.search(rest)
	if m1:		if m1:
	base = m1.group(1).replace(',dc=~~auspost~~,dc=com', '')		base = m1.group(1).replace(',dc=performiq,dc=com,dc=au', '')
	filter = m1.group(2)		filter = m1.group(2)
	rest = 'SRCH %s %s' % (base, filter)		rest = 'SRCH %s %s' % (base, filter)

PeterHarding at 07:46, 2 May 2008

2008-05-02T07:46:55Z

← Older revision		Revision as of 17:46, 2 May 2008
Line 1:		Line 1:
			=Overview=

			The script now also writes out the CN/DN and filters into files for re-processing for use in the test scripts.

	<pre>		<pre>
	$ cat analyse.py		$ cat analyse.py
Line 29:		Line 33:
	#-------------------------------------------------------------------------------		#-------------------------------------------------------------------------------

	__version__ = "1.1.0"		__version__ = "1.1.1"
	__id__ = "@(#) analyse.py [%s] 01/05/2008"		__id__ = "@(#) analyse.py [%s] 01/05/2008"

Line 65:		Line 69:
	try:		try:
	f = open(fname, 'r')		f = open(fname, 'r')
			except IOError, e:
			sys.stderr.write('[analyse] Open failed: ' + str(e) + '\n')
			sys.exit(1)

			try:
			h_bind = open('d_bind.dat', 'w')
			except IOError, e:
			sys.stderr.write('[analyse] Open failed: ' + str(e) + '\n')
			sys.exit(1)

			try:
			h_srch = open('d_srch.dat', 'w')
			except IOError, e:
			sys.stderr.write('[analyse] Open failed: ' + str(e) + '\n')
			sys.exit(1)

			try:
			h_mod = open('d_mod.dat', 'w')
			except IOError, e:
			sys.stderr.write('[analyse] Open failed: ' + str(e) + '\n')
			sys.exit(1)

			try:
			h_add = open('d_add.dat', 'w')
			except IOError, e:
			sys.stderr.write('[analyse] Open failed: ' + str(e) + '\n')
			sys.exit(1)

			try:
			h_del = open('d_del.dat', 'w')
	except IOError, e:		except IOError, e:
	sys.stderr.write('[analyse] Open failed: ' + str(e) + '\n')		sys.stderr.write('[analyse] Open failed: ' + str(e) + '\n')
Line 149:		Line 183:
	rest = 'BIND %s' % bind_dn		rest = 'BIND %s' % bind_dn
	stats[hour][BIND] += 1		stats[hour][BIND] += 1
			h_bind.write("%s\n" % bind_dn)

	m1 = p_UNBIND.search(rest)		m1 = p_UNBIND.search(rest)
Line 161:		Line 196:
	rest = 'SRCH %s %s' % (base, filter)		rest = 'SRCH %s %s' % (base, filter)
	stats[hour][SRCH] += 1		stats[hour][SRCH] += 1
			h_srch.write("%s %s\n" % (base, filter))

	m1 = p_MOD.search(rest)		m1 = p_MOD.search(rest)
Line 167:		Line 203:
	rest = 'MOD %s' % mod_dn		rest = 'MOD %s' % mod_dn
	stats[hour][MOD] += 1		stats[hour][MOD] += 1
			h_mod.write("%s\n" % mod_dn)

	m1 = p_ADD.search(rest)		m1 = p_ADD.search(rest)
Line 173:		Line 210:
	rest = 'ADD %s' % add_dn		rest = 'ADD %s' % add_dn
	stats[hour][ADD] += 1		stats[hour][ADD] += 1
			h_add.write("%s\n" % add_dn)

	m1 = p_DEL.search(rest)		m1 = p_DEL.search(rest)
Line 179:		Line 217:
	rest = 'DEL %s' % del_dn		rest = 'DEL %s' % del_dn
	stats[hour][DEL] += 1		stats[hour][DEL] += 1
			h_del.write("%s\n" % del_dn)

	if verbose_flg: print "%s %3d %s" % (time, conn, rest)		if verbose_flg: print "%s %3d %s" % (time, conn, rest)
Line 238:		Line 277:
	pid = os.getpid()		pid = os.getpid()

	log = logging.getLogger('~~analyse~~')		log = logging.getLogger('xxx')
	hdlr = logging.FileHandler(LOGFILE)		hdlr = logging.FileHandler(LOGFILE)
	fmtr = logging.Formatter('%(asctime)s %(levelname)s %(message)s')		fmtr = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

PeterHarding at 07:34, 2 May 2008

2008-05-02T07:34:17Z

← Older revision		Revision as of 17:34, 2 May 2008
Line 68:		Line 68:
	sys.stderr.write('[analyse] Open failed: ' + str(e) + '\n')		sys.stderr.write('[analyse] Open failed: ' + str(e) + '\n')
	sys.exit(1)		sys.exit(1)

	~~Data = {}~~

	s_result = 'RESULT'		s_result = 'RESULT'
Line 113:		Line 111:
	for i in range(24):		for i in range(24):
	stats.append(copy.deepcopy(counters))		stats.append(copy.deepcopy(counters))

			line_cnt = 0

	while True:		while True:
Line 118:		Line 118:

	if not line: break		if not line: break

			line_cnt += 1

	line = line.strip()		line = line.strip()
Line 179:		Line 181:

	if verbose_flg: print "%s %3d %s" % (time, conn, rest)		if verbose_flg: print "%s %3d %s" % (time, conn, rest)

	~~# Data[no] = data~~

	f.close()		f.close()
Line 226:		Line 226:
	)		)

	if debug_level > 5: INFO("Processed %d lines" % ~~(len(Data), )~~)		if debug_level > 5: INFO("Processed %d lines" % line_cnt)

	return ~~len(Data)~~		return line_cnt

	#===============================================================================		#===============================================================================

PeterHarding at 07:30, 2 May 2008

2008-05-02T07:30:29Z

← Older revision		Revision as of 17:30, 2 May 2008
Line 174:		Line 174:
	m1 = p_DEL.search(rest)		m1 = p_DEL.search(rest)
	if m1:		if m1:
	~~add_dn~~ = m1.group(1)		del_dn = m1.group(1)
	rest = 'DEL %s' % ~~add_dn~~		rest = 'DEL %s' % del_dn
	stats[hour][DEL] += 1		stats[hour][DEL] += 1

PeterHarding at 07:28, 2 May 2008

2008-05-02T07:28:59Z

← Older revision		Revision as of 17:28, 2 May 2008
Line 29:		Line 29:
	#-------------------------------------------------------------------------------		#-------------------------------------------------------------------------------

	__version__ = "1.0.0"		__version__ = "1.1.0"
	__id__ = "@(#) analyse.py [%s] 01/05/2008"		__id__ = "@(#) analyse.py [%s] 01/05/2008"

PeterHarding at 05:21, 2 May 2008

2008-05-02T05:21:32Z

@@ Line 3: / Line 3: @@
 #!/usr/bin/env python
+#
 #  Purpose:
+#
 #  $Id:$
@@ Line 36: / Line 36: @@
 debug_level   = 0
-LOGFILE       = "xxx.log"
+LOGFILE       = "log/analyse.log"
 log           = None
@@ Line 189: / Line 186: @@
     print "Hour   BIND  UNBIND    SRCH     MOD     ADD     DEL"
     for i in range(24):
-       print "%02d:  %6d  %6d  %6d  %6d  %6d  %6d" % (
+      counters[BIND]    +=  stats[i][BIND]
           i,
           stats[i][BIND],
@@ Line 197: / Line 201: @@
           stats[i][ADD],
           stats[i][DEL]
+       )
@@ Line 211: / Line 238: @@
     pid = os.getpid()
-    log  = logging.getLogger('xxx')
+    log  = logging.getLogger('analyse')
     hdlr = logging.FileHandler(LOGFILE)
     fmtr = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
@@ Line 236: / Line 263: @@
        print __doc__
        return 1
     for o, a in opts:
@@ Line 270: / Line 283: @@
     if (debug_level > 0): print "Debugging level set to %d" % debug_level
     init()
@@ Line 308: / Line 317: @@
 #-------------------------------------------------------------------------------
 </pre>
 [[Category:Python]]
 [[Category:LDAP]]

PeterHarding: New page:

 $ cat analyse.py  #!/usr/bin/env python # #  Purpose:  # #  $Id:$ # #-------------------------------------------------------------------------------  """   Analyse SLAPD logs    ... ...

2008-05-01T11:28:54Z

New page: <pre> $ cat analyse.py #!/usr/bin/env python # # Purpose: # # $Id:$ # #------------------------------------------------------------------------------- """ Analyse SLAPD logs ... ...

New page

<pre>
$ cat analyse.py
#!/usr/bin/env python
#
# Purpose:
#
# $Id:$
#
#-------------------------------------------------------------------------------

"""
Analyse SLAPD logs

...
"""
#-------------------------------------------------------------------------------

import os
import re
import sys
import copy
import getopt
import logging

#-------------------------------------------------------------------------------

from datetime import datetime

#-------------------------------------------------------------------------------

__version__ = "1.0.0"
__id__ = "@(#) analyse.py [%s] 01/05/2008"

verbose_flg = False

debug_level = 0

LOGFILE = "xxx.log"
PIDFILE = "analyse.pid"

tables = []

log = None
pid = None
file_name = None

#===============================================================================

def INFO(msg):
if log: log.info(' ' + msg)
if verbose_flg: print "[analyse] %s" % msg

#-------------------------------------------------------------------------------

def ERROR(msg):
if log: log.error(msg)
sys.stderr.write('[analyse] %s\n' % msg)

#-------------------------------------------------------------------------------

def WARNING(msg):
if log: log.warning('*****' + msg + '*****')
if verbose_flg: print "[analyse] %s" % msg

#===============================================================================

def analyse(fname):
try:
f = open(fname, 'r')
except IOError, e:
sys.stderr.write('[analyse] Open failed: ' + str(e) + '\n')
sys.exit(1)

Data = {}

s_result = 'RESULT'
p_result = re.compile(s_result)

BIND = 'BIND'
UNBIND = 'UNBIND'
SRCH = 'SRCH'
MOD = 'MOD'
ADD = 'ADD'
DEL = 'DEL'

s_BIND = r'BIND dn="([^"]*)" method=[0-9]*'
p_BIND = re.compile(s_BIND)
s_UNBIND = r'UNBIND'
p_UNBIND = re.compile(s_UNBIND)
s_SRCH = r'SRCH base="([^"]*)" scope=[0-9]* filter="([^"]*)"'
p_SRCH = re.compile(s_SRCH)
s_MOD = r'MOD dn="(.*)"'
p_MOD = re.compile(s_MOD)
s_ADD = r'ADD dn="([^"]*)"'
p_ADD = re.compile(s_ADD)
s_DEL = r'DEL dn="(.*)"'
p_DEL = re.compile(s_DEL)

pat = '(......) (.*) hx73 slapd\[[0-9]*\]: conn=([0-9]*) op=([0-9]*) (.*)'

p = re.compile(pat)

print p

stats = []

counters = {
'SRCH' : 0,
'MOD' : 0,
'ADD' : 0,
'DEL' : 0,
'BIND' : 0,
'UNBIND' : 0
}

for i in range(24):
stats.append(copy.deepcopy(counters))

while True:
line = f.readline()

if not line: break

line = line.strip()

m = p_result.search(line)

if m:
continue

m = p.search(line)

if m:
# print line
date = m.group(1)
time = m.group(2)
(hour, min, sec) = time.split(':')

hour = int(hour)
min = int(min)
sec = int(sec)

conn = int(m.group(3))
op = m.group(4)
rest = m.group(5)

m1 = p_BIND.search(rest)
if m1:
bind_dn = m1.group(1)
rest = 'BIND %s' % bind_dn
stats[hour][BIND] += 1

m1 = p_UNBIND.search(rest)
if m1:
rest = 'UNBIND'
stats[hour][UNBIND] += 1

m1 = p_SRCH.search(rest)
if m1:
base = m1.group(1).replace(',dc=auspost,dc=com', '')
filter = m1.group(2)
rest = 'SRCH %s %s' % (base, filter)
stats[hour][SRCH] += 1

m1 = p_MOD.search(rest)
if m1:
mod_dn = m1.group(1)
rest = 'MOD %s' % mod_dn
stats[hour][MOD] += 1

m1 = p_ADD.search(rest)
if m1:
add_dn = m1.group(1)
rest = 'ADD %s' % add_dn
stats[hour][ADD] += 1

m1 = p_DEL.search(rest)
if m1:
add_dn = m1.group(1)
rest = 'DEL %s' % add_dn
stats[hour][DEL] += 1

if verbose_flg: print "%s %3d %s" % (time, conn, rest)

# Data[no] = data

f.close()

print "Hour BIND UNBIND SRCH MOD ADD DEL"
for i in range(24):
print "%02d: %6d %6d %6d %6d %6d %6d" % (
i,
stats[i][BIND],
stats[i][UNBIND],
stats[i][SRCH],
stats[i][MOD],
stats[i][ADD],
stats[i][DEL]
)

if debug_level > 5: INFO("Processed %d lines" % (len(Data), ))

return len(Data)

#===============================================================================

def init():
global log
global pid

pid = os.getpid()

log = logging.getLogger('xxx')
hdlr = logging.FileHandler(LOGFILE)
fmtr = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

hdlr.setFormatter(fmtr)
log.addHandler(hdlr)
log.setLevel(logging.INFO)

INFO("Started processing")

if (not verbose_flg):
INFO("PID is %d" % pid)

#===============================================================================

def main():
global verbose_flg
global debug_level
global file_name

try:
opts, args = getopt.getopt(sys.argv[1:], "df:D:vVw?")
except getopt.error, msg:
print __doc__
return 1

try:
terminal_type = os.environ["TERM"]
except KeyError, e:
print "Set TERM environment variable and rerun!"
return 1

wrk_path = os.getcwd()
wrk_dir = os.path.basename(wrk_path)

data_dir = wrk_dir + '/DATA/'
pid_path = data_dir + PIDFILE

# os.chdir(data_dir)

for o, a in opts:
if o == '-d':
debug_level += 1
elif o == '-D':
debug_level = int(a)
elif o == '-f':
file_name = a
elif o == '-v':
verbose_flg = True
elif o == '-V':
print "[analyse] Version: %s" % __version__
return 1
elif o == '-?':
print __doc__
return 1

print "[analyse] Working directory is %s" % os.getcwd()

if (debug_level > 0): print "Debugging level set to %d" % debug_level

if args:
for arg in args:
print arg

init()

if file_name:
analyse(file_name)

return 0

#-------------------------------------------------------------------------------

if __name__ == '__main__' or __name__ == sys.argv[0]:
try:
sys.exit(main())
except KeyboardInterrupt, e:
print "[analyse] Interrupted!"

#-------------------------------------------------------------------------------

"""
Revision History:

Date Who Description
-------- --- ------------------------------------------------------------
20080501 plh Initial implementation

Problems to fix:

To Do:

Issues:

"""

#-------------------------------------------------------------------------------

</pre>

[[Category:Python]]
[[Category:LDAP]]