Difference between revisions of "Synthesizing SVT Data"

From PeformIQ Upgrade
Jump to navigation Jump to search
(New page: =Overview= Preparing data for performance tests is a very important part of the preparation phase for a load test exercise. The following scripts provide simple examples of approaches to...)
 
 
(11 intermediate revisions by the same user not shown)
Line 13: Line 13:
   print "svt%03d" % i
   print "svt%03d" % i
</pre>
</pre>
=Unquoting URL Data=
<pre>
#!/usr/bin/env python
import urllib                                                                                                                                                                     
encoded_data = """
%3CPromptEngineRuntimeContext%20xmlns%3D%22http%3A%2F%2Fwww.crystaldecisions.com%2Freport%22%20xmlns%3Axsi%3D%22http%3A%2F%2F
www.w3.org%2F2001%2FXMLSchema-instance%22%20xmlns%3Axsd%3D%22http%3A%2F%2Fwww.w3.org%2F2001%2FXMLSchema%22%20xsi%3Atype%3D%22
PromptEngineRuntimeContext%22%20version%3D%222%22%3E%3CID%3E{MachineXID}%3C%2FID%3E%3CRenderOption%20xsi%3Atype%3D%22
BOwner%26gt%3B%26lt%3B%2FOwner%26gt%3B%26lt%3BDefinition%20xsi%3Atype%3D%22Parameter%22%20id%3D%225%22%26gt%3B%26lt%3B",                                                                       
"""                                                                                                                                                                               
data = urllib.unquote(encoded_data)                                                                                                                                                             
print data                         
</pre>
=Generating Synthetic Credit Card Numbers=
<pre>
#!/usr/bin/env python
#
#
#
#-------------------------------------------------------------------------------
"""
Generate a abitrary collection of pseudo Credit card numbers which
satisfy the validation test
"""
import sys
import random
#-------------------------------------------------------------------------------
prefix      = '1234'
max_digits  = 16
max_numbers = 1000
#-------------------------------------------------------------------------------
def gen(prefix):
  no = prefix
  l = list(no)
  i = len(prefix)
  while (i < (max_digits - 1)):
      x = random.randint(0, 9)
      l.append(str(x))
      # print i, l, x
      i += 1
  no = ''.join(l)
  chk_digit = encode(no)
  delta = 10 - chk_digit
  if delta == 10:
      delta = 0
  # print chk_digit, delta
  l.append(str(delta))
  no = ''.join(l)
  return no
#-------------------------------------------------------------------------------
def encode(no):
  n  = 0
  tot = 0
  for i in range(len(no)):
      v = int(no[i])
      if i % 2 == 0:
        inc = v * 2
        if v > 4:
            n += 1
      else:
        inc = v
      tot += inc
  tot += n
  chk = tot % 10
  return chk
#-------------------------------------------------------------------------------
def validate(ccnumber):
  cnt = 0
  tot = 0
  for i in range(len(ccnumber)):
      val = int(ccnumber[i])
      if i % 2 == 0:
        inc = val * 2
        if val > 4:
            cnt += 1
      else:
        inc = val
      tot += inc
  tot += cnt
  if ((tot % 10) == 0):
      return True
  else:
      return False
#-------------------------------------------------------------------------------
def test():
  for no in nos:
      print no
      if validate(no):
        print "Card number VALID"
      else:
        print "Card number INVALID"
#-------------------------------------------------------------------------------
def main():
  i = 0
  while i < max_numbers:
      cc_no = gen(prefix)
     
      if validate(cc_no):
        print cc_no
      else:
        print "Card number %s - INVALID" % cc_no
        cc_no = None
      if cc_no:
        i += 1
      else:
        pass
#-------------------------------------------------------------------------------
if __name__ == "__main__":
    main(sys.argv[1:])
#-------------------------------------------------------------------------------
</pre>
=Generating User Profiles=
This script generated user profiles along with associated (synthetically generated) credit card numbers.
<pre>
#!/usr/bin/env python
#
#  $Id:$
#
#-------------------------------------------------------------------------------
"""
Generate an abitrary collection of pseudo Credit Card numbers which
satisfy the validation test
"""
import sys
import random
import getopt
#-------------------------------------------------------------------------------
prefix      = '1234'
max_digits  = 16
max_numbers = 1000
#-------------------------------------------------------------------------------
cost_centers = [
  ["XX100000-X", "XXX DIVISION XX100000"],
  ["XX100001-X", "XXX DIVISION XX100001"],
  ...
]
#-------------------------------------------------------------------------------
def gen(prefix):
  no = prefix
  l = list(no)
  i = len(prefix)
  while (i < (max_digits - 1)):
      x = random.randint(0, 9)
      l.append(str(x))
      # print i, l, x
      i += 1
  no = ''.join(l)
  chk_digit = encode(no)
  delta = 10 - chk_digit
  if delta == 10:
      delta = 0
  # print chk_digit, delta
  l.append(str(delta))
  no = ''.join(l)
  return no
#-------------------------------------------------------------------------------
def encode(no):
  n  = 0
  tot = 0
  for i in range(len(no)):
      v = int(no[i])
      if i % 2 == 0:
        inc = v * 2
        if v > 4:
            n += 1
      else:
        inc = v
      tot += inc
  tot += n
  chk = tot % 10
  return chk
#-------------------------------------------------------------------------------
def validate(ccnumber):
  cnt = 0
  tot = 0
  for i in range(len(ccnumber)):
      val = int(ccnumber[i])
      if i % 2 == 0:
        inc = val * 2
        if val > 4:
            cnt += 1
      else:
        inc = val
      tot += inc
  tot += cnt
  if ((tot % 10) == 0):
      return True
  else:
      return False
#-------------------------------------------------------------------------------
def test():
  for no in nos:
      print no
      if validate(no):
        print "Card number VALID"
      else:
        print "Card number INVALID"
#-------------------------------------------------------------------------------
def gen_user_data():
  out = open('Users.dat','w')
  out.write("AGS_Number,FirstName,Username,Password,CostCenter,CreditCardNumber\r\n")
  i = 0
  no_centers = len(cost_centers)
  while i < max_numbers:
      center_idx = random.randint(0, no_centers - 1)
      cost_center = cost_centers[center_idx][0]
      cc_no = gen(prefix)
     
      if validate(cc_no):
        out.write("test_%03d,%03d,test%03d,test%03d,%s,%s\r\n" % \
            (i, i, i, i, cost_center, cc_no))
      else:
        print "Card number %s - INVALID" % cc_no
        cc_no = None
      if cc_no:
        i += 1
      else:
        pass
#-------------------------------------------------------------------------------
USAGE = """\
  Usage:
    $ ./gen_user_data.py [-d] [-v] [-f <filename>]
"""
def usage():
  sys.stderr.write(USAGE)
#-------------------------------------------------------------------------------
def main(argv):
  global debug_flg
  global verbose_flg
  global file
  #----- Process command line arguments ----------------------------
  try:
      opts, args = getopt.getopt(argv, "df:hv", ["debug", "file=", "help", "verbose"])
  except getopt.GetoptError:
      usage()
      sys.exit(2)
  else:
      for opt, arg in opts:
        if opt in ("-d", "--debug"):
            debug_flg = True
        elif opt in ("-h", "--help"):
            usage()
            sys.exit(0)
        elif opt in ("-f", "--file"):
            logfile = arg
        elif opt in ("-v", "--verbose"):
            verbose_flg = True
  gen_user_data()
#-------------------------------------------------------------------------------
if __name__ == "__main__":
    main(sys.argv[1:])
#-------------------------------------------------------------------------------
</pre>
=Synthetic Expense Data=
The following was used to generate synthetic credit card usage data for importing into the application via a batch import /upload process.  This data was then used by LoadRunner scripts to journal expenses against cost codes for subsequent authorization and further processing.
<pre>
#!/usr/bin/env python
#
#---------------------------------------------------------------------
import os
import csv
import sys
import getopt
import random
import datetime
#---------------------------------------------------------------------
debug_flg    = False
max_flg      = False
random_flg    = False
verbose_flg  = False
__version__  = '1.0.2'
file          = None
no_days      = 1
date          = datetime.datetime.today()
dates        = ['070729', '070730', '070731']
xcodes        = [
                  'xxx',
...
                  'xxx'
                ]
#----- Set up credit card transactiuon type list ---------------------
cc_merchants = []
cc_merchants_in  = open("cc_merchants.csv", "rb")
reader = csv.reader(cc_merchants_in)
for row in reader:
  cc_merchants.append(row)
cc_merchants_in.close()  # Explicitly close the file *NOW*
no_merchants = len(cc_merchants)
#----- Set up credit card numbers in cc_no list ----------------------
cc_no = []
cc_in  = open("cc_no.txt", "rb")
reader = csv.reader(cc_in)
for row in reader:
  cc_no.extend(row)
cc_in.close()  # Explicitly close the file *NOW*
#----- Set up the file header and footer -----------------------------
header = """\
...
"""
footer = """\
...
"""
#----- Set up the body of the file -----------------------------------
max_no_tx =  10  # Number of transactions per CC
cost      = 1000  # Cost of the transaction
tx_id    =    0  # Number that appears in transaction description
def write_data(date):
  global cost, tx_id
  file_date    = date.strftime("%y%m%d")
  expense_date = (date - datetime.timedelta(days=1)).strftime("%y%m%d")
  print "Generating expense data for '%s'" % expense_date
  out = open(file_date,"w")
  out.write(header % (file_date, expense_date))
  for i in range(len(cc_no)):
      if random_flg:
        no_tx      = random.randint(0, max_no_tx)
      elif max_flg:
        no_tx      = 5
      else:
        no_tx      = 1
      tx_cnt        = 0
      total        = 0.0
      expense_lines = ''
      while tx_cnt < no_tx:
        cost = random.weibullvariate(alpha=3.0,beta=1) * 2000.0
        type = cc_merchants[random.randint(0, no_merchants-1)]
        expense_lines += "16,699,%.0f,,%s,,%s\n" % (cost, type[0], type[1])
        total    += cost
        tx_cnt  += 1
        tx_id    += 1
        cost    += 5
      expense_block  = "01,%s,,...,%03.0f,%d.../\n" % (cc_no[i], total, tx_cnt)
      expense_block += expense_lines
      expense_block += "99,...,%d/\n" % (tx_cnt + 2)
      out.write(expense_block)
  out.write(footer)
  out.close()
#-------------------------------------------------------------------------------
USAGE = """\
  Usage:
    $ ./gen_user_data.py [-d] [-v] [-f <filename>]
"""
def usage():
  sys.stderr.write(USAGE)
#-------------------------------------------------------------------------------
def main(argv):
  global debug_flg
  global max_flg
  global random_flg
  global verbose_flg
  global file, date, no_days
  #----- Process command line arguments ----------------------------
  try:
      opts, args = getopt.getopt(argv, "dD:f:hmn:rv",
                ["date", "debug", "file=", "help", "max", "no_days", "random", "verbose"])
  except getopt.GetoptError:
      usage()
      sys.exit(2)
  else:
      for opt, arg in opts:
        if opt in ("-d", "--debug"):
            debug_flg = True
        elif opt in ("-D", "--date"):
            target_date = arg
            date = datetime.datetime.strptime(target_date, "%Y%m%d")
            print date
        elif opt in ("-f", "--file"):
            logfile = arg
        elif opt in ("-h", "--help"):
            usage()
            sys.exit(0)
        elif opt in ("-m", "--max"):
            max_flg = True
        elif opt in ("-n", "--no_days"):
            no_days = int(arg)
        elif opt in ("-r", "--random"):
            random_flg = True
        elif opt in ("-v", "--verbose"):
            verbose_flg = True
  dates = []
  for n in range(no_days):
      d = date - datetime.timedelta(days=n)
      print d.strftime("%y%m%d")
      dates.append(d)
  # print date.strftime("%y%m%d")
  for d in dates:
      write_data(d)
#-------------------------------------------------------------------------------
if __name__ == "__main__":
    main(sys.argv[1:])
#-------------------------------------------------------------------------------
</pre>
[[Category:Python]]


=Other Examples=
=Other Examples=


* [[Python - File operations]]
* [[Python - File sorts]]


[[Category:LoadRunner]]
[[Category:LoadRunner]]
[[Category:Python]]
[[Category:Python]]

Latest revision as of 12:10, 2 April 2009

Overview

Preparing data for performance tests is a very important part of the preparation phase for a load test exercise. The following scripts provide simple examples of approaches to synthesize parameter data for use by the test scripts.

User Login IDs

#!/usr/bin/env python

print 'UserId'

for i in range(200):
   print "svt%03d" % i

Unquoting URL Data

#!/usr/bin/env python

import urllib                                                                                                                                                                       

encoded_data = """
%3CPromptEngineRuntimeContext%20xmlns%3D%22http%3A%2F%2Fwww.crystaldecisions.com%2Freport%22%20xmlns%3Axsi%3D%22http%3A%2F%2F
www.w3.org%2F2001%2FXMLSchema-instance%22%20xmlns%3Axsd%3D%22http%3A%2F%2Fwww.w3.org%2F2001%2FXMLSchema%22%20xsi%3Atype%3D%22
PromptEngineRuntimeContext%22%20version%3D%222%22%3E%3CID%3E{MachineXID}%3C%2FID%3E%3CRenderOption%20xsi%3Atype%3D%22
BOwner%26gt%3B%26lt%3B%2FOwner%26gt%3B%26lt%3BDefinition%20xsi%3Atype%3D%22Parameter%22%20id%3D%225%22%26gt%3B%26lt%3B",                                                                         
"""                                                                                                                                                                                 

data = urllib.unquote(encoded_data)                                                                                                                                                               

print data                           

Generating Synthetic Credit Card Numbers

#!/usr/bin/env python
#
#
#
#-------------------------------------------------------------------------------

"""
Generate a abitrary collection of pseudo Credit card numbers which
satisfy the validation test
"""

import sys
import random

#-------------------------------------------------------------------------------

prefix      = '1234'
max_digits  = 16
max_numbers = 1000

#-------------------------------------------------------------------------------

def gen(prefix):
   no = prefix

   l = list(no)

   i = len(prefix)

   while (i < (max_digits - 1)):
      x = random.randint(0, 9)

      l.append(str(x))
      # print i, l, x
      i += 1

   no = ''.join(l)

   chk_digit = encode(no) 

   delta = 10 - chk_digit

   if delta == 10:
      delta = 0

   # print chk_digit, delta

   l.append(str(delta))

   no = ''.join(l)

   return no

#-------------------------------------------------------------------------------

def encode(no):
   n   = 0
   tot = 0

   for i in range(len(no)):
      v = int(no[i])

      if i % 2 == 0:
         inc = v * 2
         if v > 4:
            n += 1
      else:
         inc = v

      tot += inc

   tot += n

   chk = tot % 10

   return chk

#-------------------------------------------------------------------------------

def validate(ccnumber):
   cnt = 0
   tot = 0

   for i in range(len(ccnumber)):
      val = int(ccnumber[i])

      if i % 2 == 0:
         inc = val * 2
         if val > 4:
            cnt += 1
      else:
         inc = val

      tot += inc

   tot += cnt

   if ((tot % 10) == 0):
      return True
   else:
      return False

#-------------------------------------------------------------------------------

def test():
   for no in nos:
      print no

      if validate(no):
         print "Card number VALID"
      else:
         print "Card number INVALID"

#-------------------------------------------------------------------------------

def main():
   i = 0

   while i < max_numbers:
      cc_no = gen(prefix)
      
      if validate(cc_no):
         print cc_no
      else:
         print "Card number %s - INVALID" % cc_no
         cc_no = None

      if cc_no:
         i += 1
      else:
         pass

#-------------------------------------------------------------------------------

if __name__ == "__main__":
    main(sys.argv[1:])

#-------------------------------------------------------------------------------

Generating User Profiles

This script generated user profiles along with associated (synthetically generated) credit card numbers.

#!/usr/bin/env python
#
#  $Id:$
#
#-------------------------------------------------------------------------------

"""
Generate an abitrary collection of pseudo Credit Card numbers which
satisfy the validation test
"""

import sys
import random
import getopt

#-------------------------------------------------------------------------------

prefix      = '1234'
max_digits  = 16
max_numbers = 1000

#-------------------------------------------------------------------------------

cost_centers = [
   ["XX100000-X", "XXX DIVISION XX100000"],
   ["XX100001-X", "XXX DIVISION XX100001"],
   ...
]

#-------------------------------------------------------------------------------

def gen(prefix):
   no = prefix

   l = list(no)

   i = len(prefix)

   while (i < (max_digits - 1)):
      x = random.randint(0, 9)

      l.append(str(x))
      # print i, l, x
      i += 1

   no = ''.join(l)

   chk_digit = encode(no) 

   delta = 10 - chk_digit

   if delta == 10:
      delta = 0

   # print chk_digit, delta

   l.append(str(delta))

   no = ''.join(l)

   return no

#-------------------------------------------------------------------------------

def encode(no):
   n   = 0
   tot = 0

   for i in range(len(no)):
      v = int(no[i])

      if i % 2 == 0:
         inc = v * 2
         if v > 4:
            n += 1
      else:
         inc = v

      tot += inc

   tot += n

   chk = tot % 10

   return chk

#-------------------------------------------------------------------------------

def validate(ccnumber):
   cnt = 0
   tot = 0

   for i in range(len(ccnumber)):
      val = int(ccnumber[i])

      if i % 2 == 0:
         inc = val * 2
         if val > 4:
            cnt += 1
      else:
         inc = val

      tot += inc

   tot += cnt

   if ((tot % 10) == 0):
      return True
   else:
      return False

#-------------------------------------------------------------------------------

def test():
   for no in nos:
      print no

      if validate(no):
         print "Card number VALID"
      else:
         print "Card number INVALID"

#-------------------------------------------------------------------------------

def gen_user_data():
   out = open('Users.dat','w')

   out.write("AGS_Number,FirstName,Username,Password,CostCenter,CreditCardNumber\r\n")

   i = 0

   no_centers = len(cost_centers)

   while i < max_numbers:
      center_idx = random.randint(0, no_centers - 1)

      cost_center = cost_centers[center_idx][0]

      cc_no = gen(prefix)
      
      if validate(cc_no):
         out.write("test_%03d,%03d,test%03d,test%03d,%s,%s\r\n" % \
             (i, i, i, i, cost_center, cc_no))
      else:
         print "Card number %s - INVALID" % cc_no
         cc_no = None

      if cc_no:
         i += 1
      else:
         pass

#-------------------------------------------------------------------------------

USAGE = """\

  Usage:

     $ ./gen_user_data.py [-d] [-v] [-f <filename>]

"""


def usage():
   sys.stderr.write(USAGE)

#-------------------------------------------------------------------------------

def main(argv):
   global debug_flg
   global verbose_flg
   global file

   #----- Process command line arguments ----------------------------

   try:
      opts, args = getopt.getopt(argv, "df:hv", ["debug", "file=", "help", "verbose"])
   except getopt.GetoptError:
      usage()
      sys.exit(2)
   else:
      for opt, arg in opts:
         if opt in ("-d", "--debug"):
            debug_flg = True
         elif opt in ("-h", "--help"):
            usage()
            sys.exit(0)
         elif opt in ("-f", "--file"):
            logfile = arg
         elif opt in ("-v", "--verbose"):
            verbose_flg = True

   gen_user_data()

#-------------------------------------------------------------------------------

if __name__ == "__main__":
    main(sys.argv[1:])

#-------------------------------------------------------------------------------

Synthetic Expense Data

The following was used to generate synthetic credit card usage data for importing into the application via a batch import /upload process. This data was then used by LoadRunner scripts to journal expenses against cost codes for subsequent authorization and further processing.

#!/usr/bin/env python
#
#---------------------------------------------------------------------

import os
import csv
import sys
import getopt
import random
import datetime

#---------------------------------------------------------------------

debug_flg     = False
max_flg       = False
random_flg    = False
verbose_flg   = False

__version__   = '1.0.2'

file          = None

no_days       = 1
date          = datetime.datetime.today()

dates         = ['070729', '070730', '070731']

xcodes        = [
                  'xxx', 
...
                  'xxx'
                ]


#----- Set up credit card transactiuon type list ---------------------

cc_merchants = []

cc_merchants_in  = open("cc_merchants.csv", "rb")

reader = csv.reader(cc_merchants_in)

for row in reader:
  cc_merchants.append(row)

cc_merchants_in.close()  # Explicitly close the file *NOW*

no_merchants = len(cc_merchants)

#----- Set up credit card numbers in cc_no list ----------------------

cc_no = []

cc_in  = open("cc_no.txt", "rb")

reader = csv.reader(cc_in)

for row in reader:
  cc_no.extend(row)

cc_in.close()  # Explicitly close the file *NOW*

#----- Set up the file header and footer -----------------------------

header = """\
...
"""

footer = """\
...
"""

#----- Set up the body of the file -----------------------------------

max_no_tx =   10  # Number of transactions per CC
cost      = 1000  # Cost of the transaction
tx_id     =    0  # Number that appears in transaction description

def write_data(date):
   global cost, tx_id

   file_date    = date.strftime("%y%m%d")
   expense_date = (date - datetime.timedelta(days=1)).strftime("%y%m%d")

   print "Generating expense data for '%s'" % expense_date

   out = open(file_date,"w")

   out.write(header % (file_date, expense_date))

   for i in range(len(cc_no)): 
      if random_flg:
         no_tx      = random.randint(0, max_no_tx)
      elif max_flg:
         no_tx      = 5
      else:
         no_tx      = 1

      tx_cnt        = 0
      total         = 0.0
      expense_lines = ''

      while tx_cnt < no_tx:
         cost = random.weibullvariate(alpha=3.0,beta=1) * 2000.0
         type = cc_merchants[random.randint(0, no_merchants-1)]

         expense_lines += "16,699,%.0f,,%s,,%s\n" % (cost, type[0], type[1])

         total    += cost
         tx_cnt   += 1
         tx_id    += 1
         cost     += 5

      expense_block  = "01,%s,,...,%03.0f,%d.../\n" % (cc_no[i], total, tx_cnt)
      expense_block += expense_lines
      expense_block += "99,...,%d/\n" % (tx_cnt + 2)

      out.write(expense_block)

   out.write(footer)
   out.close()

#-------------------------------------------------------------------------------

USAGE = """\

  Usage:

     $ ./gen_user_data.py [-d] [-v] [-f <filename>]

"""

def usage():
   sys.stderr.write(USAGE)

#-------------------------------------------------------------------------------

def main(argv):
   global debug_flg
   global max_flg
   global random_flg
   global verbose_flg
   global file, date, no_days

   #----- Process command line arguments ----------------------------

   try:
      opts, args = getopt.getopt(argv, "dD:f:hmn:rv",
                 ["date", "debug", "file=", "help", "max", "no_days", "random", "verbose"])
   except getopt.GetoptError:
      usage()
      sys.exit(2)
   else:
      for opt, arg in opts:
         if opt in ("-d", "--debug"):
            debug_flg = True
         elif opt in ("-D", "--date"):
            target_date = arg
            date = datetime.datetime.strptime(target_date, "%Y%m%d")
            print date
         elif opt in ("-f", "--file"):
            logfile = arg
         elif opt in ("-h", "--help"):
            usage()
            sys.exit(0)
         elif opt in ("-m", "--max"):
            max_flg = True
         elif opt in ("-n", "--no_days"):
            no_days = int(arg)
         elif opt in ("-r", "--random"):
            random_flg = True
         elif opt in ("-v", "--verbose"):
            verbose_flg = True

   dates = []

   for n in range(no_days):
      d = date - datetime.timedelta(days=n)
      print d.strftime("%y%m%d")
      dates.append(d)

   # print date.strftime("%y%m%d")

   for d in dates:
      write_data(d)

#-------------------------------------------------------------------------------

if __name__ == "__main__":
    main(sys.argv[1:])

#-------------------------------------------------------------------------------

Other Examples