Sqlite setup.py

From PeformIQ Upgrade
Jump to navigation Jump to search

Sample Script

Sets up and works with an sqlite data source:


#!/usr/bin/env python
#
#       Author:  Peter Harding  <plh@performiq.com.au>
#
#                Mobile:  0418 375 085
#  
#           @(#) [1.0.01] sqlite_setup.py 2011-06-12
#
#
# NAME
#   sqlite_setup.py - Setup sqlite DB files
#
# SYNOPSIS
#   sqlite_setup.py [-dv]
#
# PARAMETERS
#   See __doc__ below
#
# DESCRIPTION
#   Manage sqlite repository for NABC test data.
#
# RETURNS
#   0 for successful completion, 1 for any error
#
# FILES
#   ...
#
#-----------------------------------------------------------------------
"""
Usage:

   $ sqlite_setup.py [-dv] 

Parameters:

   -d              Debug
   -w <date>       When (e.g.  -w  20110613)
   -t              User test data (equivalent to -w  TST)
   -v              Verbose


"""
#-----------------------------------------------------------------------

import os
import re
import sys
import glob
import time
import getopt
import sqlite3

#-----------------------------------------------------------------------

from datetime import datetime, timedelta

#-----------------------------------------------------------------------

__version__    = "1.0.0"

debug_lvl      = 0
quiet_flg      = False
verbose_flg    = False
freq_flg       = True
countries      = []
despatches     = {}
departures     = {}
wrk_dir        = None
f_log          = None

DB_NAME        = 'nabc.sqlite'


p_event_type   = re.compile(r':EventType>(.*)<')

#=======================================================================

class SqliteDB:
   Conn   = None
   DROP   = "DROP TABLE %s"

   CREATE = {
          'User' :  """
              create table User (
                 ID          INTEGER,
                 Username    TEXT,
                 LastName    TEXT,
                 FirstName   TEXT,
                 Org         TEXT,
                 Email       TEXT,
                 Roll        TEXT,
                 Active      INTEGER,
                 PageNo      INTEGER,
	         LastDT      DATE
              )""",

          'Org' : """
              create table Org (
                 ID      INTEGER,
                 OrgName TEXT,
                 NoUsers INTEGER,
                 LastDT  DATE
              )"""
           }

   SCHEMA = ".schema User Org"

   #--------------------------------------------------------------------

   def __init__(self, file):
      self.File        = file
      self.Conn        = sqlite3.connect(file)

   #--------------------------------------------------------------------

   def Close(self):
      if self.Conn:
         self.Conn.close()
         self.Conn = None

   #--------------------------------------------------------------------

   def Create(self):
      if self.Conn:
         c = self.Conn.cursor()

         c.execute(self.CREATE['User'])

         self.Conn.commit()

         c.execute(self.CREATE['Org'])

         self.Conn.commit()

         c.close()

   #--------------------------------------------------------------------

   def Schema(self):
      if self.Conn:
         c = self.Conn.cursor()

         c.execute(self.SCHEMA)

         c.close()

   #--------------------------------------------------------------------

   def Drop(self):
      if self.Conn:
         c = self.Conn.cursor()

         c.execute(self.DROP % 'User')

         self.Conn.commit()

         c.execute(self.DROP % 'Org')

         self.Conn.commit()

         c.close()

   #--------------------------------------------------------------------
   """

   """

   def Insert(self, event):
      if self.Conn:
         c = self.Conn.cursor()

         SQL = "insert into User values (%s, '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s')" % (
            customer.ID,
            customer.Username,
            customer.LastName,
            customer.FirstName,
            customer.Org,
            customer.Email,
            customer.Role,
            customer.Active,
            customer.PageNo,
            customer.LastDT.strftime("%Y-%m-%d %H:%M:%S"),
         )

         if verbose_flg:   print SQL

         c.execute(SQL)

         self.Conn.commit()

   #--------------------------------------------------------------------

   def Select():
      if self.Conn:
         c = self.Conn.cursor()

         QUERY = "select * from Org"

         c.execute(QUERY)

         c.close()


   #--------------------------------------------------------------------

   def __str__(self):
      return self.DbFile

#=======================================================================

class User:
   pass

   #--------------------------------------------------------------------

   def __init__(self, row):
      l                 = row.split(',')
      self.ID           = l[0]
      self.Username     = l[1]
      self.LastName     = l[2]
      self.FirstName    = l[3]
      self.Company      = l[4]
      self.Email        = l[5]
      self.Role         = l[6]
      self.Active       = l[7]
      self.PageNo       = l[8]
      self.Timestamp    = datetime.strptime(l[9], "%Y-%m-%d %H:%M:%S")

   #--------------------------------------------------------------------

   def __str__(self):
      return "%s: %s %s (%s)" % (
          self.ID,
          self.Username,
          self.FirstName,
          self.LastName,
      )

#=======================================================================

def load_file(name=None):
   if not name:
      return None

   file_path = '%s.csv' % name

   try:
      f_in = open(file_path, 'r')
   except IOError, msg:
      sys.stderr.write(f + ': cannot open: ' + `msg` + '\n')
      sys.exit(1)

   line_cnt    = 0
   no_events   = 0
   events      = []

   while True:
      line = f_in.readline()

      if not line: break

      line = line[:-1]

      event = Event(line)

      line_cnt += 1

      events.append(event)

   f_in.close()

   return events

#-----------------------------------------------------------------------

def db_import(names=None):
   global wrk_dir

   if not name:
      return False

   wrk_dir = os.getcwd()

   if not names:
      digits = '[0-9]' * 8
      names  = glob.glob(digits)

   for name in names:
      db_name = '%s.db' % name

      connection = SqliteDB(db_name)

      connection.Create()

      events = load_file(name)

      for event in events:
         connection.Insert(event)

      connection.Close()
 
   return True

#-----------------------------------------------------------------------

def create(db_name):
   connection = SqliteDB(db_name)

   connection.Create()

   connection.Close()

#-----------------------------------------------------------------------

def drop(db_name):
   connection = SqliteDB(db_name)

   connection.Drop()

   connection.Close()

#-----------------------------------------------------------------------

def schema(db_name):
   connection = SqliteDB(db_name)

   connection.Schema()

   connection.Close()

#-----------------------------------------------------------------------

def main():
   global debug_lvl
   global quiet_flg, verbose_flg
   global f_log

   try:
      username = os.environ['USER']
   except:
      print "Set USER environment variable and re-run"
      sys.exit(0)

   try:
      opts, args = getopt.getopt(sys.argv[1:], "CdD:f:lo:qstvVw:?X")
   except getopt.error, msg:
      print __doc__
      return 1

   names   = []
   log_flg = False

   db_name = DB_NAME

   for opt, arg in opts:
      if opt == '-d':
         debug_lvl       += 1
      elif opt == '-D':
         debug_lvl        = int(arg)
      elif opt == '-C':
         create(db_name)
         return 0
      elif opt == '-f':
         fname            = arg
      elif opt == '-l':
         log_flg          = True
      elif opt == '-n':
         db_name          = arg
      elif opt == '-o':
         orig             = arg
      elif opt == '-q':
         quiet_flg        = True
      elif opt == '-w':
         names.append(arg)
      elif opt == '-s':
         schema(db_name)
         return 0
      elif opt == '-t':
         names.append('TST')
      elif opt == '-v':
         verbose_flg      = True
      elif opt == '-V':
         print "Version: %s" % __version__
         return 0
      elif opt == '-X':
         drop(db_name)
         return 0
      elif opt == '-?':
         print __doc__
         return 0

   if log_flg:
      try:
         f_log = open('loader.log', 'a+')
      except IOError, msg:
         sys.stderr.write(f + ': cannot open: ' + `msg` + '\n')
         sys.exit(1)

   # db_import(names)

   return 0

#-----------------------------------------------------------------------

if __name__ == '__main__' or __name__ == sys.argv[0]:
   sys.exit(main())

#-----------------------------------------------------------------------

"""
Revision History:

     Date     Who   Description
   --------   ---   --------------------------------------------------
   20110612   plh   Initial implementation

Problems to fix:

To Do:

Issues:


Sqlite Help:

   .backup ?DB? FILE      Backup DB (default "main") to FILE
   .bail ON|OFF           Stop after hitting an error.  Default OFF
   .databases             List names and files of attached databases
   .dump ?TABLE? ...      Dump the database in an SQL text format
                            If TABLE specified, only dump tables matching
                            LIKE pattern TABLE.
   .echo ON|OFF           Turn command echo on or off
   .exit                  Exit this program
   .explain ?ON|OFF?      Turn output mode suitable for EXPLAIN on or off.
                            With no args, it turns EXPLAIN on.
   .header(s) ON|OFF      Turn display of headers on or off
   .help                  Show this message
   .import FILE TABLE     Import data from FILE into TABLE
   .indices ?TABLE?       Show names of all indices
                            If TABLE specified, only show indices for tables
                            matching LIKE pattern TABLE.
   .load FILE ?ENTRY?     Load an extension library
   .log FILE|off          Turn logging on or off.  FILE can be stderr/stdout
   .mode MODE ?TABLE?     Set output mode where MODE is one of:
                            csv      Comma-separated values
                            column   Left-aligned columns.  (See .width)
                            html     HTML <table> code
                            insert   SQL insert statements for TABLE
                            line     One value per line
                            list     Values delimited by .separator string
                            tabs     Tab-separated values
                            tcl      TCL list elements
   .nullvalue STRING      Print STRING in place of NULL values
   .output FILENAME       Send output to FILENAME
   .output stdout         Send output to the screen
   .prompt MAIN CONTINUE  Replace the standard prompts
   .quit                  Exit this program
   .read FILENAME         Execute SQL in FILENAME
   .restore ?DB? FILE     Restore content of DB (default "main") from FILE
   .schema ?TABLE?        Show the CREATE statements
                            If TABLE specified, only show tables matching
                            LIKE pattern TABLE.
   .separator STRING      Change separator used by output mode and .import
   .show                  Show the current values for various settings
   .stats ON|OFF          Turn stats on or off
   .tables ?TABLE?        List names of tables
                            If TABLE specified, only list tables matching
                            LIKE pattern TABLE.
   .timeout MS            Try opening locked tables for MS milliseconds
   .width NUM1 NUM2 ...   Set column widths for "column" mode
   .timer ON|OFF          Turn the CPU timer measurement on or off
"""