Examples - tst sqlite.py
Jump to navigation
Jump to search
Using Sqlite
#!/usr/bin/env python
#
# Author: Peter Harding <plh@performiq.com.au>
#
# PerformIQ Pty. Ltd.
# Level 6,
# 170 Queen Street,
# MELBOURNE, VIC, 3000
#
# Fax: 03 9578 7810
# Mobile: 0418 375 085
#
# Copyright (C) Peter Harding, 2003-2011
# All rights reserved
#
# @(#) [1.0.01] datasetup.py 2011-06-12
#
#
# NAME
# datasetup.py - Setup data for JMS injection script
#
# SYNOPSIS
# datasetup.py [-dv]
#
# PARAMETERS
# See __doc__ below
#
# DESCRIPTION
# Analyse barcodes
#
# RETURNS
# 0 for successful completion, 1 for any error
#
# FILES
# ...
#
#-----------------------------------------------------------------------
"""
Usage:
$ datasetup.py [-dv]
Parameters:
-d Debug
-w <date> When (e.g. -w 20110613)
-t User test data (equivalent to -w TST)
-v Verbose
"""
#-----------------------------------------------------------------------
import os
import re
import sys
import glob
import time
import getopt
import sqlite3
#-----------------------------------------------------------------------
from datetime import datetime, timedelta
#-----------------------------------------------------------------------
__version__ = "1.0.0"
debug_lvl = 0
quiet_flg = False
verbose_flg = False
freq_flg = True
countries = []
despatches = {}
departures = {}
wrk_dir = None
f_log = None
p_event_type = re.compile(r':EventType>(.*)<')
#=======================================================================
class SqliteDB:
Conn = None
DROP = "DROP TABLE %s"
CREATE = {
'Customer' : """
create table Customer (
ID INTEGER,
Username TEXT,
LastName TEXT,
FirstName TEXT,
Company TEXT,
Email TEXT,
Roll TEXT,
Active INTEGER,
PageNo INTEGER,
LastDT DATE
)""",
'Company' : """
create table Company (
ID INTEGER,
Company TEXT,
NoUsers INTEGER,
LastDT DATE
)"""
}
SCHEMA = ".schema Events Company"
#--------------------------------------------------------------------
def __init__(self, file):
self.File = file
self.Conn = sqlite3.connect(file)
#--------------------------------------------------------------------
def Close(self):
if self.Conn:
self.Conn.close()
self.Conn = None
#--------------------------------------------------------------------
def Create(self):
if self.Conn:
c = self.Conn.cursor()
c.execute(self.CREATE['Customer'])
self.Conn.commit()
c.execute(self.CREATE['Company'])
self.Conn.commit()
c.close()
#--------------------------------------------------------------------
def Schema(self):
if self.Conn:
c = self.Conn.cursor()
c.execute(self.SCHEMA)
c.close()
#--------------------------------------------------------------------
def Drop(self):
if self.Conn:
c = self.Conn.cursor()
c.execute(self.DROP % 'Customer')
self.Conn.commit()
c.execute(self.DROP % 'Company')
self.Conn.commit()
c.close()
#--------------------------------------------------------------------
"""
"""
def Insert(self, event):
if self.Conn:
c = self.Conn.cursor()
SQL = "insert into Customer values (%s, '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s')" % (
customer.ID,
customer.Username,
customer.LastName,
customer.FirstName,
customer.Company,
customer.Email,
customer.Role,
customer.Active,
customer.PageNo,
customer.LastDT.strftime("%Y-%m-%d %H:%M:%S"),
)
if verbose_flg: print SQL
c.execute(SQL)
self.Conn.commit()
#--------------------------------------------------------------------
def Select():
if self.Conn:
c = self.Conn.cursor()
QUERY = "select * from Customers"
c.execute(QUERY)
c.close()
#--------------------------------------------------------------------
def __str__(self):
return self.DbFile
#=======================================================================
class Customer:
pass
#--------------------------------------------------------------------
def __init__(self, row):
l = row.split(',')
self.ID = l[0]
self.Username = l[1]
self.LastName = l[2]
self.FirstName = l[3]
self.Company = l[4]
self.Email = l[5]
self.Role = l[6]
self.Active = l[7]
self.PageNo = l[8]
self.Timestamp = datetime.strptime(l[9], "%Y-%m-%d %H:%M:%S")
#--------------------------------------------------------------------
def __str__(self):
return "%s: %s %s (%s)" % (
self.ID,
self.Username,
self.FirstName,
self.LastName,
)
#=======================================================================
def load_file(name=None):
if not name:
return None
file_path = '%s.csv' % name
try:
f_in = open(file_path, 'r')
except IOError, msg:
sys.stderr.write(f + ': cannot open: ' + `msg` + '\n')
sys.exit(1)
line_cnt = 0
no_events = 0
events = []
while True:
line = f_in.readline()
if not line: break
line = line[:-1]
event = Event(line)
line_cnt += 1
events.append(event)
f_in.close()
return events
#-----------------------------------------------------------------------
def db_import(names=None):
global wrk_dir
if not name:
return False
wrk_dir = os.getcwd()
if not names:
digits = '[0-9]' * 8
names = glob.glob(digits)
for name in names:
db_name = '%s.db' % name
connection = SqliteDB(db_name)
connection.Create()
events = load_file(name)
for event in events:
connection.Insert(event)
connection.Close()
return True
#-----------------------------------------------------------------------
db_name = 'pcms_data.db'
def create(db_name):
connection = SqliteDB(db_name)
connection.Create()
connection.Close()
#-----------------------------------------------------------------------
def drop(db_name):
connection = SqliteDB(db_name)
connection.Drop()
connection.Close()
#-----------------------------------------------------------------------
def schema(db_name):
connection = SqliteDB(db_name)
connection.Schema()
connection.Close()
#-----------------------------------------------------------------------
def main():
global debug_lvl
global quiet_flg, verbose_flg
global f_log
try:
username = os.environ['USER']
except:
print "Set USER environment variable and re-run"
sys.exit(0)
try:
opts, args = getopt.getopt(sys.argv[1:], "CdD:f:lo:qstvVw:?X")
except getopt.error, msg:
print __doc__
return 1
names = []
log_flg = False
db_name = 'pcms_data.db'
for opt, arg in opts:
if opt == '-d':
debug_lvl += 1
elif opt == '-D':
debug_lvl = int(arg)
elif opt == '-C':
create(db_name)
return 0
elif opt == '-f':
fname = arg
elif opt == '-l':
log_flg = True
elif opt == '-n':
db_name = arg
elif opt == '-o':
orig = arg
elif opt == '-q':
quiet_flg = True
elif opt == '-w':
names.append(arg)
elif opt == '-s':
schema(db_name)
return 0
elif opt == '-t':
names.append('TST')
elif opt == '-v':
verbose_flg = True
elif opt == '-V':
print "Version: %s" % __version__
return 0
elif opt == '-X':
drop(db_name)
return 0
elif opt == '-?':
print __doc__
return 0
if log_flg:
try:
f_log = open('loader.log', 'a+')
except IOError, msg:
sys.stderr.write(f + ': cannot open: ' + `msg` + '\n')
sys.exit(1)
db_import(names)
return 0
#-----------------------------------------------------------------------
if __name__ == '__main__' or __name__ == sys.argv[0]:
sys.exit(main())
#-----------------------------------------------------------------------
"""
Revision History:
Date Who Description
-------- --- --------------------------------------------------
20110612 plh Initial implementation
Problems to fix:
To Do:
Issues:
Sqlite Help:
.backup ?DB? FILE Backup DB (default "main") to FILE
.bail ON|OFF Stop after hitting an error. Default OFF
.databases List names and files of attached databases
.dump ?TABLE? ... Dump the database in an SQL text format
If TABLE specified, only dump tables matching
LIKE pattern TABLE.
.echo ON|OFF Turn command echo on or off
.exit Exit this program
.explain ?ON|OFF? Turn output mode suitable for EXPLAIN on or off.
With no args, it turns EXPLAIN on.
.header(s) ON|OFF Turn display of headers on or off
.help Show this message
.import FILE TABLE Import data from FILE into TABLE
.indices ?TABLE? Show names of all indices
If TABLE specified, only show indices for tables
matching LIKE pattern TABLE.
.load FILE ?ENTRY? Load an extension library
.log FILE|off Turn logging on or off. FILE can be stderr/stdout
.mode MODE ?TABLE? Set output mode where MODE is one of:
csv Comma-separated values
column Left-aligned columns. (See .width)
html HTML <table> code
insert SQL insert statements for TABLE
line One value per line
list Values delimited by .separator string
tabs Tab-separated values
tcl TCL list elements
.nullvalue STRING Print STRING in place of NULL values
.output FILENAME Send output to FILENAME
.output stdout Send output to the screen
.prompt MAIN CONTINUE Replace the standard prompts
.quit Exit this program
.read FILENAME Execute SQL in FILENAME
.restore ?DB? FILE Restore content of DB (default "main") from FILE
.schema ?TABLE? Show the CREATE statements
If TABLE specified, only show tables matching
LIKE pattern TABLE.
.separator STRING Change separator used by output mode and .import
.show Show the current values for various settings
.stats ON|OFF Turn stats on or off
.tables ?TABLE? List names of tables
If TABLE specified, only list tables matching
LIKE pattern TABLE.
.timeout MS Try opening locked tables for MS milliseconds
.width NUM1 NUM2 ... Set column widths for "column" mode
.timer ON|OFF Turn the CPU timer measurement on or off
"""