Skip to content

Commit 6d09116

Browse files
WiddleWabbitWiddleWabbit
committed
Added functionality to Run Recorded Queries on a Specified Database
1 parent fc9b700 commit 6d09116

File tree

6 files changed

+550
-360
lines changed

6 files changed

+550
-360
lines changed

‎README

Lines changed: 76 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,76 @@
1-
# --- LIST VARIABLES NOTES ---
2-
3-
# SQLLOG LIST
4-
# name: sqllog
5-
#
6-
# Type: List of Lists
7-
# Indexes: Enum Specified with COL enum class
8-
# [COL.Line][COL.String][COL.Type][COL.QueryNo][COL.User][COL.Save][COL.Query]
9-
# Formats:
10-
# [Integer][String][String][String][String][Integer][String]
11-
# Example:
12-
# [
13-
# [1]["Full SQL Connect Statment with Query Number Etc"]["Connect"]["1"]["deoem"][0]["SQL Connect Statement Only"]
14-
# [2]["Full SQL Query Statment with Query Number Etc"]["Query"]["1"]["deoem"][1]["SQL Query Statement Only"]
15-
# [3]["Full SQL Quit Statment with Query Number Etc"]["Quit"]["1"]["deoem"][0]["SQL Quit Statement Only"]
16-
# ]
17-
18-
# CONFIGURATION LIST
19-
# name: configuration
20-
#
21-
# Type: Dictionary of Lists
22-
# Indexes: Enum Specified with CONFIG enum class
23-
# 'DatabaseUser' : [CONFIG.Database, [CONFIG.IgnoreTables], CONFIG.Filter]
24-
# Formats:
25-
# [String][[String][List of Strings][Integer]]
26-
# Example:
27-
# [
28-
# 'user1' : [database1, [list, of, tables, to, ignore], 1]
29-
# 'user2' : [database2, [list, of, tables, to, ignore], 0]
30-
# 'user3' : [database3, [list, of, tables, to, ignore], 0]
31-
# ]
32-
33-
# RECORDABLE QUERY LIST
34-
# name: recordable
35-
#
36-
# Type: List
37-
# Formats: Strings
38-
#
39-
# Example:
40-
# ['INSERT', 'UPDATE', 'DROP']
1+
2+
--- README ---
3+
4+
5+
This script is in development to try and setup a system allowing certain or all database changes to be recorded
6+
and later executed based on a configuration file.
7+
8+
The specific queries etc are able to be set per database with a global configuration for which queries should be
9+
recorded.
10+
These recorded changes can then be run on another database allowing the developer/s to move updates or ecommerce
11+
orders easily and automatically from a development to a production website and vice verca.
12+
13+
14+
15+
--- TODO ---
16+
17+
- Feature
18+
- Lock Functionality
19+
- Silent Running and Verbose
20+
21+
- sqlrun.py
22+
- If there is a start transaction at the end of the command file with some
23+
commands. Do not process it or its commands as they may be rolled back.
24+
25+
- conf.py
26+
- Create a splitConfig function to replace the multiple repeat code instances.
27+
28+
- Tidyup
29+
- index -> recorded_cmd
30+
- next_index -> next_cmd
31+
- Set full script timer
32+
- clean .value from end of enums
33+
- Add required permission values to readme
34+
- Update Lists in readme
35+
36+
37+
--- LIST VARIABLES NOTES ---
38+
39+
SQLLOG LIST
40+
name: sqllog
41+
42+
Type: List of Lists
43+
Indexes: Enum Specified with COL enum class
44+
[COL.Line][COL.String][COL.Type][COL.QueryNo][COL.User][COL.Save][COL.Query]
45+
Formats:
46+
[Integer][String][String][String][String][Integer][String]
47+
Example:
48+
[
49+
[1]["Full SQL Connect Statment with Query Number Etc"]["Connect"]["1"]["deoem"][0]["SQL Connect Statement Only"]
50+
[2]["Full SQL Query Statment with Query Number Etc"]["Query"]["1"]["deoem"][1]["SQL Query Statement Only"]
51+
[3]["Full SQL Quit Statment with Query Number Etc"]["Quit"]["1"]["deoem"][0]["SQL Quit Statement Only"]
52+
]
53+
54+
CONFIGURATION LIST
55+
name: configuration
56+
57+
Type: Dictionary of Lists
58+
Indexes: Enum Specified with CONFIG enum class
59+
'DatabaseUser' : [CONFIG.Database, [CONFIG.IgnoreTables], CONFIG.Filter]
60+
Formats:
61+
[String][[String][List of Strings][Integer]]
62+
Example:
63+
[
64+
'user1' : [database1, [list, of, tables, to, ignore], 1]
65+
'user2' : [database2, [list, of, tables, to, ignore], 0]
66+
'user3' : [database3, [list, of, tables, to, ignore], 0]
67+
]
68+
69+
RECORDABLE QUERY LIST
70+
name: recordable
71+
72+
Type: List
73+
Formats: Strings
74+
75+
Example:
76+
['INSERT', 'UPDATE', 'DROP']

‎conf.py

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@
2020
class CONFIG(IntEnum):
2121
Database = 0
2222
IgnoreTables = 1
23-
Filter = 2
23+
Export = 2
24+
Filter = 3
25+
Pass = 4
26+
Run = 5
2427

2528
class GLOBAL(IntEnum):
2629
Export = 0
@@ -66,7 +69,6 @@ def getConfig(config_file):
6669
# FETCH RECORDABLE QUERIES
6770
# Define an array for the recordable sql queries, processed and unprocessed
6871
record = []
69-
# recordable = []
7072
count = 0
7173

7274
# Try to fetch the recordable sql queries from the config and input them into the array
@@ -81,7 +83,6 @@ def getConfig(config_file):
8183
# We strip them individually so that SQL commands with a space do not have them removed
8284
for sqlcmd in record:
8385
sqlcmd = sqlcmd.strip()
84-
# #recordable.insert(count, sqlcmd)
8586
configuration["Global"][GLOBAL.Record.value].insert(count, sqlcmd)
8687
count += 1
8788
except:
@@ -122,15 +123,36 @@ def getConfig(config_file):
122123
for x in range(len(configuration[current_user][CONFIG.IgnoreTables.value])):
123124
configuration[current_user][CONFIG.IgnoreTables.value][x] = '`' + configuration[current_user][CONFIG.IgnoreTables.value][x] + '`'
124125

126+
# Set the database export location
127+
try:
128+
configuration[current_user].insert(CONFIG.Export.value, configuration["Global"][GLOBAL.Export.value] + db)
129+
except:
130+
parser.error("Unable to set database export location")
131+
125132
# Try to get the filter value from the user
126133
try:
127134
configuration[current_user].insert(CONFIG.Filter.value, config.get(db, 'filter_output'))
128135
except:
129136
configuration[current_user].insert(CONFIG.Filter.value, '0')
130137

138+
# Try to fetch the users password
139+
try:
140+
configuration[current_user].insert(CONFIG.Pass.value, config.get(db, 'password'))
141+
except:
142+
configuration[current_user].insert(CONFIG.Pass.value, 'None')
143+
144+
# Try to separate out databases to run queries on, if there are multiple, then store
145+
try:
146+
runon = re.sub('[\s+]', '', config.get(db, 'runon')).split(',')
147+
configuration[current_user].insert(CONFIG.Run.value, runon)
148+
except:
149+
print(runon)
150+
parser.error("Unable to read specified runon configuration")
151+
131152
configuration["Unknown User"] = []
132153
configuration["Unknown User"].insert(CONFIG.Database.value, "None")
133154
configuration["Unknown User"].insert(CONFIG.IgnoreTables.value, "None")
134155
configuration["Unknown User"].insert(CONFIG.Filter.value, "0")
156+
configuration["Unknown User"].insert(CONFIG.Pass.value, "None")
135157

136158
return configuration

‎config/example

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
####################################################################################################################
2+
#
13
# The SYSTEM section is REQUIRED.
24
# System houses the configuration for databases and values for what queries to track
35
#
@@ -25,20 +27,27 @@
2527
# databases = site_database, database_2
2628
# record = DELETE, DROP, FLUSH, START, COMMIT
2729
# export = /root/scripts/working/export/
30+
#
31+
####################################################################################################################
32+
2833

2934
[SYSTEM]
35+
3036
databases = database1, database2, database3
3137
record = INSERT, DELETE, UPDATE, CREATE, DROP, FLUSH, HASH, REHASH, ALTER, START, COMMIT, ROLLBACK
3238
export = /root/exports/
3339

40+
41+
####################################################################################################################
42+
#
3443
# Each database listed is REQUIRED to have a section for it listed below
3544
# Each database section should be specifed with its name exactly the same as it was specified
3645
# in the system section of the configuration file
3746
#
3847
# Example:
3948
# [database1]
4049
#
41-
# Each database section should contain the user field to specifiy a user to associate with
50+
# Each database section should contain the user field to specifiy a database user to associate with
4251
# the database.
4352
#
4453
# Example:
@@ -55,6 +64,14 @@ export = /root/exports/
5564
# Example:
5665
# filter_output = 1
5766
#
67+
# Each database has the option to have its recorded queries run (if the monitor script has the argument called)
68+
# The value should be a comma serarated list of databases to run the queries on
69+
# In order to run the queries the database to run the queries on must be specified as well and the user, password permissions
70+
# etc correct for that database
71+
#
72+
# Example:
73+
# runon = database2
74+
#
5875
# The ignore tables field can also be specified to mark the tables that should not be saved for
5976
# a particular database. This should be specified in a comma separated list without quotes
6077
# If there are no tables to be ignored just leave it blank
@@ -65,23 +82,30 @@ export = /root/exports/
6582
# Example Database Configuration:
6683
# [site_database]
6784
# user = website_user
85+
# password = user_password
6886
# ignore_tables = cron_schedule, cron_exec
87+
# filter_output = 1
88+
# runon = database2
6989
#
90+
####################################################################################################################
7091

7192
[database1]
7293
user = user1
7394
password = password1
7495
ignore_tables = cron_schedule
7596
filter_output = 1
97+
runon =
7698

7799
[database2]
78100
user = user2
79101
password = password2
80102
ignore_tables = cron_schedule, sequence_order_0, sequence_order_1, sequence_invoice_0, sequence_invoice_1, sequence_shipment_0, sequence_shipment_1, sequence_creditmemo_0, sequence_creditmemo_1, quote_id_mask, shipping_tablerate, gift_message, quote, quote_item, quote_item_option, quote_payment, quote_shipping_rate, checkout_agreement_store, checkout_agreement, quote_address, quote_address_item, sales_order, downloadable_link_purchased, downloadable_link_purchased_item, sales_order_payment, sales_order_address, sales_payment_transaction, vault_payment_token_order_payment_link, vault_payment_token, sales_order_status_history, sales_order_grid, sales_order_item, sales_order_tax_item, sales_order_tax, sales_invoice, sales_invoice_item, sales_invoice_comment, sales_invoice_grid, sales_shipment, sales_shipment_item, sales_shipment_track, sales_shipment_comment, sales_shipment_grid, sales_creditmemo, sales_creditmemo_comment, sales_creditmemo_grid, sales_creditmemo_item, sales_invoiced_aggregated, sales_invoiced_aggregated_order, sales_bestsellers_aggregated_monthly, sales_bestsellers_aggregated_yearly, sales_bestsellers_aggregated_daily, sales_shipping_aggregated, sales_shipping_aggregated_order, sales_order_aggregated_created, sales_order_aggregated_updated, sales_order_status_state, sales_order_status, sales_order_status_label, sales_sequence_meta, sales_sequence_profile, sales_refunded_agreggregated, sales_refunded_aggregated_order
81103
filter_output = 0
104+
runon = database1
82105

83106
[database3]
84107
user = user3
85108
password = password3
86109
ignore_tables =
87110
filter_output = 1
111+
runon =

‎runsql.py

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#!/usr/bin/python
2+
3+
import os.path
4+
import os
5+
import re
6+
import MySQLdb
7+
from enum import Enum
8+
from enum import IntEnum
9+
10+
import conf
11+
12+
#configuration = conf.getConfig("oem")
13+
14+
# ##############################
15+
# --- SCRIPT REQUIREMENTS ---
16+
# ##############################
17+
#
18+
# Requires Python Version 2.6.6 or above.
19+
# Requires Python 3.4 Enum Backported
20+
# - enum34 1.1.6
21+
# Requires MYSQL-python
22+
# - MySQLdb 1.2.5
23+
24+
def runSQL(user, configuration):
25+
26+
# If it is an actual user and there is a database set for it to run its queries on
27+
if (user != "Unknown User" and user != "Global") and (configuration[user][conf.CONFIG.Run][0] != ''):
28+
29+
# For each database it has set to run queries on
30+
for database in configuration[user][conf.CONFIG.Run]:
31+
32+
# Find the user of the database to write to
33+
for entry in configuration:
34+
try:
35+
if configuration[entry][conf.CONFIG.Database.value] == database:
36+
dbuser = entry
37+
break
38+
else:
39+
continue
40+
except:
41+
continue
42+
43+
# Fetch the password for the database user
44+
dbpass = configuration[dbuser][conf.CONFIG.Pass.value]
45+
46+
# Connect up to the database
47+
db = MySQLdb.connect('localhost', dbuser, dbpass, database)
48+
cursor = db.cursor()
49+
50+
# Setup error and filename variables
51+
err = 0
52+
tmp_file = configuration[user][conf.CONFIG.Export.value] + "_tmp"
53+
54+
# Open the file containing sql commands ##########################NEEDS TO BE ADDED TO CONFIGURATION
55+
with open(configuration[user][conf.CONFIG.Export.value]) as file:
56+
57+
# Open our tmp file to store lines in case of an error
58+
tmp = open(tmp_file, 'w')
59+
60+
for line in file:
61+
62+
if err == 0:
63+
64+
# Try executing the command
65+
try:
66+
cursor.execute(line)
67+
68+
# Should an error occur we stop processing commands by setting err to 1 and write the line to our tmp file
69+
# To stop on a warning as well we should add ,MySQLdb.Warning inside brackets
70+
except (MySQLdb.Error) as error:
71+
print(error)
72+
tmp.write(line)
73+
err = 1
74+
except:
75+
print("Unknown Error")
76+
tmp.write(line)
77+
err = 1
78+
79+
# We have hit some form of error, stop processing and just write all the lines to tmp
80+
# if we kept processing we might end up with commands run out of order, wrong primary keys, other errors etc
81+
else:
82+
83+
tmp.write(line)
84+
85+
# Close the database connection
86+
db.close()
87+
# Close the tmp file
88+
tmp.close()
89+
90+
# If we hit an error store a backup of the sqlfile we ran commands from in the same directory
91+
# and rename the tmp_file which contains the errored command and all following, to be the original file
92+
if err == 1:
93+
94+
os.rename(configuration[user][conf.CONFIG.Export.value], configuration[user][conf.CONFIG.Export.value] + "_backup")
95+
os.rename(tmp_file, configuration[user][conf.CONFIG.Export.value])
96+
97+
# Otherwise if there were no errors just delete both files
98+
else:
99+
100+
os.remove(configuration[user][conf.CONFIG.Export.value])
101+
os.remove(tmp_file)
102+
103+
#runSQL('/root/scripts/cron/sqlexports/oem_magento', configuration)

0 commit comments

Comments
 (0)