diff --git a/mozilla/webtools/new-graph/bulk.cgi b/mozilla/webtools/new-graph/bulk.cgi new file mode 100755 index 00000000000..2ecc89fa4a6 --- /dev/null +++ b/mozilla/webtools/new-graph/bulk.cgi @@ -0,0 +1,227 @@ +#!/usr/bin/env python + +import cgitb; cgitb.enable() + +import sys +import cgi +import time +import re + +from pysqlite2 import dbapi2 as sqlite + +#if var is a valid number returns a value other than None +def checkNumber(var): + if var is None: + return 1 + reNumber = re.compile('^[0-9.]*$') + return reNumber.match(var) + +#if var is a valid string returns a value other than None +def checkString(var): + if var is None: + return 1 + reString = re.compile('^[0-9A-Za-z._()\- ]*$') + return reString.match(var) + +print "Content-type: text/plain\n\n" +link_format = "RETURN:%s:%.2f:%sspst=range&spstart=%d&spend=%d&bpst=cursor&bpstart=%d&bpend=%d&m1tid=%d&m1bl=0&m1avg=0\n" +link_str = "" + +DBPATH = "db/data.sqlite" + +form = cgi.FieldStorage() + +# incoming query string has the following parameters: +# type=discrete|continuous +# indicates discrete vs. continuous dataset, defaults to continuous +# value=n +# (REQUIRED) value to be recorded as the actual test value +# tbox=foopy +# (REQUIRED) name of the tinderbox reporting the value (or rather, the name that is to be given this set of data) +# testname=test +# (REQUIRED) the name of this test +# data=rawdata +# raw data for this test +# time=seconds +# time since the epoch in GMT of this test result; if ommitted, current time at time of script run is used +# date +# date that the test was run - this is for discrete graphs +# branch=1.8.1,1.8.0 or 1.9.0 +# name of the branch that the build was generated for +# branchid=id +# date of the build +# http://wiki.mozilla.org/MozillaQualityAssurance:Build_Ids + +#takes as input a file for parsing in csv with the format: +# value,testname,tbox,time,data,branch,branchid,type,data + +# Create the DB schema if it doesn't already exist +# XXX can pull out dataset_info.machine and dataset_info.{test,test_type} into two separate tables, +# if we need to. +db = sqlite.connect(DBPATH) +try: + db.execute("CREATE TABLE dataset_info (id INTEGER PRIMARY KEY AUTOINCREMENT, type STRING, machine STRING, test STRING, test_type STRING, extra_data STRING, branch STRING, date INTEGER);") + db.execute("CREATE TABLE dataset_values (dataset_id INTEGER, time INTEGER, value FLOAT);") + db.execute("CREATE TABLE dataset_branchinfo (dataset_id INTEGER, time INTEGER, branchid STRING);") + db.execute("CREATE TABLE dataset_extra_data (dataset_id INTEGER, time INTEGER, data BLOB);"); + db.execute("CREATE TABLE annotations (dataset_id INTEGER, time INTEGER, value STRING);") + db.execute("CREATE INDEX datasets_id_idx ON dataset_values(dataset_id);") + db.execute("CREATE INDEX datasets_time_idx ON dataset_values(time);") + db.execute("CREATE INDEX datasets_time_id_idx ON dataset_values(dataset_id, time);") + db.execute("CREATE INDEX datasets_info_idx on dataset_info(type, machine, test, test_type, extra_data, branch, date);") + db.commit() +except: + pass + +# value,testname,tbox,time,data,branch,branchid,type,data + +fields = ["value", "testname", "tbox", "timeval", "date", "branch", "branchid", "type", "data"] +strFields = ["type", "data", "tbox", "testname", "branch", "branchid"] +numFields = ["date", "timeval", "value"] +d_ids = [] +all_ids = [] +all_types = [] +if form.has_key("filename"): + val = form["filename"] + if val.file: + print "found a file" + for line in val.file: + line = line.rstrip("\n\r") + contents = line.split(',') + if len(contents) < 7: + print "Incompatable file format" + sys.exit(500) + for field, content in zip(fields, contents): + globals()[field] = content + for strField in strFields: + if not globals().has_key(strField): + continue + if not checkString(globals()[strField]): + print "Invalid string arg: ", strField, " '" + globals()[strField] + "'" + sys.exit(500) + for numField in numFields: + if not globals().has_key(numField): + continue + if not checkNumber(globals()[numField]): + print "Invalid string arg: ", numField, " '" + globals()[numField] + "'" + sys.exit(500) + + #do some checks to ensure that we are enforcing the requirement rules of the script + if (not type): + type = "continuous" + + if (not timeval): + timeval = int(time.time()) + + if (type == "discrete") and (not date): + print "Bad args, need a valid date" + sys.exit(500) + + if (not value) or (not tbox) or (not testname): + print "Bad args" + sys.exit(500) + + + # figure out our dataset id + setid = -1 + + while setid == -1: + cur = db.cursor() + cur.execute("SELECT id FROM dataset_info WHERE type=? AND machine=? AND test=? AND test_type=? AND extra_data=? AND branch=? AND date=? limit 1", + (type, tbox, testname, "perf", "branch="+branch, branch, date)) + res = cur.fetchall() + cur.close() + + if len(res) == 0: + db.execute("INSERT INTO dataset_info (type, machine, test, test_type, extra_data, branch, date) VALUES (?,?,?,?,?,?,?)", + (type, tbox, testname, "perf", "branch="+branch, branch, date)) + else: + setid = res[0][0] + + db.execute("INSERT INTO dataset_values (dataset_id, time, value) VALUES (?,?,?)", (setid, timeval, value)) + db.execute("INSERT INTO dataset_branchinfo (dataset_id, time, branchid) VALUES (?,?,?)", (setid, timeval, branchid)) + if data and data != "": + db.execute("INSERT INTO dataset_extra_data (dataset_id, time, data) VALUES (?,?,?)", (setid, timeval, data)) + + if (type == "discrete"): + if not setid in d_ids: + d_ids.append(setid) + if not setid in all_ids: + all_ids.append(setid) + all_types.append(type) + + for setid, type in zip(all_ids, all_types): + cur = db.cursor() + cur.execute("SELECT MIN(time), MAX(time), test FROM dataset_values, dataset_info WHERE dataset_id = ? and id = dataset_id", (setid,)) + res = cur.fetchall() + cur.close() + tstart = res[0][0] + tend = res[0][1] + testname = res[0][2] + if type == "discrete": + link_str += (link_format % (testname, float(-1), "dgraph.html#name=" + testname + "&", tstart, tend, tstart, tend, setid,)) + else: + tstart = 0 + link_str += (link_format % (testname, float(-1), "graph.html#",tstart, tend, tstart, tend, setid,)) + + #this code auto-adds a set of continuous data for each series of discrete data sets - creating an overview of the data + # generated by a given test (matched by machine, test, test_type, extra_data and branch) + for setid in d_ids: + cur = db.cursor() + #throw out the largest value and take the average of the rest + cur.execute("SELECT AVG(value) FROM dataset_values WHERE dataset_id = ? and value != (SELECT MAX(value) from dataset_values where dataset_id = ?)", (setid, setid,)) + res = cur.fetchall() + cur.close() + avg = res[0][0] + if avg is not None: + cur = db.cursor() + cur.execute("SELECT machine, test, test_type, extra_data, branch, date FROM dataset_info WHERE id = ?", (setid,)) + res = cur.fetchall() + cur.close() + tbox = res[0][0] + testname = res[0][1] + test_type = res[0][2] + extra_data = res[0][3] + branch = str(res[0][4]) + timeval = res[0][5] + date = '' + cur = db.cursor() + cur.execute("SELECT branchid FROM dataset_branchinfo WHERE dataset_id = ?", (setid,)) + res = cur.fetchall() + cur.close() + branchid = res[0][0] + dsetid = -1 + while dsetid == -1 : + cur = db.cursor() + cur.execute("SELECT id from dataset_info where type=? AND machine=? AND test=? AND test_type=? AND extra_data=? AND branch=? AND date=? limit 1", + ("continuous", tbox, testname+"_avg", "perf", "branch="+branch, branch, date)) + res = cur.fetchall() + cur.close() + if len(res) == 0: + db.execute("INSERT INTO dataset_info (type, machine, test, test_type, extra_data, branch, date) VALUES (?,?,?,?,?,?,?)", + ("continuous", tbox, testname+"_avg", "perf", "branch="+branch, branch, date)) + else: + dsetid = res[0][0] + cur = db.cursor() + cur.execute("SELECT * FROM dataset_values WHERE dataset_id=? AND time=? limit 1", (dsetid, timeval)) + res = cur.fetchall() + cur.close() + if len(res) == 0: + db.execute("INSERT INTO dataset_values (dataset_id, time, value) VALUES (?,?,?)", (dsetid, timeval, avg)) + db.execute("INSERT INTO dataset_branchinfo (dataset_id, time, branchid) VALUES (?,?,?)", (dsetid, timeval, branchid)) + else: + db.execute("UPDATE dataset_values SET value=? WHERE dataset_id=? AND time=?", (avg, dsetid, timeval)) + db.execute("UPDATE dataset_branchinfo SET branchid=? WHERE dataset_id=? AND time=?", (branchid, dsetid, timeval)) + cur = db.cursor() + cur.execute("SELECT MIN(time), MAX(time) FROM dataset_values WHERE dataset_id = ?", (dsetid,)) + res = cur.fetchall() + cur.close() + tstart = 0 + tend = res[0][1] + link_str += (link_format % (testname, float(avg), "graph.html#", tstart, tend, tstart, tend, dsetid,)) + + db.commit() +print "Inserted." +print link_str + +sys.exit() diff --git a/mozilla/webtools/new-graph/collect.cgi b/mozilla/webtools/new-graph/collect.cgi index 7ba4bd0b6d8..6334b7235d5 100755 --- a/mozilla/webtools/new-graph/collect.cgi +++ b/mozilla/webtools/new-graph/collect.cgi @@ -24,7 +24,7 @@ def checkString(var): return reString.match(var) print "Content-type: text/plain\n\n" -link_format = "RETURN:%.2f:%s#spst=range&spstart=%d&spend=%d&bpst=cursor&bpstart=%d&bpend=%d&m1tid=%d&m1bl=0&m1avg=0\n" +link_format = "RETURN:%.2f:%sspst=range&spstart=%d&spend=%d&bpst=cursor&bpstart=%d&bpend=%d&m1tid=%d&m1bl=0&m1avg=0\n" link_str = "" DBPATH = "db/data.sqlite" @@ -128,10 +128,10 @@ cur.close() tstart = res[0][0] tend = res[0][1] if type == "discrete": - link_str += (link_format % (float(-1), "dgraph.html",tstart, tend, tstart, tend, setid,)) + link_str += (link_format % (float(-1), "dgraph.html#name=" + testname + "&", tstart, tend, tstart, tend, setid,)) else: tstart = 0 - link_str += (link_format % (float(-1), "graph.html",tstart, tend, tstart, tend, setid,)) + link_str += (link_format % (float(-1), "graph.html#",tstart, tend, tstart, tend, setid,)) #this code auto-adds a set of continuous data for each series of discrete data sets - creating an overview of the data @@ -178,7 +178,7 @@ if type == "discrete" : cur.close() tstart = 0 tend = res[0][1] - link_str += (link_format % (float(avg), "graph.html", tstart, tend, tstart, tend, setid,)) + link_str += (link_format % (float(avg), "graph.html#", tstart, tend, tstart, tend, setid,)) db.commit() print "Inserted." diff --git a/mozilla/webtools/new-graph/dgraph.html b/mozilla/webtools/new-graph/dgraph.html index b9b93679f28..dcaea4d4674 100644 --- a/mozilla/webtools/new-graph/dgraph.html +++ b/mozilla/webtools/new-graph/dgraph.html @@ -32,10 +32,21 @@ +