Revamp the script to handle the 'TEST=simple' output.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@142559 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 67 insertions, 74 deletions

diff --git a/utils/release/findRegressions.py b/utils/release/findRegressions.py
index e801dab4ab..7629c8b4fb 100755
--- a/utils/release/findRegressions.py
+++ b/utils/release/findRegressions.py
@@ -1,68 +1,60 @@
 #!/usr/bin/python
-import re, string, sys, os, time
+import re, string, sys, os, time, math
 
 DEBUG = 0
-testDirName = 'llvm-test'
-test      = ['compile', 'llc', 'jit', 'cbe']
-exectime     = ['llc-time', 'jit-time', 'cbe-time',]
-comptime     = ['llc', 'jit-comptime', 'compile']
 
-(tp, exp) = ('compileTime_', 'executeTime_')
+(tp, exp) = ('compile', 'exec')
 
 def parse(file):
-  f=open(file, 'r')
+  f = open(file, 'r')
   d = f.read()
   
-  #Cleanup weird stuff
-  d = re.sub(r',\d+:\d','', d)
-   
+  # Cleanup weird stuff
+  d = re.sub(r',\d+:\d', '', d)
+
   r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)
-   
+
   test = {}
   fname = ''
   for t in r:
     if DEBUG:
       print t
+
     if t[0] == 'PASS' or t[0] == 'FAIL' :
-      tmp = t[2].split(testDirName)
+      tmp = t[2].split('llvm-test/')
       
       if DEBUG:
         print tmp
-      
+
       if len(tmp) == 2:
         fname = tmp[1].strip('\r\n')
       else:
         fname = tmp[0].strip('\r\n')
-      
-      if not test.has_key(fname) :
+
+      if not test.has_key(fname):
         test[fname] = {}
-      
-      for k in test:
-        test[fname][k] = 'NA'
-        test[fname][t[1]] = t[0]
-        if DEBUG:
-          print test[fname][t[1]]
+
+      test[fname][t[1] + ' state'] = t[0]
+      test[fname][t[1] + ' time'] = float('nan')
     else :
       try:
         n = t[0].split('RESULT-')[1]
-        
+
         if DEBUG:
-          print n;
+          print "n == ", n;
         
-        if n == 'llc' or n == 'jit-comptime' or n == 'compile':
-          test[fname][tp + n] = float(t[2].split(' ')[2])
+        if n == 'compile-success':
+          test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n'))
+
+        elif n == 'exec-success':
+          test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n'))
           if DEBUG:
-            print test[fname][tp + n]
-        
-        elif n.endswith('-time') :
-            test[fname][exp + n] = float(t[2].strip('\r\n'))
-            if DEBUG:
-              print test[fname][exp + n]
-        
+            print test[fname][string.replace(n, '-success', '')]
+
         else :
-          print "ERROR!"
+          # print "ERROR!"
           sys.exit(1)
-      
+
       except:
           continue
 
@@ -72,59 +64,60 @@ def parse(file):
 def diffResults(d_old, d_new):
 
   for t in sorted(d_old.keys()) :
-    if DEBUG:
-      print t
-        
-    if d_new.has_key(t) :
-    
+    if d_new.has_key(t):
+
       # Check if the test passed or failed.
-      for x in test:
+      for x in ['compile state', 'compile time', 'exec state', 'exec time']:
+
+        if not d_old[t].has_key(x) and not d_new[t].has_key(x):
+          continue
+
         if d_old[t].has_key(x):
           if d_new[t].has_key(x):
+
             if d_old[t][x] == 'PASS':
               if d_new[t][x] != 'PASS':
-                print t + " *** REGRESSION (" + x + ")\n"
+                print t + " *** REGRESSION (" + x + " now fails)"
             else:
               if d_new[t][x] == 'PASS':
-                print t + " * NEW PASS (" + x + ")\n"
-                
-          else :
-            print t + "*** REGRESSION (" + x + ")\n"
-        
-        # For execution time, if there is no result, its a fail.
-        for x in exectime:
-          if d_old[t].has_key(tp + x):
-            if not d_new[t].has_key(tp + x):
-              print t + " *** REGRESSION (" + tp + x + ")\n"
-                
-          else :
-            if d_new[t].has_key(tp + x):
-              print t + " * NEW PASS (" + tp + x + ")\n"
-
-       
-        for x in comptime:
-          if d_old[t].has_key(exp + x):
-            if not d_new[t].has_key(exp + x):
-              print t + " *** REGRESSION (" + exp + x + ")\n"
-                
+                print t + " * NEW PASS (" + x + " now fails)"
+
           else :
-            if d_new[t].has_key(exp + x):
-              print t + " * NEW PASS (" + exp + x + ")\n"
-              
+            print t + "*** REGRESSION (" + x + " now fails)"
+
+        if x == 'compile state' or x == 'exec state':
+          continue
+
+        # For execution time, if there is no result it's a fail.
+        if not d_old[t].has_key(x) and not d_new[t].has_key(x):
+          continue
+        elif not d_new[t].has_key(x):
+          print t + " *** REGRESSION (" + x + ")"
+        elif not d_old[t].has_key(x):
+          print t + " * NEW PASS (" + x + ")"
+
+        if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
+          continue
+
+        elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]):
+          print t + " * NEW PASS (" + x + ")"
+
+        elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
+          print t + " *** REGRESSION (" + x + ")"
+
+        if d_new[t][x] > d_old[t][x] and \
+              (d_new[t][x] - d_old[t][x]) / d_new[t][x] > .05:
+          print t + " *** REGRESSION (" + x + ")"
+
     else :
-      print t + ": Removed from test-suite.\n"
-    
+      print t + ": Removed from test-suite."
 
-#Main
+# Main
 if len(sys.argv) < 3 :
-    print 'Usage:', sys.argv[0], \
-          '<old log> <new log>'
-    sys.exit(-1)
+  print 'Usage:', sys.argv[0], '<old log> <new log>'
+  sys.exit(-1)
 
 d_old = parse(sys.argv[1])
 d_new = parse(sys.argv[2])
 
-
 diffResults(d_old, d_new)
-
-