I wrote this script to scan the tinderbox logs, looking for an intermittent problem. Chandler was set up to log "critical" messages to chandler.log with information about the problem. (You't need shell access on paniolo to run this there, naturally.)

#!/usr/bin/env python

import os, sys, gzip, pdb

# last checkin related to this problem
cutoff = 1161368400

# no problems seen from last checkin until this point,
# so can from here from now on.
cutoff = 1161729267

def scan(path):
  f = gzip.open(path)
  found = False
  # pdb.set_trace()
  msg = 'TestReminderProcessing CRITICAL: '
  for s in f:
    msgIndex = s.find(msg)
    if msgIndex == -1:
      continue

    msgText = s[msgIndex + len(msg):].rstrip()
    if msgText.startswith('Checking') or \
       msgText.startswith('Sleeping') or \
       msgText.startswith('Waiting for') or \
       msgText.startswith('Reminders done') or \
       msgText.startswith('Done waiting') or \
       msgText.startswith('testTime=') or \
       msgText.startswith('Dismissing'):
      continue

    if not found:
      found = True
      print "%s:\n  %s" % (path, msgText)
    else:
      print "  %s" % msgText
  f.close()

maxDateSeen = 0

for root, dirs, files in os.walk('/home/tinderbox/docs'):
  for f in files:
    if not f.endswith('.gz'):
      continue

    fDate = int(f[:f.index('.')])
    #pdb.set_trace()
    if fDate <= cutoff:
      continue

    fPath = os.path.join(root, f)
    scan(fPath)

    if maxDateSeen < fDate:
        maxDateSeen = fDate

print "Done; max date seen is %d" % maxDateSeen

-- BryanStearns - 25 Oct 2006