#! /usr/bin/env python import os, commands, sys, getopt, math def main(): try: opts, args = getopt.getopt(sys.argv[1:], "e:n:d:s:f:j:v:q:pth", ["events=", "name=", "dir=", "subjobs=", "files=", "jo=", "version=", "queue=", "prestage", "test", "help"]) except getopt.GetoptError: print "Option parsing error. Try '" + sys.argv[0] + " -h' for usage." sys.exit(2) name = "Job" events = "-1" sampleDir = name subjobs = 1 nfiles = -1 jo = '/user2/jkretz/testarea/14.4.0/WorkArea/wzana/LivZBosonExample_topOptions.py' # jo = '/user2/jkretz/testarea/14.4.0/WorkArea/wzana/LivZBosonGentest_topOptions.py' version = '14.4.0' queue = 'medium' test = False prestageFiles = False for opt, val in opts: if opt in ("-n", "--name"): name = val if opt in ("-e", "--events"): events = val if opt in ("-d", "--dir"): sampleDir = val if opt in ("-s", "--subjobs"): subjobs = int(val) if opt in ("-f", "--files"): nfiles = int(val) if opt in ("-j", "--jo"): jo = val if opt in ("-v", "--version"): version = val if opt in ("-q", "--queue"): queue = val if opt in ("-p", "--prestage"): prestageFiles = True if opt in ("-t", "--test"): test = True if opt in ("-h", "--help"): pass pass print "*"*40 print "Creating %s subjob(s) for %s" % (subjobs,name) print "*"*40 files = listDirectory(sampleDir, nfiles); filesPerSub = int(math.ceil(len(files)/float(subjobs))) print 'filesPerSub ', filesPerSub if os.path.exists(name): commands.getstatusoutput('rm -rf %s' % name) os.mkdir(name) os.mkdir('%s/output' % name) for isub in xrange(subjobs): subFiles = files[isub*filesPerSub:(isub+1)*filesPerSub] fileListName = makeFileList(name, isub, subFiles, prestageFiles) joCopyName = makeJobOpts(name, isub, jo, fileListName) athenaCmd = makeAthenaCommand(name, isub, events, joCopyName) batchName = makeBatchScript(name, isub, athenaCmd, version, queue) if test: continue submit(name, batchName) pass return def listDirectory(sampleDir, nfiles = -1): fullFiles = [] # first resolve wild cards - only in the last part of the path name! if sampleDir.find('*') >= 0: if sampleDir.split('/')[1] == 'dpm': base,tempname = sampleDir.rsplit('/',1) cmd = 'dpns-ls %s | grep %s ' % (base, tempname.replace('*','.*')) print cmd rc, tmpdirlist = commands.getstatusoutput('%s' % cmd) newsampleDir = '' for ddd in tmpdirlist.split('\n'): newsampleDir += '%s/%s,' % (base, ddd) sampleDir = newsampleDir.rstrip(',') else: cmd = 'ls -d %s' % (sampleDir) print cmd rc, tmpdirlist = commands.getstatusoutput('%s' % cmd) newsampleDir = '' for ddd in tmpdirlist.split('\n'): newsampleDir += '%s,' % ddd sampleDir = newsampleDir.rstrip(',') for directory in sampleDir.split(','): cmd = 'ls' protocol = '' if sampleDir.split('/')[1] == 'dpm': cmd = 'dpns-ls -l ' protocol = 'rfio:' cmd += ' %s' % directory if protocol == 'rfio:': cmd += ' | egrep -v " 0 Mar| 0 Feb| 0 Jan| 0 Dec|DQ2" | awk -F \' \' \'{print $NF}\'' print cmd rc, files = commands.getstatusoutput('%s' % cmd) fullFiles += ['%s%s/%s' % (protocol, directory.rstrip('/'), f) for f in files.split('\n')] if rc: print "Unable to list directory. Exiting" sys.exit(1) pass pass if nfiles != -1: return fullFiles[:nfiles] return fullFiles def makeFileList(name, isub, files, prestageFiles): filename = '%s/%s/%sFiles.%s.py' % (os.getcwd(), name, name, isub) fileList = open(filename, 'w') fileList.write('myInputCollections = [\n') [fileList.write(' "%s",\n' %f) for f in files] fileList.write(']\n') fileList.write('UseFileStager = %s\n' % prestageFiles) return filename def makeJobOpts(name, isub, joName, listName): copyName = '%s/%s' % (name, joName.split('/')[-1].replace('.py', '.%s.%s.py' % (name, isub))) copyName = '%s/%s' % (os.getcwd(), copyName) # write file list inclusion header fileJo = open(copyName, 'w') fileJo.write("#======= Added by Batch script ======\n") fileJo.write('include ("%s")\n' % listName) fileJo.write("#====================================\n\n") fileJo.close() # append the "real" jo rc, out = commands.getstatusoutput('cat %s >> %s' % (joName, copyName)) if rc: print "Unable to append %s to %s. Exiting" % (joName, copyName) sys.exit(2) pass return copyName def makeAthenaCommand(name, isub, events, joName): # the ATHENA options need to be used inside the job option files! outname = 'Ntuple.%s.%s.root' % (name, isub) cmd = 'athena.py -c \'MyOutput="%s";MyEvents=%s\' %s' % (outname, events, joName) print cmd return cmd def makeBatchScript(name, isub, cmd, version, queue): filename = '%s/%s.Submit.%s.sh' % (name, name, isub) fileBatch = open(filename, 'w') fileBatch.write( \ """ #! /bin/bash -f #PBS -q %s #PBS -N %s echo \"---- Starting PBS job at `date` -----\" export DBRELEASE_OVERRIDE=6.5.2 source ~/cmthome/setup.sh -tag=%s,32 source /batchsoft/glite/etc/profile.d/grid-env.sh mkdir /tmp/%s_%s_${PBS_JOBID} cd /tmp/%s_%s_${PBS_JOBID} cp /user2/jkretz/testarea/14.4.0/WorkArea/wzana/LivZBoson_preselection.py . cp /user2/jkretz/testarea/14.4.0/WorkArea/wzana/LivZBosonGentest_preselection.py . cp /user2/jkretz/testarea/14.4.0/WorkArea/wzana/LivFileStagerConfig.py . time %s >& %s.%s.log gzip *.log scp *.root *.txt *.log.gz alpha:%s/%s/output/. cd - rm -rf /tmp/%s_%s_${PBS_JOBID} echo \"---- Finished PBS job at `date` -----\" """ % (queue,\ name, \ version, \ name, isub, \ name, isub, \ cmd, name, isub, \ os.getcwd(), name, \ name, isub) \ ) return filename def submit(name, filename): cmd = 'qsub -o %s -e %s %s' % (name, name, filename) print cmd rc, out = commands.getstatusoutput(cmd) print out if rc: print "Unable to subit job" sys.exit(3) pass return if __name__ == '__main__': main()