LUXSimUsingHTCondor: Difference between revisions

Jump to navigation Jump to search

LUXSimUsingHTCondor (view source)

Revision as of 19:20, 21 April 2017

109 bytes added ,  21 April 2017
→‎In order to use the computing resources at Wisconsin, one can submit HTCondor jobs to run simulation
 
(22 intermediate revisions by 2 users not shown)
== Submit HTCondor Jobs ==
 
*==== In order to use the computing resources at Wisconsin, one can submit HTCondor jobs to run simulation. ====
 
<nowiki>
ssh lzlogin01.hep.wisc.edu
ssh <username>@lzlogin01.hep.wisc.edu
 
# Make a valid voms-proxy
voms-proxy-init --voms=lz --valid=142:00
 
# Run farmoutrunWiscJobs script to submit jobs
# To get help about options
python /cvmfs/lz.opensciencegrid.org/sw/bin/farmoutGeneric.py \
# python /cvmfs/lz.opensciencegrid.org/sw/bin/runWiscJobs.py -h
--WorkFlow="test” \
--nJobs=2 \
--Executable=“/data/tapas/lzsim/submit/RunLUXSim.sh” \
--TransferInputFile="/data/tapas/lzsim/submit/test.mac” \
--UserEnv="MyOutputName=test_ “ \
-o "test”
--Arguments="test.mac”
 
python /cvmfs/lz.opensciencegrid.org/sw/bin/runWiscJobs.py \
* Above input macro and Executable script can be used an example
--WorkFlow="test.mac_test1" \
* RunLUXSim.sh uses HEAD version of LUXSim
--Executable="/afs/hep.wisc.edu/user/cvuosalo/work/lz/RunLUXSim.sh" \
--Experiment=lz \
--nJobs=10 \
--TransferInputFile="/afs/hep.wisc.edu/user/cvuosalo/work/lz/test.mac" \
--HDFSProdDir=outfiles \
--UserEnv="UserName=`whoami` " \
--OutputDir="test.mac" \
--Arguments="test.mac"
 
# If you want your output copied to HDFS, have your executable copy the output
* Example RunLUXSim.sh looks like this :
# files to the directory you specify with --HDFSProdDir.
 
</nowiki>
#!/bin/bash
wdir=$(pwd)
ver=HEAD
export ROOTSYS=/cvmfs/lz.opensciencegrid.org/ROOT/v5.34.32/slc6_gcc44_x86_64/root
export PATH=${ROOTSYS}/bin:/sbin:/usr/sbin:/usr/bin:/bin:/usr/local/bin
export LD_LIBRARY_PATH=${ROOTSYS}/lib:/lib64:/usr/lib64:/lib:/usr/lib
source /cvmfs/lz.opensciencegrid.org/geant4/etc/geant4env.sh
tmpDir=$(mktemp -d --tmpdir="./")
echo "Linking CVMFS LUXSim area to temp dir. $tmpDir"
cd $tmpDir
ln -s /cvmfs/lz.opensciencegrid.org/LUXSim/$ver/* .
echo "MyOutputName=$MyOutputDir/$MyOutputName$MyRandomNumber"
cp ../*.mac .
./LUXSimExecutable $@
uname=$(whoami)
for i in *.bin
do
if [ -s "${i}" ]; then
# Copy .bin files to storage /hdfs/lz/user/
echo Trying... lcg-cp "${i}" -bvvD srmv2 srm://cmssrm2.hep.wisc.edu:8443/srm/v2/server?SFN=/hdfs/lz/user/${uname}/${MyOutputDir}/"${i}"
lcg-cp "${i}" -bvvD srmv2 srm://cmssrm2.hep.wisc.edu:8443/srm/v2/server?SFN=/hdfs/lz/user/${uname}/${MyOutputDir}/"${i}"
# Convert .bin file to .root file
/cvmfs/lz.opensciencegrid.org/LUXSim/$ver/tools/LUXRootReader "${i}"
else
echo "Something went wrong... No bin files made. Look at the log"
fi
done
# Copy root files to storage
for n in *.root
do
if [ -s "${n}" ]; then
echo Trying... lcg-cp "${n}" -bvvD srmv2 srm://cmssrm2.hep.wisc.edu:8443/srm/v2/server?SFN=/hdfs/lz/user/${uname}/${MyOutputDir}/"${n}"
lcg-cp "${n}" -bvvD srmv2 srm://cmssrm2.hep.wisc.edu:8443/srm/v2/server?SFN=/hdfs/lz/user/${uname}/${MyOutputDir}/"${n}"
else
echo "Something went wrong... No Root files made. Look at the log"
fi
done
 
*==== Following is anExecutable example: testRunLUXSim.macsh : ====
 
<nowiki>
#!/bin/bash
 
wdir=$(pwd)
ver=release-4.4.2
 
# Source the environment needed to run LUXSim
export ROOTSYS=/cvmfs/lz.opensciencegrid.org/ROOT/v5.34.32/slc6_gcc44_x86_64/root
export PATH=${ROOTSYS}/bin:/sbin:/usr/sbin:/usr/bin:/bin:/usr/local/bin
export LD_LIBRARY_PATH=${ROOTSYS}/lib:/lib64:/usr/lib64:/lib:/usr/lib
source /cvmfs/lz.opensciencegrid.org/geant4/etc/geant4env.sh geant4.9.5.p02
 
# Temporary directory that gets created on the execute node
tmpDir=$(mktemp -d --tmpdir="./")
echo "Linking CVMFS LUXSim area to temp dir. $tmpDir"
cd $tmpDir
# ln -s /cvmfs/lz.opensciencegrid.org/LUXSim/release-4.3.2/* .
ln -s /cvmfs/lz.opensciencegrid.org/LUXSim/$ver/* .
 
# echo "MyOutputName=$MyOutputDir/$MyOutputName$MyRandomNumber"
# export MyOutputName=$MyOutputDir/$MyOutputName$MyRandomNumber
cp ../*.mac .
 
# Run the LUXSim
./LUXSimExecutable $@
 
mkdir -p ../outfiles
 
for i in *.bin
do
if [ -s "${i}" ]; then
# Convert .bin file to .root file
echo /cvmfs/lz.opensciencegrid.org/LUXSim/$ver/tools/LUXRootReader "${i}"
/cvmfs/lz.opensciencegrid.org/LUXSim/$ver/tools/LUXRootReader "${i}"
(cd ../outfiles ; ln -s ../$tmpDir/${i} .)
 
else
echo "Something went wrong... No bin files made. Look at the log"
exit 1
fi
done
cd ../outfiles
ln -s ../$tmpDir/*.root .
 
 
</nowiki>
 
==== Example test.mac ====
 
<nowiki>
/run/verbose 0
/control/verbose 0
/tracking/verbose 0
/grdm/verbose 0
 
/run/initialize
 
/LUXSim/io/alwaysRecordPrimary true
# Output folder
/LUXSim/io/updateFrequency 100
/LUXSim/io/outputDir .
/LUXSim/io/updateFrequencyoutputName 1EnergyDeposition_
/LUXSim/physicsList/useOpticalProcesses true
 
 
/LUXSim/detector/select LZDetector
/LUXSim/detector/gridWires onoff
/LUXSim/detector/topGrid on
 
/LUXSim/detector/printEFields
 
/LUXSim/detector/update
 
/LUXSim/detector/recordLevel LiquidXenonTarget 3
/LUXSim/materials/LXeTeflonRefl 0.95
/LUXSim/detector/recordLevelOptPhot LiquidXenonTarget 3
/LUXSim/materials/GXeTeflonRefl 0.75
/LUXSim/detector/recordLevelThermElec LiquidXenonTarget 3
/LUXSim/materials/LXeAbsorption 100 m
/LUXSim/materials/GXeAbsorption 500 m
/LUXSim/materials/LXeSteelRefl 0.2
/LUXSim/materials/GXeSteelRefl 0.2
/LUXSim/materials/AlUnoxidizedQuartzRefl 0.9
/LUXSim/materials/GXeRayleigh 500 m
/LUXSim/materials/LXeRayleigh 0.3 m
 
/gps/particle e-
/gps/energy 530 keV
/gps/position 0 0 75 cm
#this is approximate center. See ElectronCollection for whole volume
/gps/ang/type iso
 
/LUXSim/physicsList/useOpticalProcesses true
# Output .bin files
 
/LUXSim/beamOn 5
exit
 
</nowiki>
 
==== Files in the storage area ====
/control/getEnv MyRandomNumber
* Some events are available in the Wisconsin LZ storage area and can be accessed using the following command :
/control/getEnv MyOutputName
/LUXSim/io/outputName {MyOutputName}
/LUXSim/randomSeed {MyRandomNumber}
/LUXSim/physicsList/driftElecAttenuation 1.5 m
/LUXSim/beamOn 10
 
xrdfs g22n02.hep.wisc.edu:31094 ls /lz/user/tapas/LZSim/pmt-Co_gdecay_1_pro_sci_ioni_drift/
exit</nowiki>
xrdfs g22n02.hep.wisc.edu:31094 ls /lz/user/tapas/LZSim/pmt-Co_gdecay_1_pro_sci_ioni_drift_gammma100KeV
* copy the files using xrdcp
xrdcp -d 1 -v \
root://cmsxrootd.hep.wisc.edu//lz/user/tapas/LZSim/pmt-Co_gdecay_1_pro_sci_ioni_drift_gammma100KeV/filename.root .
Retrieved from "https://wiki.physics.wisc.edu/lz/index.php/Special:MobileDiff/70...133"

Navigation menu