Merge pull request #57 from djw8605/multi_origin

Use redirector to enable multi origins

Author: djw8605

.travis.yml

@@ -3,18 +3,16 @@ env:
   matrix:
   - OS_TYPE=centos OS_VERSION=6
   - OS_TYPE=centos OS_VERSION=7
-  - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://stash.osgconnect.net"
   - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://xrd-cache-1.t2.ucsd.edu"
   - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://mwt2-stashcache.campuscluster.illinois.edu"
   - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://hcc-stash.unl.edu"
   - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://osgxroot.usatlas.bnl.gov"
-  - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="its-condor-xrootd1.syr.edu"
-  - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://stash.osgconnect.net"
+  - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://its-condor-xrootd1.syr.edu"
   - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://xrd-cache-1.t2.ucsd.edu"
   - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://mwt2-stashcache.campuscluster.illinois.edu"
   - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://hcc-stash.unl.edu"
   - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://osgxroot.usatlas.bnl.gov"
-  - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="its-condor-xrootd1.syr.edu"
+  - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://its-condor-xrootd1.syr.edu"
 
 
 services:

bin/caches.json

@@ -1,9 +1,7 @@
 [ 
-    {"name":"root://student02.ci-connect.net", "status":0, "longitude":-87.603, "latitude":41.78}, 
-    {"name":"root://stash.osgconnect.net", "status":1, "longitude":-87.603, "latitude":41.78},
-    {"name":"root://cm3.bu.edu", "status":0, "longitude":-72.6167, "latitude":42.2042},
     {"name":"root://xrd-cache-1.t2.ucsd.edu", "status":1, "longitude":-117.2359, "latitude":32.8807},
     {"name":"root://mwt2-stashcache.campuscluster.illinois.edu", "status":1, "longitude":-88.23, "latitude":40.107487},
     {"name":"root://hcc-stash.unl.edu", "status":1, "longitude":-96.702991, "latitude":40.812957},
-    {"name":"root://osgxroot.usatlas.bnl.gov", "status":1, "longitude":-72.9333, "latitude":40.8167}
+    {"name":"root://osgxroot.usatlas.bnl.gov", "status":1, "longitude":-72.9333, "latitude":40.8167},
+    {"name":"root://its-condor-xrootd1.syr.edu", "status":1, "longitude":-72.9333, "latitude":40.8167}
 ]

bin/stashcp

@@ -15,26 +15,16 @@ import shutil
 import logging
 from urlparse import urlparse
 
-
+main_redirector = "root://redirector.osgstorage.org"
+stash_origin = "root://stash.osgconnect.net"
 
 TIMEOUT = 300
 DIFF = TIMEOUT * 10
 
 
 def doStashCpSingle(sourceFile, destination, cache, debug=False):
 
-    logging.debug("Checking size of file.")
-    (xrdfs_stdout, xrdfs_stderr) = subprocess.Popen(["xrdfs", "root://stash.osgconnect.net", "stat", sourceFile], stdout=subprocess.PIPE).communicate()
-    xrdcp_version = subprocess.Popen(['echo $(xrdcp -V 2>&1)'], stdout=subprocess.PIPE, shell=True).communicate()[0][:-1]
-    try:
-        fileSize = int(re.findall(r"Size:   \d+", xrdfs_stdout)[0].split(":   ")[1])
-    except Exception as e:
-        sys.stderr.write("Unable to find size of file\n")
-        print str(xrdfs_stdout)
-        sys.stderr.write(str(xrdfs_stderr))
-        sys.stderr.write("\n")
-        return 1
-    logging.debug("Size of the file %s is %s", sourceFile, str(fileSize))
+    
     #cache=get_best_stashcache()
     logging.debug("Using Cache %s", cache)
 
@@ -44,8 +34,7 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False):
     filename = destination + '/' + sourceFile.split('/')[-1]
 
     payload = {}
-    payload['xrdcp_version'] = xrdcp_version
-    payload['filesize'] = fileSize
+    
     payload['filename'] = sourceFile
     payload['sitename'] = sitename
     payload.update(parse_job_ad())
@@ -66,12 +55,14 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False):
             logging.debug("Succesfully copied file from CVMFS!")
             end1 = int(time.time()*1000)
             dlSz=os.stat(destination).st_size
+            filesize = os.stat(cvmfs_file).st_size
             dltime=end1-start1
             destSpace=1
             status = 'Success'
             payload['timestamp']=end1
             payload['host']="CVMFS"
             payload['download_size']=dlSz
+            payload['filesize'] = filesize
             payload['download_time']=dltime
             payload['destination_space']=destSpace
             payload['status']=status
@@ -89,6 +80,23 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False):
     else:
         logging.debug("CVMFS File does not exist")
 
+    # Now check the size of the file with xrootd
+    logging.debug("Checking size of file.")
+    (xrdfs_stdout, xrdfs_stderr) = subprocess.Popen(["xrdfs", main_redirector, "stat", sourceFile], stdout=subprocess.PIPE).communicate()
+    xrdcp_version = subprocess.Popen(['echo $(xrdcp -V 2>&1)'], stdout=subprocess.PIPE, shell=True).communicate()[0][:-1]
+    try:
+        fileSize = int(re.findall(r"Size:   \d+", xrdfs_stdout)[0].split(":   ")[1])
+    except Exception as e:
+        sys.stderr.write("Unable to find size of file\n")
+        print str(xrdfs_stdout)
+        sys.stderr.write(str(xrdfs_stderr))
+        sys.stderr.write("\n")
+        return 1
+    logging.debug("Size of the file %s is %s", sourceFile, str(fileSize))
+    
+    payload['xrdcp_version'] = xrdcp_version
+    payload['filesize'] = fileSize
+    
     end1=int(time.time()*1000)
     payload['end1']=end1
     payload['start1']=start1
@@ -123,8 +131,8 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False):
         es_send(payload)
 
     else: #pull from origin
-        logging.warning("XrdCP from cache failed on %s, pulling from origin", cache)
-        cache="root://stash.osgconnect.net"
+        logging.warning("XrdCP from cache failed on %s, pulling from main redirector", cache)
+        cache=main_redirector
         start3 = int(time.time()*1000)
         xrd_exit=timed_transfer(filename=sourceFile, debug=debug, cache=cache, destination=destination)
         end3=int(time.time()*1000)
@@ -179,9 +187,9 @@ def parse_job_ad():
     return temp_list
 
 def dostashcpdirectory(sourceDir, destination, cache, debug=False):
-    sourceItems = subprocess.Popen(["xrdfs", "root://stash.osgconnect.net", "ls", sourceDir], stdout=subprocess.PIPE).communicate()[0].split()
+    sourceItems = subprocess.Popen(["xrdfs", stash_origin, "ls", sourceDir], stdout=subprocess.PIPE).communicate()[0].split()
     for remote_file in sourceItems:
-        command2 = 'xrdfs root://stash.osgconnect.net stat '+ remote_file + ' | grep "IsDir" | wc -l'
+        command2 = 'xrdfs ' + stash_origin + ' stat '+ remote_file + ' | grep "IsDir" | wc -l'
         isdir=subprocess.Popen([command2],stdout=subprocess.PIPE,shell=True).communicate()[0].split()[0]
         if isdir!='0':
             result = dostashcpdirectory(remote_file, destination, cache, debug)

bin/stashcp2/tests/test_inside_docker.sh

@@ -42,7 +42,7 @@ cp /StashCache/bin/stashcp2/tests/job.ad ./.job.ad
 
 # Test against a file that is known to not exist
 set +e
-/StashCache/bin/stashcp --cache=$XRD_CACHE /blah/does/not/exist
+/StashCache/bin/stashcp --cache=$XRD_CACHE /blah/does/not/exist ./
 if [ $? -eq 0 ]; then
   echo "Failed to exit with non-zero exit status when it should have"
   exit 1
@@ -79,5 +79,15 @@ fi
 /StashCache/bin/stashcp --cache=$XRD_CACHE -d -r /user/dweitzel/public/blast/queries ./
 ls -lah
 
+rm -rf queries
+
+/StashCache/bin/stashcp --cache=$XRD_CACHE -d /xenon/rucio/x1t_SR001_170419_1605_mv/73/90/XENON1T-0-000008000-000008999-000001000.zip ./
+
+result=`md5sum XENON1T-0-000008000-000008999-000001000.zip | awk '{print $1;}'`
+if [ "$result" != "dd00dd6a6b1e0de4a3b8ecf1a34b24b3" ]; then
+  exit 1
+fi
+
+