1from __future__ import absolute_import 2import logger 3 4import os 5try: 6 from urllib import urlretrieve 7except ImportError: 8 from urllib.request import urlretrieve 9try: 10 import urlparse as urlparse_local # novermin 11except ImportError: 12 from urllib import parse as urlparse_local 13import config.base 14import socket 15 16# Fix parsing for nonstandard schemes 17urlparse_local.uses_netloc.extend(['bk', 'ssh', 'svn']) 18 19class Retriever(logger.Logger): 20 def __init__(self, sourceControl, clArgs = None, argDB = None): 21 logger.Logger.__init__(self, clArgs, argDB) 22 self.sourceControl = sourceControl 23 self.stamp = None 24 return 25 26 def genericRetrieve(self, url, root, package): 27 '''Fetch the gzipped tarfile indicated by url and expand it into root 28 - All the logic for removing old versions, updating etc. must move''' 29 30 # copy a directory 31 if url.startswith('dir://'): 32 import shutil 33 dir = url[6:] 34 if not os.path.isdir(dir): raise RuntimeError('Url begins with dir:// but is not a directory') 35 36 if os.path.isdir(os.path.join(root,os.path.basename(dir))): shutil.rmtree(os.path.join(root,os.path.basename(dir))) 37 if os.path.isfile(os.path.join(root,os.path.basename(dir))): os.unlink(os.path.join(root,os.path.basename(dir))) 38 39 shutil.copytree(dir,os.path.join(root,os.path.basename(dir))) 40 return 41 42 if url.startswith('link://'): 43 import shutil 44 dir = url[7:] 45 if not os.path.isdir(dir): raise RuntimeError('Url begins with link:// but it is not pointing to a directory') 46 47 if os.path.islink(os.path.join(root,os.path.basename(dir))): os.unlink(os.path.join(root,os.path.basename(dir))) 48 if os.path.isfile(os.path.join(root,os.path.basename(dir))): os.unlink(os.path.join(root,os.path.basename(dir))) 49 if os.path.isdir(os.path.join(root,os.path.basename(dir))): shutil.rmtree(os.path.join(root,os.path.basename(dir))) 50 os.symlink(os.path.abspath(dir),os.path.join(root,os.path.basename(dir))) 51 return 52 53 if url.startswith('git://'): 54 if not hasattr(self.sourceControl, 'git'): return 55 import shutil 56 dir = url[6:] 57 if os.path.isdir(dir): 58 if not os.path.isdir(os.path.join(dir,'.git')): raise RuntimeError('Url begins with git:// and is a directory but but does not have a .git subdirectory') 59 60 newgitrepo = os.path.join(root,'git.'+package) 61 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 62 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 63 64 try: 65 config.base.Configure.executeShellCommand(self.sourceControl.git+' clone '+dir+' '+newgitrepo, log = self.log) 66 except RuntimeError as e: 67 self.logPrint('ERROR: '+str(e)) 68 err = str(e) 69 failureMessage = '''\ 70Unable to download package %s from: %s 71* If URL specified manually - perhaps there is a typo? 72* If your network is disconnected - please reconnect and rerun ./configure 73* Or perhaps you have a firewall blocking the download 74* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 75* or you can download the above URL manually, to /yourselectedlocation 76 and use the configure option: 77 --download-%s=/yourselectedlocation 78''' % (package.upper(), url, package) 79 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 80 return 81 82 if url.startswith('hg://'): 83 if not hasattr(self.sourceControl, 'hg'): return 84 85 newgitrepo = os.path.join(root,'hg.'+package) 86 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 87 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 88 try: 89 config.base.Configure.executeShellCommand(self.sourceControl.hg+' clone '+url[5:]+' '+newgitrepo) 90 except RuntimeError as e: 91 self.logPrint('ERROR: '+str(e)) 92 err = str(e) 93 failureMessage = '''\ 94Unable to download package %s from: %s 95* If URL specified manually - perhaps there is a typo? 96* If your network is disconnected - please reconnect and rerun ./configure 97* Or perhaps you have a firewall blocking the download 98* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 99* or you can download the above URL manually, to /yourselectedlocation 100 and use the configure option: 101 --download-%s=/yourselectedlocation 102''' % (package.upper(), url, package) 103 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 104 return 105 106 if url.startswith('ssh://hg@'): 107 if not hasattr(self.sourceControl, 'hg'): return 108 109 newgitrepo = os.path.join(root,'hg.'+package) 110 if os.path.isdir(newgitrepo): shutil.rmtree(newgitrepo) 111 if os.path.isfile(newgitrepo): os.unlink(newgitrepo) 112 try: 113 config.base.Configure.executeShellCommand(self.sourceControl.hg+' clone '+url+' '+newgitrepo) 114 except RuntimeError as e: 115 self.logPrint('ERROR: '+str(e)) 116 err = str(e) 117 failureMessage = '''\ 118Unable to download package %s from: %s 119* If URL specified manually - perhaps there is a typo? 120* If your network is disconnected - please reconnect and rerun ./configure 121* Or perhaps you have a firewall blocking the download 122* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 123* or you can download the above URL manually, to /yourselectedlocation 124 and use the configure option: 125 --download-%s=/yourselectedlocation 126''' % (package.upper(), url, package) 127 raise RuntimeError('Unable to download '+package+'\n'+err+failureMessage) 128 return 129 130 # get the tarball file name from the URL 131 filename = os.path.basename(urlparse_local.urlparse(url)[2]) 132 localFile = os.path.join(root,'_d_'+filename) 133 ext = os.path.splitext(localFile)[1] 134 if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']: 135 raise RuntimeError('Unknown compression type in URL: '+ url) 136 self.logPrint('Downloading '+url+' to '+localFile) 137 if os.path.exists(localFile): 138 os.unlink(localFile) 139 140 try: 141 sav_timeout = socket.getdefaulttimeout() 142 socket.setdefaulttimeout(30) 143 urlretrieve(url, localFile) 144 socket.setdefaulttimeout(sav_timeout) 145 except Exception as e: 146 socket.setdefaulttimeout(sav_timeout) 147 failureMessage = '''\ 148Unable to download package %s from: %s 149* If URL specified manually - perhaps there is a typo? 150* If your network is disconnected - please reconnect and rerun ./configure 151* Or perhaps you have a firewall blocking the download 152* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 153* or you can download the above URL manually, to /yourselectedlocation/%s 154 and use the configure option: 155 --download-%s=/yourselectedlocation/%s 156''' % (package.upper(), url, filename, package, filename) 157 raise RuntimeError(failureMessage) 158 159 self.logPrint('Extracting '+localFile) 160 if ext in ['.zip','.ZIP']: 161 config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log) 162 output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log) 163 dirname = os.path.normpath(output[0].strip()) 164 else: 165 failureMessage = '''\ 166Downloaded package %s from: %s is not a tarball. 167[or installed python cannot process compressed files] 168* If you are behind a firewall - please fix your proxy and rerun ./configure 169 For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to http://proxyout.lanl.gov 170* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually 171* or you can download the above URL manually, to /yourselectedlocation/%s 172 and use the configure option: 173 --download-%s=/yourselectedlocation/%s 174''' % (package.upper(), url, filename, package, filename) 175 import tarfile 176 try: 177 tf = tarfile.open(os.path.join(root, localFile)) 178 except tarfile.ReadError as e: 179 raise RuntimeError(str(e)+'\n'+failureMessage) 180 if not tf: raise RuntimeError(failureMessage) 181 #git puts 'pax_global_header' as the first entry and some tar utils process this as a file 182 firstname = tf.getnames()[0] 183 if firstname == 'pax_global_header': 184 firstmember = tf.getmembers()[1] 185 else: 186 firstmember = tf.getmembers()[0] 187 # some tarfiles list packagename/ but some list packagename/filename in the first entry 188 if firstmember.isdir(): 189 dirname = firstmember.name 190 else: 191 dirname = os.path.dirname(firstmember.name) 192 tf.extractall(root) 193 tf.close() 194 195 # fix file permissions for the untared tarballs. 196 try: 197 # check if 'dirname' is set' 198 if dirname: 199 config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log) 200 else: 201 self.logPrintBox('WARNING: Could not determine dirname extracted by '+localFile+' to fix file permissions') 202 except RuntimeError as e: 203 raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e)) 204 os.unlink(localFile) 205 return 206