1import logger 2 3import os 4import urllib 5import urlparse 6import config.base 7import socket 8 9# Fix parsing for nonstandard schemes 10urlparse.uses_netloc.extend(['bk', 'ssh', 'svn']) 11 12class Retriever(logger.Logger): 13 def __init__(self, sourceControl, clArgs = None, argDB = None): 14 logger.Logger.__init__(self, clArgs, argDB) 15 self.sourceControl = sourceControl 16 self.stamp = None 17 return 18 19 def getAuthorizedUrl(self, url): 20 '''This returns a tuple of the unauthorized and authorized URLs for the given URL, and a flag indicating which was input''' 21 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 22 if not location: 23 url = urlparse.urlunparse(('', '', path, parameters, query, fragment)) 24 authUrl = None 25 wasAuth = 0 26 else: 27 index = location.find('@') 28 if index >= 0: 29 login = location[0:index] 30 authUrl = url 31 url = urlparse.urlunparse((scheme, location[index+1:], path, parameters, query, fragment)) 32 wasAuth = 1 33 else: 34 login = location.split('.')[0] 35 authUrl = urlparse.urlunparse((scheme, login+'@'+location, path, parameters, query, fragment)) 36 wasAuth = 0 37 return (url, authUrl, wasAuth) 38 39 def testAuthorizedUrl(self, authUrl): 40 '''Raise an exception if the URL cannot receive an SSH login without a password''' 41 if not authUrl: 42 raise RuntimeError('Url is empty') 43 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(authUrl) 44 return self.executeShellCommand('echo "quit" | ssh -oBatchMode=yes '+location) 45 46 def genericRetrieve(self, url, root, name): 47 '''Fetch the gzipped tarfile indicated by url and expand it into root 48 - All the logic for removing old versions, updating etc. must move''' 49 50 # get the tarball file name from the URL 51 filename = os.path.basename(urlparse.urlparse(url)[2]) 52 localFile = os.path.join(root,'_d_'+filename) 53 ext = os.path.splitext(localFile)[1] 54 if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']: 55 raise RuntimeError('Unknown compression type in URL: '+ url) 56 self.logPrint('Downloading '+url+' to '+localFile) 57 if os.path.exists(localFile): 58 os.unlink(localFile) 59 60 try: 61 sav_timeout = socket.getdefaulttimeout() 62 socket.setdefaulttimeout(30) 63 urllib.urlretrieve(url, localFile) 64 socket.setdefaulttimeout(sav_timeout) 65 except Exception, e: 66 socket.setdefaulttimeout(sav_timeout) 67 failureMessage = '''\ 68Unable to download package %s from: %s 69* If URL specified manually - perhaps there is a typo? 70* If your network is disconnected - please reconnect and rerun ./configure 71* Or perhaps you have a firewall blocking the download 72* Alternatively, you can download the above URL manually, to /yourselectedlocation/%s 73 and use the configure option: 74 --download-%s=/yourselectedlocation/%s 75''' % (name, url, filename, name.lower(), filename) 76 raise RuntimeError(failureMessage) 77 78 self.logPrint('Extracting '+localFile) 79 if ext in ['.zip','.ZIP']: 80 config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log) 81 output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log) 82 dirname = os.path.normpath(output[0].strip()) 83 else: 84 failureMessage = '''\ 85Downloaded package %s from: %s is not a tarball. 86[or installed python cannot process compressed files] 87* If you are behind a firewall - please fix your proxy and rerun ./configure 88 For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to http://proxyout.lanl.gov 89* Alternatively, you can download the above URL manually, to /yourselectedlocation/%s 90 and use the configure option: 91 --download-%s=/yourselectedlocation/%s 92''' % (name, url, filename, name.lower(), filename) 93 import tarfile 94 try: 95 tf = tarfile.open(os.path.join(root, localFile)) 96 except tarfile.ReadError, e: 97 raise RuntimeError(str(e)+'\n'+failureMessage) 98 if not tf: raise RuntimeError(failureMessage) 99 #git puts 'pax_global_header' as the first entry and some tar utils process this as a file 100 firstname = tf.getnames()[0] 101 if firstname == 'pax_global_header': 102 firstmember = tf.getmembers()[1] 103 else: 104 firstmember = tf.getmembers()[0] 105 # some tarfiles list packagename/ but some list packagename/filename in the first entry 106 if firstmember.isdir(): 107 dirname = firstmember.name 108 else: 109 dirname = os.path.dirname(firstmember.name) 110 if hasattr(tf,'extractall'): #python 2.5+ 111 tf.extractall(root) 112 else: 113 for tfile in tf.getmembers(): 114 tf.extract(tfile,root) 115 tf.close() 116 117 # fix file permissions for the untared tarballs. 118 try: 119 # check if 'dirname' is set' 120 if dirname: 121 config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log) 122 else: 123 self.logPrintBox('WARNING: Could not determine dirname extracted by '+localFile+' to fix file permissions') 124 except RuntimeError, e: 125 raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e)) 126 os.unlink(localFile) 127 return 128 129 def ftpRetrieve(self, url, root, name,force): 130 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via ftp', 3, 'install') 131 return self.genericRetrieve(url, root, name) 132 133 def httpRetrieve(self, url, root, name,force): 134 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via http', 3, 'install') 135 return self.genericRetrieve(url, root, name) 136 137 def fileRetrieve(self, url, root, name,force): 138 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via cp', 3, 'install') 139 return self.genericRetrieve(url, root, name) 140 141 def svnRetrieve(self, url, root, name,force): 142 if not hasattr(self.sourceControl, 'svn'): 143 raise RuntimeError('Cannot retrieve a SVN repository since svn was not found') 144 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via svn', 3, 'install') 145 try: 146 config.base.Configure.executeShellCommand(self.sourceControl.svn+' checkout http'+url[3:]+' '+os.path.join(root, name)) 147 except RuntimeError: 148 pass 149 150 151 # This is the old code for updating a BK repository 152 # Stamp used to be stored with a url 153 def bkUpdate(self): 154 if not self.stamp is None and url in self.stamp: 155 if not self.stamp[url] == self.bkHeadRevision(root): 156 raise RuntimeError('Existing stamp for '+url+' does not match revision of repository in '+root) 157 (url, authUrl, wasAuth) = self.getAuthorizedUrl(self.getBKParentURL(root)) 158 if not wasAuth: 159 self.debugPrint('Changing parent from '+url+' --> '+authUrl, 1, 'install') 160 output = self.executeShellCommand('cd '+root+'; bk parent '+authUrl) 161 try: 162 self.testAuthorizedUrl(authUrl) 163 output = self.executeShellCommand('cd '+root+'; bk pull') 164 except RuntimeError, e: 165 (url, authUrl, wasAuth) = self.getAuthorizedUrl(self.getBKParentURL(root)) 166 if wasAuth: 167 self.debugPrint('Changing parent from '+authUrl+' --> '+url, 1, 'install') 168 output = self.executeShellCommand('cd '+root+'; bk parent '+url) 169 output = self.executeShellCommand('cd '+root+'; bk pull') 170 else: 171 raise e 172 return 173 174 def bkClone(self, url, root, name): 175 '''Clone a Bitkeeper repository located at url into root/name 176 - If self.stamp exists, clone only up to that revision''' 177 failureMessage = '''\ 178Unable to bk clone %s 179You may be off the network. Connect to the internet and run ./configure again 180or from the directory %s try: 181 bk clone %s 182and if that succeeds then rerun ./configure 183''' % (name, root, url, name) 184 try: 185 if not self.stamp is None and url in self.stamp: 186 (output, error, status) = self.executeShellCommand('bk clone -r'+self.stamp[url]+' '+url+' '+os.path.join(root, name)) 187 else: 188 (output, error, status) = self.executeShellCommand('bk clone '+url+' '+os.path.join(root, name)) 189 except RuntimeError, e: 190 status = 1 191 output = str(e) 192 error = '' 193 if status: 194 if output.find('ommand not found') >= 0: 195 failureMessage = 'Unable to locate bk (Bitkeeper) to download repository; make sure bk is in your path' 196 elif output.find('Cannot resolve host') >= 0: 197 failureMessage = output+'\n'+error+'\n'+failureMessage 198 else: 199 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 200 try: 201 self.bkClone(urlparse.urlunparse(('http', location, path, parameters, query, fragment)), root, name) 202 except RuntimeError, e: 203 failureMessage += '\n'+str(e) 204 else: 205 return 206 raise RuntimeError(failureMessage) 207 return 208 209 def bkRetrieve(self, url, root, name): 210 if not hasattr(self.sourceControl, 'bk'): 211 raise RuntimeError('Cannot retrieve a BitKeeper repository since BK was not found') 212 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via bk', 3, 'install') 213 (url, authUrl, wasAuth) = self.getAuthorizedUrl(url) 214 try: 215 self.testAuthorizedUrl(authUrl) 216 self.bkClone(authUrl, root, name) 217 except RuntimeError: 218 pass 219 else: 220 return 221 return self.bkClone(url, root, name) 222 223 def retrieve(self, url, root = None, canExist = 0, force = 0): 224 '''Retrieve the project corresponding to url 225 - If root is None, the local root directory is automatically determined. If the project 226 was already installed, this root is used. Otherwise a guess is made based upon the url. 227 - If canExist is True and the root exists, an update is done instead of a full download. 228 The canExist is automatically true if the project has been installed. The retrievalCanExist 229 flag can also be used to set this. 230 - If force is True, a full download is mandated. 231 Providing the root is an easy way to make a copy, for instance when making tarballs. 232 ''' 233 if root is None: 234 root = self.getInstallRoot(url) 235 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 236 if hasattr(self,scheme+'Retrieve'): 237 getattr(self, scheme+'Retrieve')(url, os.path.abspath(root), canExist, force) 238 else: 239 raise RuntimeError('Invalid transport for retrieval: '+scheme) 240 return 241 242 ############################################## 243 # This is the old shit 244 ############################################## 245 def removeRoot(self, root, canExist, force = 0): 246 '''Returns 1 if removes root''' 247 if os.path.exists(root): 248 if canExist: 249 if force: 250 import shutil 251 shutil.rmtree(root) 252 return 1 253 else: 254 return 0 255 else: 256 raise RuntimeError('Root directory '+root+' already exists') 257 return 1 258 259 def getBKParentURL(self, root): 260 '''Return the parent URL for the BK repository at "root"''' 261 return self.executeShellCommand('cd '+root+'; bk parent')[21:] 262 263 def bkHeadRevision(self, root): 264 '''Return the last change set revision in the repository''' 265 return self.executeShellCommand('cd '+root+'; bk changes -and:REV: | head -1') 266 267 def bkfileRetrieve(self, url, root, canExist = 0, force = 0): 268 self.debugPrint('Retrieving '+url+' --> '+root+' via local bk', 3, 'install') 269 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 270 return self.bkRetrieve(urlparse.urlunparse(('file', location, path, parameters, query, fragment)), root, canExist, force) 271 272 def sshRetrieve(self, url, root, canExist = 0, force = 0): 273 command = 'hg clone '+url+' '+os.path.join(root,os.path.basename(url)) 274 output = config.base.Configure.executeShellCommand(command) 275 return root 276 277 def oldRetrieve(self, url, root = None, canExist = 0, force = 0): 278 '''Retrieve the project corresponding to url 279 - If root is None, the local root directory is automatically determined. If the project 280 was already installed, this root is used. Otherwise a guess is made based upon the url. 281 - If canExist is True and the root exists, an update is done instead of a full download. 282 The canExist is automatically true if the project has been installed. The retrievalCanExist 283 flag can also be used to set this. 284 - If force is True, a full download is mandated. 285 Providing the root is an easy way to make a copy, for instance when making tarballs. 286 ''' 287 origUrl = url 288 url = self.getMappedUrl(origUrl) 289 project = self.getInstalledProject(url) 290 if not project is None and root is None: 291 root = project.getRoot() 292 canExist = 1 293 if root is None: 294 root = self.getInstallRoot(origUrl) 295 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 296 try: 297 if self.argDB['retrievalCanExist']: 298 canExist = 1 299 return getattr(self, scheme+'Retrieve')(url, os.path.abspath(root), canExist, force) 300 except AttributeError: 301 raise RuntimeError('Invalid transport for retrieval: '+scheme) 302