1import logger 2 3import os 4import urllib 5import urlparse 6import config.base 7# Fix parsing for nonstandard schemes 8urlparse.uses_netloc.extend(['bk', 'ssh', 'svn']) 9 10class Retriever(logger.Logger): 11 def __init__(self, sourceControl, clArgs = None, argDB = None): 12 logger.Logger.__init__(self, clArgs, argDB) 13 self.sourceControl = sourceControl 14 self.stamp = None 15 return 16 17 def getAuthorizedUrl(self, url): 18 '''This returns a tuple of the unauthorized and authorized URLs for the given URL, and a flag indicating which was input''' 19 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 20 if not location: 21 url = urlparse.urlunparse(('', '', path, parameters, query, fragment)) 22 authUrl = None 23 wasAuth = 0 24 else: 25 index = location.find('@') 26 if index >= 0: 27 login = location[0:index] 28 authUrl = url 29 url = urlparse.urlunparse((scheme, location[index+1:], path, parameters, query, fragment)) 30 wasAuth = 1 31 else: 32 login = location.split('.')[0] 33 authUrl = urlparse.urlunparse((scheme, login+'@'+location, path, parameters, query, fragment)) 34 wasAuth = 0 35 return (url, authUrl, wasAuth) 36 37 def testAuthorizedUrl(self, authUrl): 38 '''Raise an exception if the URL cannot receive an SSH login without a password''' 39 if not authUrl: 40 raise RuntimeError('Url is empty') 41 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(authUrl) 42 return self.executeShellCommand('echo "quit" | ssh -oBatchMode=yes '+location) 43 44 def genericRetrieve(self, url, root, name): 45 '''Fetch the gzipped tarfile indicated by url and expand it into root 46 - All the logic for removing old versions, updating etc. must move''' 47 48 # get the tarball file name from the URL 49 filename = os.path.basename(urlparse.urlparse(url)[2]) 50 localFile = os.path.join(root,'_d_'+filename) 51 ext = os.path.splitext(localFile)[1] 52 if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']: 53 raise RuntimeError('Unknown compression type in URL: '+ url) 54 self.logPrint('Downloading '+url+' to '+localFile) 55 if os.path.exists(localFile): 56 os.unlink(localFile) 57 58 try: 59 urllib.urlretrieve(url, localFile) 60 except Exception, e: 61 failureMessage = '''\ 62Unable to download package %s from: %s 63* If URL specified manually - perhaps there is a typo? 64* If your network is disconnected - please reconnect and rerun ./configure 65* Or perhaps you have a firewall blocking the download 66* Alternatively, you can download the above URL manually, to /yourselectedlocation/%s 67 and use the configure option: 68 --download-%s=/yourselectedlocation/%s 69''' % (name, url, filename, name.lower(), filename) 70 raise RuntimeError(failureMessage) 71 72 self.logPrint('Extracting '+localFile) 73 if ext in ['.zip','.ZIP']: 74 config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log) 75 output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log) 76 dirname = os.path.normpath(output[0].strip()) 77 else: 78 failureMessage = '''\ 79Downloaded package %s from: %s is not a tarball. 80[or installed python cannot process compressed files] 81* If you are behind a firewall - please fix your proxy and rerun ./configure 82 For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to http://proxyout.lanl.gov 83* Alternatively, you can download the above URL manually, to /yourselectedlocation/%s 84 and use the configure option: 85 --download-%s=/yourselectedlocation/%s 86''' % (name, url, filename, name.lower(), filename) 87 import tarfile 88 try: 89 tf = tarfile.open(os.path.join(root, localFile)) 90 except tarfile.ReadError, e: 91 raise RuntimeError(str(e)+'\n'+failureMessage) 92 if not tf: raise RuntimeError(failureMessage) 93 #git puts 'pax_global_header' as the first entry and some tar utils process this as a file 94 firstname = tf.getnames()[0] 95 if firstname == 'pax_global_header': 96 firstmember = tf.getmembers()[1] 97 else: 98 firstmember = tf.getmembers()[0] 99 # some tarfiles list packagename/ but some list packagename/filename in the first entry 100 if firstmember.isdir(): 101 dirname = firstmember.name 102 else: 103 dirname = os.path.dirname(firstmember.name) 104 if hasattr(tf,'extractall'): #python 2.5+ 105 tf.extractall(root) 106 else: 107 for tfile in tf.getmembers(): 108 tf.extract(tfile,root) 109 tf.close() 110 111 # fix file permissions for the untared tarballs. 112 try: 113 # check if 'dirname' is set' 114 if dirname: 115 config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log) 116 else: 117 self.logPrintBox('WARNING: Could not determine dirname extracted by '+localFile+' to fix file permissions') 118 except RuntimeError, e: 119 raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e)) 120 os.unlink(localFile) 121 return 122 123 def ftpRetrieve(self, url, root, name,force): 124 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via ftp', 3, 'install') 125 return self.genericRetrieve(url, root, name) 126 127 def httpRetrieve(self, url, root, name,force): 128 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via http', 3, 'install') 129 return self.genericRetrieve(url, root, name) 130 131 def fileRetrieve(self, url, root, name,force): 132 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via cp', 3, 'install') 133 return self.genericRetrieve(url, root, name) 134 135 def svnRetrieve(self, url, root, name,force): 136 if not hasattr(self.sourceControl, 'svn'): 137 raise RuntimeError('Cannot retrieve a SVN repository since svn was not found') 138 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via svn', 3, 'install') 139 try: 140 config.base.Configure.executeShellCommand(self.sourceControl.svn+' checkout http'+url[3:]+' '+os.path.join(root, name)) 141 except RuntimeError: 142 pass 143 144 145 # This is the old code for updating a BK repository 146 # Stamp used to be stored with a url 147 def bkUpdate(self): 148 if not self.stamp is None and url in self.stamp: 149 if not self.stamp[url] == self.bkHeadRevision(root): 150 raise RuntimeError('Existing stamp for '+url+' does not match revision of repository in '+root) 151 (url, authUrl, wasAuth) = self.getAuthorizedUrl(self.getBKParentURL(root)) 152 if not wasAuth: 153 self.debugPrint('Changing parent from '+url+' --> '+authUrl, 1, 'install') 154 output = self.executeShellCommand('cd '+root+'; bk parent '+authUrl) 155 try: 156 self.testAuthorizedUrl(authUrl) 157 output = self.executeShellCommand('cd '+root+'; bk pull') 158 except RuntimeError, e: 159 (url, authUrl, wasAuth) = self.getAuthorizedUrl(self.getBKParentURL(root)) 160 if wasAuth: 161 self.debugPrint('Changing parent from '+authUrl+' --> '+url, 1, 'install') 162 output = self.executeShellCommand('cd '+root+'; bk parent '+url) 163 output = self.executeShellCommand('cd '+root+'; bk pull') 164 else: 165 raise e 166 return 167 168 def bkClone(self, url, root, name): 169 '''Clone a Bitkeeper repository located at url into root/name 170 - If self.stamp exists, clone only up to that revision''' 171 failureMessage = '''\ 172Unable to bk clone %s 173You may be off the network. Connect to the internet and run ./configure again 174or from the directory %s try: 175 bk clone %s 176and if that succeeds then rerun ./configure 177''' % (name, root, url, name) 178 try: 179 if not self.stamp is None and url in self.stamp: 180 (output, error, status) = self.executeShellCommand('bk clone -r'+self.stamp[url]+' '+url+' '+os.path.join(root, name)) 181 else: 182 (output, error, status) = self.executeShellCommand('bk clone '+url+' '+os.path.join(root, name)) 183 except RuntimeError, e: 184 status = 1 185 output = str(e) 186 error = '' 187 if status: 188 if output.find('ommand not found') >= 0: 189 failureMessage = 'Unable to locate bk (Bitkeeper) to download repository; make sure bk is in your path' 190 elif output.find('Cannot resolve host') >= 0: 191 failureMessage = output+'\n'+error+'\n'+failureMessage 192 else: 193 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 194 try: 195 self.bkClone(urlparse.urlunparse(('http', location, path, parameters, query, fragment)), root, name) 196 except RuntimeError, e: 197 failureMessage += '\n'+str(e) 198 else: 199 return 200 raise RuntimeError(failureMessage) 201 return 202 203 def bkRetrieve(self, url, root, name): 204 if not hasattr(self.sourceControl, 'bk'): 205 raise RuntimeError('Cannot retrieve a BitKeeper repository since BK was not found') 206 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via bk', 3, 'install') 207 (url, authUrl, wasAuth) = self.getAuthorizedUrl(url) 208 try: 209 self.testAuthorizedUrl(authUrl) 210 self.bkClone(authUrl, root, name) 211 except RuntimeError: 212 pass 213 else: 214 return 215 return self.bkClone(url, root, name) 216 217 def retrieve(self, url, root = None, canExist = 0, force = 0): 218 '''Retrieve the project corresponding to url 219 - If root is None, the local root directory is automatically determined. If the project 220 was already installed, this root is used. Otherwise a guess is made based upon the url. 221 - If canExist is True and the root exists, an update is done instead of a full download. 222 The canExist is automatically true if the project has been installed. The retrievalCanExist 223 flag can also be used to set this. 224 - If force is True, a full download is mandated. 225 Providing the root is an easy way to make a copy, for instance when making tarballs. 226 ''' 227 if root is None: 228 root = self.getInstallRoot(url) 229 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 230 if hasattr(self,scheme+'Retrieve'): 231 getattr(self, scheme+'Retrieve')(url, os.path.abspath(root), canExist, force) 232 else: 233 raise RuntimeError('Invalid transport for retrieval: '+scheme) 234 return 235 236 ############################################## 237 # This is the old shit 238 ############################################## 239 def removeRoot(self, root, canExist, force = 0): 240 '''Returns 1 if removes root''' 241 if os.path.exists(root): 242 if canExist: 243 if force: 244 import shutil 245 shutil.rmtree(root) 246 return 1 247 else: 248 return 0 249 else: 250 raise RuntimeError('Root directory '+root+' already exists') 251 return 1 252 253 def getBKParentURL(self, root): 254 '''Return the parent URL for the BK repository at "root"''' 255 return self.executeShellCommand('cd '+root+'; bk parent')[21:] 256 257 def bkHeadRevision(self, root): 258 '''Return the last change set revision in the repository''' 259 return self.executeShellCommand('cd '+root+'; bk changes -and:REV: | head -1') 260 261 def bkfileRetrieve(self, url, root, canExist = 0, force = 0): 262 self.debugPrint('Retrieving '+url+' --> '+root+' via local bk', 3, 'install') 263 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 264 return self.bkRetrieve(urlparse.urlunparse(('file', location, path, parameters, query, fragment)), root, canExist, force) 265 266 def sshRetrieve(self, url, root, canExist = 0, force = 0): 267 command = 'hg clone '+url+' '+os.path.join(root,os.path.basename(url)) 268 output = config.base.Configure.executeShellCommand(command) 269 return root 270 271 def oldRetrieve(self, url, root = None, canExist = 0, force = 0): 272 '''Retrieve the project corresponding to url 273 - If root is None, the local root directory is automatically determined. If the project 274 was already installed, this root is used. Otherwise a guess is made based upon the url. 275 - If canExist is True and the root exists, an update is done instead of a full download. 276 The canExist is automatically true if the project has been installed. The retrievalCanExist 277 flag can also be used to set this. 278 - If force is True, a full download is mandated. 279 Providing the root is an easy way to make a copy, for instance when making tarballs. 280 ''' 281 origUrl = url 282 url = self.getMappedUrl(origUrl) 283 project = self.getInstalledProject(url) 284 if not project is None and root is None: 285 root = project.getRoot() 286 canExist = 1 287 if root is None: 288 root = self.getInstallRoot(origUrl) 289 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 290 try: 291 if self.argDB['retrievalCanExist']: 292 canExist = 1 293 return getattr(self, scheme+'Retrieve')(url, os.path.abspath(root), canExist, force) 294 except AttributeError: 295 raise RuntimeError('Invalid transport for retrieval: '+scheme) 296