1import logger 2 3import os 4import urllib 5import urlparse 6import config.base 7# Fix parsing for nonstandard schemes 8urlparse.uses_netloc.extend(['bk', 'ssh', 'svn']) 9 10class Retriever(logger.Logger): 11 def __init__(self, sourceControl, clArgs = None, argDB = None): 12 logger.Logger.__init__(self, clArgs, argDB) 13 self.sourceControl = sourceControl 14 self.stamp = None 15 return 16 17 def getAuthorizedUrl(self, url): 18 '''This returns a tuple of the unauthorized and authorized URLs for the given URL, and a flag indicating which was input''' 19 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 20 if not location: 21 url = urlparse.urlunparse(('', '', path, parameters, query, fragment)) 22 authUrl = None 23 wasAuth = 0 24 else: 25 index = location.find('@') 26 if index >= 0: 27 login = location[0:index] 28 authUrl = url 29 url = urlparse.urlunparse((scheme, location[index+1:], path, parameters, query, fragment)) 30 wasAuth = 1 31 else: 32 login = location.split('.')[0] 33 authUrl = urlparse.urlunparse((scheme, login+'@'+location, path, parameters, query, fragment)) 34 wasAuth = 0 35 return (url, authUrl, wasAuth) 36 37 def testAuthorizedUrl(self, authUrl): 38 '''Raise an exception if the URL cannot receive an SSH login without a password''' 39 if not authUrl: 40 raise RuntimeError('Url is empty') 41 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(authUrl) 42 return self.executeShellCommand('echo "quit" | ssh -oBatchMode=yes '+location) 43 44 def genericRetrieve(self, url, root, name): 45 '''Fetch the gzipped tarfile indicated by url and expand it into root 46 - All the logic for removing old versions, updating etc. must move''' 47 48 # get the tarball file name from the URL 49 filename = os.path.basename(urlparse.urlparse(url)[2]) 50 localFile = os.path.join(root,'_d_'+filename) 51 ext = os.path.splitext(localFile)[1] 52 if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']: 53 raise RuntimeError('Unknown compression type in URL: '+ url) 54 self.logPrint('Downloading '+url+' to '+localFile) 55 if os.path.exists(localFile): 56 os.unlink(localFile) 57 58 try: 59 urllib.urlretrieve(url, localFile) 60 except Exception, e: 61 failureMessage = '''\ 62Unable to download package %s from: %s 63* If URL specified manually - perhaps there is a typo? 64* If your network is disconnected - please reconnect and rerun ./configure 65* Or perhaps you have a firewall blocking the download 66* Alternatively, you can download the above URL manually, to /yourselectedlocation/%s 67 and use the configure option: 68 --download-%s=/yourselectedlocation/%s 69''' % (name, url, filename, name.lower(), filename) 70 raise RuntimeError(failureMessage) 71 72 self.logPrint('Extracting '+localFile) 73 if ext in ['.zip','.ZIP']: 74 config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log) 75 output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log) 76 dirname = os.path.normpath(output[0].strip()) 77 else: 78 failureMessage = '''\ 79Downloaded package %s from: %s is not a tarball. 80[or installed python cannot process compressed files] 81* If you are behind a firewall - please fix your proxy and rerun ./configure 82 For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to http://proxyout.lanl.gov 83* Alternatively, you can download the above URL manually, to /yourselectedlocation/%s 84 and use the configure option: 85 --download-%s=/yourselectedlocation/%s 86''' % (name, url, filename, name.lower(), filename) 87 import tarfile 88 try: 89 tf = tarfile.open(os.path.join(root, localFile)) 90 except tarfile.ReadError, e: 91 raise RuntimeError(str(e)+'\n'+failureMessage) 92 # some tarfiles list packagename/ but some list packagename/filename in the first entry 93 if not tf: raise RuntimeError(failureMessage) 94 if not tf.firstmember: raise RuntimeError(failureMessage) 95 if tf.firstmember.isdir(): 96 dirname = tf.firstmember.name 97 else: 98 dirname = os.path.dirname(tf.firstmember.name) 99 if hasattr(tf,'extractall'): #python 2.5+ 100 tf.extractall(root) 101 else: 102 for tfile in tf.getmembers(): 103 tf.extract(tfile,root) 104 tf.close() 105 106 # fix file permissions for the untared tarballs. 107 try: 108 config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log) 109 except RuntimeError, e: 110 raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e)) 111 os.unlink(localFile) 112 return 113 114 def ftpRetrieve(self, url, root, name,force): 115 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via ftp', 3, 'install') 116 return self.genericRetrieve(url, root, name) 117 118 def httpRetrieve(self, url, root, name,force): 119 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via http', 3, 'install') 120 return self.genericRetrieve(url, root, name) 121 122 def fileRetrieve(self, url, root, name,force): 123 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via cp', 3, 'install') 124 return self.genericRetrieve(url, root, name) 125 126 def svnRetrieve(self, url, root, name,force): 127 if not hasattr(self.sourceControl, 'svn'): 128 raise RuntimeError('Cannot retrieve a SVN repository since svn was not found') 129 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via svn', 3, 'install') 130 try: 131 config.base.Configure.executeShellCommand(self.sourceControl.svn+' checkout http'+url[3:]+' '+os.path.join(root, name)) 132 except RuntimeError: 133 pass 134 135 136 # This is the old code for updating a BK repository 137 # Stamp used to be stored with a url 138 def bkUpdate(self): 139 if not self.stamp is None and url in self.stamp: 140 if not self.stamp[url] == self.bkHeadRevision(root): 141 raise RuntimeError('Existing stamp for '+url+' does not match revision of repository in '+root) 142 (url, authUrl, wasAuth) = self.getAuthorizedUrl(self.getBKParentURL(root)) 143 if not wasAuth: 144 self.debugPrint('Changing parent from '+url+' --> '+authUrl, 1, 'install') 145 output = self.executeShellCommand('cd '+root+'; bk parent '+authUrl) 146 try: 147 self.testAuthorizedUrl(authUrl) 148 output = self.executeShellCommand('cd '+root+'; bk pull') 149 except RuntimeError, e: 150 (url, authUrl, wasAuth) = self.getAuthorizedUrl(self.getBKParentURL(root)) 151 if wasAuth: 152 self.debugPrint('Changing parent from '+authUrl+' --> '+url, 1, 'install') 153 output = self.executeShellCommand('cd '+root+'; bk parent '+url) 154 output = self.executeShellCommand('cd '+root+'; bk pull') 155 else: 156 raise e 157 return 158 159 def bkClone(self, url, root, name): 160 '''Clone a Bitkeeper repository located at url into root/name 161 - If self.stamp exists, clone only up to that revision''' 162 failureMessage = '''\ 163Unable to bk clone %s 164You may be off the network. Connect to the internet and run ./configure again 165or from the directory %s try: 166 bk clone %s 167and if that succeeds then rerun ./configure 168''' % (name, root, url, name) 169 try: 170 if not self.stamp is None and url in self.stamp: 171 (output, error, status) = self.executeShellCommand('bk clone -r'+self.stamp[url]+' '+url+' '+os.path.join(root, name)) 172 else: 173 (output, error, status) = self.executeShellCommand('bk clone '+url+' '+os.path.join(root, name)) 174 except RuntimeError, e: 175 status = 1 176 output = str(e) 177 error = '' 178 if status: 179 if output.find('ommand not found') >= 0: 180 failureMessage = 'Unable to locate bk (Bitkeeper) to download repository; make sure bk is in your path' 181 elif output.find('Cannot resolve host') >= 0: 182 failureMessage = output+'\n'+error+'\n'+failureMessage 183 else: 184 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 185 try: 186 self.bkClone(urlparse.urlunparse(('http', location, path, parameters, query, fragment)), root, name) 187 except RuntimeError, e: 188 failureMessage += '\n'+str(e) 189 else: 190 return 191 raise RuntimeError(failureMessage) 192 return 193 194 def bkRetrieve(self, url, root, name): 195 if not hasattr(self.sourceControl, 'bk'): 196 raise RuntimeError('Cannot retrieve a BitKeeper repository since BK was not found') 197 self.logPrint('Retrieving '+url+' --> '+os.path.join(root, name)+' via bk', 3, 'install') 198 (url, authUrl, wasAuth) = self.getAuthorizedUrl(url) 199 try: 200 self.testAuthorizedUrl(authUrl) 201 self.bkClone(authUrl, root, name) 202 except RuntimeError: 203 pass 204 else: 205 return 206 return self.bkClone(url, root, name) 207 208 def retrieve(self, url, root = None, canExist = 0, force = 0): 209 '''Retrieve the project corresponding to url 210 - If root is None, the local root directory is automatically determined. If the project 211 was already installed, this root is used. Otherwise a guess is made based upon the url. 212 - If canExist is True and the root exists, an update is done instead of a full download. 213 The canExist is automatically true if the project has been installed. The retrievalCanExist 214 flag can also be used to set this. 215 - If force is True, a full download is mandated. 216 Providing the root is an easy way to make a copy, for instance when making tarballs. 217 ''' 218 if root is None: 219 root = self.getInstallRoot(url) 220 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 221 if hasattr(self,scheme+'Retrieve'): 222 getattr(self, scheme+'Retrieve')(url, os.path.abspath(root), canExist, force) 223 else: 224 raise RuntimeError('Invalid transport for retrieval: '+scheme) 225 return 226 227 ############################################## 228 # This is the old shit 229 ############################################## 230 def removeRoot(self, root, canExist, force = 0): 231 '''Returns 1 if removes root''' 232 if os.path.exists(root): 233 if canExist: 234 if force: 235 import shutil 236 shutil.rmtree(root) 237 return 1 238 else: 239 return 0 240 else: 241 raise RuntimeError('Root directory '+root+' already exists') 242 return 1 243 244 def getBKParentURL(self, root): 245 '''Return the parent URL for the BK repository at "root"''' 246 return self.executeShellCommand('cd '+root+'; bk parent')[21:] 247 248 def bkHeadRevision(self, root): 249 '''Return the last change set revision in the repository''' 250 return self.executeShellCommand('cd '+root+'; bk changes -and:REV: | head -1') 251 252 def bkfileRetrieve(self, url, root, canExist = 0, force = 0): 253 self.debugPrint('Retrieving '+url+' --> '+root+' via local bk', 3, 'install') 254 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 255 return self.bkRetrieve(urlparse.urlunparse(('file', location, path, parameters, query, fragment)), root, canExist, force) 256 257 def sshRetrieve(self, url, root, canExist = 0, force = 0): 258 command = 'hg clone '+url+' '+os.path.join(root,os.path.basename(url)) 259 output = config.base.Configure.executeShellCommand(command) 260 return root 261 262 def oldRetrieve(self, url, root = None, canExist = 0, force = 0): 263 '''Retrieve the project corresponding to url 264 - If root is None, the local root directory is automatically determined. If the project 265 was already installed, this root is used. Otherwise a guess is made based upon the url. 266 - If canExist is True and the root exists, an update is done instead of a full download. 267 The canExist is automatically true if the project has been installed. The retrievalCanExist 268 flag can also be used to set this. 269 - If force is True, a full download is mandated. 270 Providing the root is an easy way to make a copy, for instance when making tarballs. 271 ''' 272 origUrl = url 273 url = self.getMappedUrl(origUrl) 274 project = self.getInstalledProject(url) 275 if not project is None and root is None: 276 root = project.getRoot() 277 canExist = 1 278 if root is None: 279 root = self.getInstallRoot(origUrl) 280 (scheme, location, path, parameters, query, fragment) = urlparse.urlparse(url) 281 try: 282 if self.argDB['retrievalCanExist']: 283 canExist = 1 284 return getattr(self, scheme+'Retrieve')(url, os.path.abspath(root), canExist, force) 285 except AttributeError: 286 raise RuntimeError('Invalid transport for retrieval: '+scheme) 287