xref: /petsc/config/BuildSystem/retrieval.py (revision 3307d110e72ee4e6d2468971620073eb5ff93529) 
1from __future__ import absolute_import
2import logger
3
4import os
5try:
6  from urllib import urlretrieve
7except ImportError:
8  from urllib.request import urlretrieve
9try:
10  import urlparse as urlparse_local # novermin
11except ImportError:
12  from urllib import parse as urlparse_local # novermin
13import config.base
14import socket
15import shutil
16
17# Fix parsing for nonstandard schemes
18urlparse_local.uses_netloc.extend(['bk', 'ssh', 'svn'])
19
20class Retriever(logger.Logger):
21  def __init__(self, sourceControl, clArgs = None, argDB = None):
22    logger.Logger.__init__(self, clArgs, argDB)
23    self.sourceControl = sourceControl
24    self.gitsubmodules = []
25    self.gitprereq = 1
26    self.git_urls = []
27    self.hg_urls = []
28    self.dir_urls = []
29    self.link_urls = []
30    self.tarball_urls = []
31    self.stamp = None
32    return
33
34  def isGitURL(self, url):
35    parsed = urlparse_local.urlparse(url)
36    if (parsed[0] == 'git') or (parsed[0] == 'ssh' and parsed[2].endswith('.git')) or (parsed[0] == 'https' and parsed[2].endswith('.git')):
37      return True
38    elif os.path.isdir(url) and self.isDirectoryGitRepo(url):
39      return True
40    return False
41
42  def setupURLs(self,packagename,urls,gitsubmodules,gitprereq):
43    self.packagename = packagename
44    self.gitsubmodules = gitsubmodules
45    self.gitprereq = gitprereq
46    for url in urls:
47      parsed = urlparse_local.urlparse(url)
48      if self.isGitURL(url):
49        self.git_urls.append(self.removePrefix(url,'git://'))
50      elif parsed[0] == 'hg'or (parsed[0] == 'ssh' and parsed[1].startswith('hg@')):
51        self.hg_urls.append(self.removePrefix(url,'hg://'))
52      elif parsed[0] == 'dir' or os.path.isdir(url):
53        self.dir_urls.append(self.removePrefix(url,'dir://'))
54      elif parsed[0] == 'link':
55        self.link_urls.append(self.removePrefix(url,'link://'))
56      else:
57        # check for ftp.mcs.anl.gov - and use https://,www.mcs.anl.gov,ftp://
58        if url.find('ftp.mcs.anl.gov'):
59          https_url = url.replace('http://','https://').replace('ftp://','http://')
60          self.tarball_urls.extend([https_url,https_url.replace('ftp.mcs.anl.gov/pub/petsc/','www.mcs.anl.gov/petsc/mirror/'),https_url.replace('https://','ftp')])
61
62  def isDirectoryGitRepo(self, directory):
63    if not hasattr(self.sourceControl, 'git'):
64      self.logPrint('git not found in self.sourceControl - cannot evaluate isDirectoryGitRepo(): '+directory)
65      return False
66    from config.base import Configure
67    for loc in ['.git','']:
68      cmd = '%s rev-parse --resolve-git-dir  %s'  % (self.sourceControl.git, os.path.join(directory,loc))
69      (output, error, ret) = Configure.executeShellCommand(cmd, checkCommand = Configure.passCheckCommand, log = self.log)
70      if not ret:
71        return True
72    return False
73
74  @staticmethod
75  def removeTarget(t):
76    if os.path.islink(t) or os.path.isfile(t):
77      os.unlink(t) # same as os.remove(t)
78    elif os.path.isdir(t):
79      shutil.rmtree(t)
80
81  @staticmethod
82  def getDownloadFailureMessage(package, url, filename=None):
83    slashFilename = '/'+filename if filename else ''
84    return '''\
85Unable to download package %s from: %s
86* If URL specified manually - perhaps there is a typo?
87* If your network is disconnected - please reconnect and rerun ./configure
88* Or perhaps you have a firewall blocking the download
89* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually
90* or you can download the above URL manually, to /yourselectedlocation%s
91  and use the configure option:
92  --download-%s=/yourselectedlocation%s
93    ''' % (package.upper(), url, slashFilename, package, slashFilename)
94
95  @staticmethod
96  def removePrefix(url,prefix):
97    '''Replacement for str.removeprefix() supported only since Python 3.9'''
98    if url.startswith(prefix):
99      return url[len(prefix):]
100    return url
101
102  def generateURLs(self):
103    if hasattr(self.sourceControl, 'git') and self.gitprereq:
104      for url in self.git_urls:
105        yield('git',url)
106    else:
107      self.logPrint('Git not found or gitprereq check failed! skipping giturls: '+str(self.git_urls)+'\n')
108    if hasattr(self.sourceControl, 'hg'):
109      for url in self.hg_urls:
110        yield('hg',url)
111    else:
112      self.logPrint('Hg not found - skipping hgurls: '+str(self.hg_urls)+'\n')
113    for url in self.dir_urls:
114      yield('dir',url)
115    for url in self.link_urls:
116      yield'link',(url)
117    for url in self.tarball_urls:
118      yield('tarball',url)
119
120  def genericRetrieve(self,proto,url,root):
121    '''Fetch package from version control repository or tarfile indicated by URL and extract it into root'''
122    if proto == 'git':
123      return self.gitRetrieve(url,root)
124    elif proto == 'hg':
125      return self.hgRetrieve(url,root)
126    elif proto == 'dir':
127      return self.dirRetrieve(url,root)
128    elif proto == 'link':
129      self.linkRetrieve(url,root)
130    elif proto == 'tarball':
131      self.tarballRetrieve(url,root)
132
133  def dirRetrieve(self, url, root):
134    self.logPrint('Retrieving %s as directory' % url, 3, 'install')
135    if not os.path.isdir(url): raise RuntimeError('URL %s is not a directory' % url)
136
137    t = os.path.join(root,os.path.basename(url))
138    self.removeTarget(t)
139    shutil.copytree(url,t)
140
141  def linkRetrieve(self, url, root):
142    self.logPrint('Retrieving %s as link' % url, 3, 'install')
143    if not os.path.isdir(url): raise RuntimeError('URL %s is not pointing to a directory' % url)
144
145    t = os.path.join(root,os.path.basename(url))
146    self.removeTarget(t)
147    os.symlink(os.path.abspath(url),t)
148
149  def gitRetrieve(self, url, root):
150    self.logPrint('Retrieving %s as git repo' % url, 3, 'install')
151    if not hasattr(self.sourceControl, 'git'):
152      raise RuntimeError('self.sourceControl.git not set')
153    if os.path.isdir(url) and not self.isDirectoryGitRepo(url):
154      raise RuntimeError('URL %s is a directory but not a git repository' % url)
155
156    newgitrepo = os.path.join(root,'git.'+self.packagename)
157    self.removeTarget(newgitrepo)
158
159    try:
160      submodopt =''
161      for itm in self.gitsubmodules:
162        submodopt += ' --recurse-submodules='+itm
163      config.base.Configure.executeShellCommand('%s clone %s %s %s' % (self.sourceControl.git, submodopt, url, newgitrepo), log = self.log, timeout = 120.0)
164    except  RuntimeError as e:
165      self.logPrint('ERROR: '+str(e))
166      err = str(e)
167      failureMessage = self.getDownloadFailureMessage(self.packagename, url)
168      raise RuntimeError('Unable to clone '+self.packagename+'\n'+err+failureMessage)
169
170  def hgRetrieve(self, url, root):
171    self.logPrint('Retrieving %s as hg repo' % url, 3, 'install')
172    if not hasattr(self.sourceControl, 'hg'):
173      raise RuntimeError('self.sourceControl.hg not set')
174
175    newgitrepo = os.path.join(root,'hg.'+self.packagename)
176    self.removeTarget(newgitrepo)
177    try:
178      config.base.Configure.executeShellCommand('%s clone %s %s' % (self.sourceControl.hg, url, newgitrepo), log = self.log, timeout = 120.0)
179    except  RuntimeError as e:
180      self.logPrint('ERROR: '+str(e))
181      err = str(e)
182      failureMessage = self.getDownloadFailureMessage(self.packagename, url)
183      raise RuntimeError('Unable to clone '+self.packagename+'\n'+err+failureMessage)
184
185  def tarballRetrieve(self, url, root):
186    parsed = urlparse_local.urlparse(url)
187    filename = os.path.basename(parsed[2])
188    localFile = os.path.join(root,'_d_'+filename)
189    self.logPrint('Retrieving %s as tarball to %s' % (url,localFile) , 3, 'install')
190    ext =  os.path.splitext(localFile)[1]
191    if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']:
192      raise RuntimeError('Unknown compression type in URL: '+ url)
193
194    self.removeTarget(localFile)
195
196    if parsed[0] == 'file' and not parsed[1]:
197      url = parsed[2]
198    if os.path.exists(url):
199      if not os.path.isfile(url):
200        raise RuntimeError('Local path exists but is not a regular file: '+ url)
201      # copy local file
202      shutil.copyfile(url, localFile)
203    else:
204      # fetch remote file
205      try:
206        sav_timeout = socket.getdefaulttimeout()
207        socket.setdefaulttimeout(30)
208        urlretrieve(url, localFile)
209        socket.setdefaulttimeout(sav_timeout)
210      except Exception as e:
211        socket.setdefaulttimeout(sav_timeout)
212        failureMessage = self.getDownloadFailureMessage(self.packagename, url, filename)
213        raise RuntimeError(failureMessage)
214
215    self.logPrint('Extracting '+localFile)
216    if ext in ['.zip','.ZIP']:
217      config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log)
218      output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log)
219      dirname = os.path.normpath(output[0].strip())
220    else:
221      failureMessage = '''\
222Downloaded package %s from: %s is not a tarball.
223[or installed python cannot process compressed files]
224* If you are behind a firewall - please fix your proxy and rerun ./configure
225  For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to  http://proxyout.lanl.gov
226* You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually
227* or you can download the above URL manually, to /yourselectedlocation/%s
228  and use the configure option:
229  --download-%s=/yourselectedlocation/%s
230''' % (self.packagename.upper(), url, filename, self.packagename, filename)
231      import tarfile
232      try:
233        tf  = tarfile.open(os.path.join(root, localFile))
234      except tarfile.ReadError as e:
235        raise RuntimeError(str(e)+'\n'+failureMessage)
236      if not tf: raise RuntimeError(failureMessage)
237      #git puts 'pax_global_header' as the first entry and some tar utils process this as a file
238      firstname = tf.getnames()[0]
239      if firstname == 'pax_global_header':
240        firstmember = tf.getmembers()[1]
241      else:
242        firstmember = tf.getmembers()[0]
243      # some tarfiles list packagename/ but some list packagename/filename in the first entry
244      if firstmember.isdir():
245        dirname = firstmember.name
246      else:
247        dirname = os.path.dirname(firstmember.name)
248      tf.extractall(root)
249      tf.close()
250
251    # fix file permissions for the untared tarballs.
252    try:
253      # check if 'dirname' is set'
254      if dirname:
255        config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find  '+dirname + ' -type d -name "*" -exec chmod a+rx {} \;', log = self.log)
256      else:
257        self.logPrintBox('WARNING: Could not determine dirname extracted by '+localFile+' to fix file permissions')
258    except RuntimeError as e:
259      raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e))
260    os.unlink(localFile)
261