xref: /petsc/config/BuildSystem/RDict.py (revision 2fac8a03f163eebefc7dd21311cf84e2c3ee1604)
1179860b2SJed Brown'''A remote dictionary server
2179860b2SJed Brown
3179860b2SJed Brown    RDict is a typed, hierarchical, persistent dictionary intended to manage
4179860b2SJed Brown    all arguments or options for a program. The interface remains exactly the
5179860b2SJed Brown    same as dict, but the storage is more complicated.
6179860b2SJed Brown
7179860b2SJed Brown    Argument typing is handled by wrapping all values stored in the dictionary
8179860b2SJed Brown    with nargs.Arg or a subclass. A user can call setType() to set the type of
9179860b2SJed Brown    an argument without any value being present. Whenever __getitem__() or
10179860b2SJed Brown    __setitem__() is called, values are extracted or replaced in the wrapper.
11179860b2SJed Brown    These wrappers can be accessed directly using getType(), setType(), and
12179860b2SJed Brown    types().
13179860b2SJed Brown
14179860b2SJed Brown    Hierarchy is allowed using a single "parent" dictionary. All operations
15179860b2SJed Brown    cascade to the parent. For instance, the length of the dictionary is the
16179860b2SJed Brown    number of local keys plus the number of keys in the parent, and its
17179860b2SJed Brown    parent, etc. Also, a dictionary need not have a parent. If a key does not
18d5b43468SJose E. Roman    appear in the local dictionary, the call if passed to the parent. However,
19179860b2SJed Brown    in this case we see that local keys can shadow those in a parent.
20179860b2SJed Brown    Communication with the parent is handled using sockets, with the parent
21179860b2SJed Brown    being a server and the interactive dictionary a client.
22179860b2SJed Brown
23179860b2SJed Brown    The default persistence mechanism is a pickle file, RDict.db, written
24179860b2SJed Brown    whenever an argument is changed locally. A timer thread is created after
25179860b2SJed Brown    an initial change, so that many rapid changes do not cause many writes.
26179860b2SJed Brown    Each dictionary only saves its local entries, so all parents also
27179860b2SJed Brown    separately save data in different RDict.db files. Each time a dictionary
28179860b2SJed Brown    is created, the current directory is searched for an RDict.db file, and
29179860b2SJed Brown    if found the contents are loaded into the dictionary.
30179860b2SJed Brown
31179860b2SJed Brown    This script also provides some default actions:
32179860b2SJed Brown
33179860b2SJed Brown      - server [parent]
34179860b2SJed Brown        Starts a server in the current directory with an optional parent. This
35179860b2SJed Brown        server will accept socket connections from other dictionaries and act
36179860b2SJed Brown        as a parent.
37179860b2SJed Brown
38179860b2SJed Brown      - client [parent]
39179860b2SJed Brown        Creates a dictionary in the current directory with an optional parent
40179860b2SJed Brown        and lists the contents. Notice that the contents may come from either
41179860b2SJed Brown        an RDict.db file in the current directory, or from the parent.
42179860b2SJed Brown
43179860b2SJed Brown      - clear [parent]
44179860b2SJed Brown        Creates a dictionary in the current directory with an optional parent
45179860b2SJed Brown        and clears the contents. Notice that this will also clear the parent.
46179860b2SJed Brown
47179860b2SJed Brown      - insert <parent> <key> <value>
48179860b2SJed Brown        Creates a dictionary in the current directory with a parent, and inserts
49179860b2SJed Brown        the key-value pair. If "parent" is "None", no parent is assigned.
50179860b2SJed Brown
51179860b2SJed Brown      - remove <parent> <key>
52179860b2SJed Brown        Creates a dictionary in the current directory with a parent, and removes
53179860b2SJed Brown        the given key. If "parent" is "None", no parent is assigned.
54179860b2SJed Brown'''
555b6bfdb9SJed Brownfrom __future__ import print_function
565b6bfdb9SJed Brownfrom __future__ import absolute_import
57179860b2SJed Browntry:
58179860b2SJed Brown  import project          # This is necessary for us to create Project objects on load
59179860b2SJed Brown  import build.buildGraph # This is necessary for us to create BuildGraph objects on load
60179860b2SJed Brownexcept ImportError:
61179860b2SJed Brown  pass
62179860b2SJed Brownimport nargs
63179860b2SJed Brown
64492432c8SJed Brownimport pickle
65179860b2SJed Brownimport os
66179860b2SJed Brownimport sys
67179860b2SJed BrownuseThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:])
68179860b2SJed Brownif useThreads is None:
6992626d4aSBarry Smith  useThreads = 0 # workaround issue with parallel configure
7092626d4aSBarry Smithelif useThreads == 'no' or useThreads == '0':
7192626d4aSBarry Smith  useThreads = 0
7292626d4aSBarry Smithelif useThreads == 'yes' or useThreads == '1':
7331de54b5SMatthew G. Knepley  useThreads = 1
74179860b2SJed Brownelse:
7592626d4aSBarry Smith  raise RuntimeError('Unknown option value for --useThreads ',useThreads)
76179860b2SJed Brown
77179860b2SJed Brownclass RDict(dict):
78179860b2SJed Brown  '''An RDict is a typed dictionary, which may be hierarchically composed. All elements derive from the
79179860b2SJed BrownArg class, which wraps the usual value.'''
80179860b2SJed Brown  # The server will self-shutdown after this many seconds
81179860b2SJed Brown  shutdownDelay = 60*60*5
82179860b2SJed Brown
83179860b2SJed Brown  def __init__(self, parentAddr = None, parentDirectory = None, load = 1, autoShutdown = 1, readonly = False):
84179860b2SJed Brown    import atexit
85179860b2SJed Brown    import time
86179860b2SJed Brown
87179860b2SJed Brown    self.logFile         = None
88179860b2SJed Brown    self.setupLogFile()
89179860b2SJed Brown    self.target          = ['default']
90179860b2SJed Brown    self.parent          = None
91179860b2SJed Brown    self.saveTimer       = None
92179860b2SJed Brown    self.shutdownTimer   = None
93179860b2SJed Brown    self.lastAccess      = time.time()
94179860b2SJed Brown    self.saveFilename    = 'RDict.db'
95179860b2SJed Brown    self.addrFilename    = 'RDict.loc'
96179860b2SJed Brown    self.parentAddr      = parentAddr
97179860b2SJed Brown    self.isServer        = 0
98179860b2SJed Brown    self.readonly        = readonly
99179860b2SJed Brown    self.parentDirectory = parentDirectory
100492432c8SJed Brown    self.stopCmd         = pickle.dumps(('stop',))
101179860b2SJed Brown    self.writeLogLine('Greetings')
102179860b2SJed Brown    self.connectParent(self.parentAddr, self.parentDirectory)
103179860b2SJed Brown    if load: self.load()
104179860b2SJed Brown    if autoShutdown and useThreads:
105179860b2SJed Brown      atexit.register(self.shutdown)
106179860b2SJed Brown    self.writeLogLine('SERVER: Last access '+str(self.lastAccess))
107179860b2SJed Brown    return
108179860b2SJed Brown
109179860b2SJed Brown  def __getstate__(self):
110*9bb2fd42SSatish Balay    '''Remove any parent socket object, and the log file from the dictionary before pickling'''
111179860b2SJed Brown    self.writeLogLine('Pickling RDict')
112179860b2SJed Brown    d = self.__dict__.copy()
113179860b2SJed Brown    if 'parent'    in d: del d['parent']
114179860b2SJed Brown    if 'saveTimer' in d: del d['saveTimer']
115179860b2SJed Brown    if '_setCommandLine' in d: del d['_setCommandLine']
116179860b2SJed Brown    del d['logFile']
117179860b2SJed Brown    return d
118179860b2SJed Brown
119179860b2SJed Brown  def __setstate__(self, d):
120*9bb2fd42SSatish Balay    '''Reconnect the parent socket object and reopen the log file after unpickling'''
121c6ef1b5bSJed Brown    self.logFile  = open('RDict.log', 'a')
122179860b2SJed Brown    self.writeLogLine('Unpickling RDict')
123179860b2SJed Brown    self.__dict__.update(d)
124179860b2SJed Brown    self.connectParent(self.parentAddr, self.parentDirectory)
125179860b2SJed Brown    return
126179860b2SJed Brown
127179860b2SJed Brown  def setupLogFile(self, filename = 'RDict.log'):
128179860b2SJed Brown    if not self.logFile is None:
129179860b2SJed Brown      self.logFile.close()
130179860b2SJed Brown    if os.path.isfile(filename) and os.stat(filename).st_size > 10*1024*1024:
131179860b2SJed Brown      if os.path.isfile(filename+'.bkp'):
132179860b2SJed Brown        os.remove(filename+'.bkp')
133179860b2SJed Brown      os.rename(filename, filename+'.bkp')
134c6ef1b5bSJed Brown      self.logFile = open(filename, 'w')
135179860b2SJed Brown    else:
136c6ef1b5bSJed Brown      self.logFile = open(filename, 'a')
137179860b2SJed Brown    return
138179860b2SJed Brown
139179860b2SJed Brown  def writeLogLine(self, message):
140179860b2SJed Brown    '''Writes the message to the log along with the current time'''
141179860b2SJed Brown    import time
142179860b2SJed Brown    self.logFile.write('('+str(os.getpid())+')('+str(id(self))+')'+message+' ['+time.asctime(time.localtime())+']\n')
143179860b2SJed Brown    self.logFile.flush()
144179860b2SJed Brown    return
145179860b2SJed Brown
146179860b2SJed Brown  def __len__(self):
147179860b2SJed Brown    '''Returns the length of both the local and parent dictionaries'''
148179860b2SJed Brown    length = dict.__len__(self)
149179860b2SJed Brown    if not self.parent is None:
150179860b2SJed Brown      length = length + self.send()
151179860b2SJed Brown    return length
152179860b2SJed Brown
153179860b2SJed Brown  def getType(self, key):
154179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Returns the Arg object or None if not found.'''
1553b049ba2SJed Brown    try:
1563b049ba2SJed Brown      value = dict.__getitem__(self, key)
1573b049ba2SJed Brown      self.writeLogLine('getType: Getting local type for '+key+' '+str(value))
1583b049ba2SJed Brown      return value
1593b049ba2SJed Brown    except KeyError:
1603b049ba2SJed Brown      pass
1613b049ba2SJed Brown    if self.parent:
162179860b2SJed Brown      return self.send(key)
163179860b2SJed Brown    return None
164179860b2SJed Brown
1653b049ba2SJed Brown  def dict_has_key(self, key):
1663b049ba2SJed Brown    """Utility to check whether the key is present in the dictionary without RDict side-effects."""
1673b049ba2SJed Brown    return key in dict(self)
1683b049ba2SJed Brown
169179860b2SJed Brown  def __getitem__(self, key):
170179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Returns the value of the Arg.
171179860b2SJed Brown       - If the value has not been set, the user will be prompted for input'''
1723b049ba2SJed Brown    if self.dict_has_key(key):
173179860b2SJed Brown      self.writeLogLine('__getitem__: '+key+' has local type')
174179860b2SJed Brown      pass
175179860b2SJed Brown    elif not self.parent is None:
176179860b2SJed Brown      self.writeLogLine('__getitem__: Checking parent value')
177179860b2SJed Brown      if self.send(key, operation = 'has_key'):
178179860b2SJed Brown        self.writeLogLine('__getitem__: Parent has value')
179179860b2SJed Brown        return self.send(key)
180179860b2SJed Brown      else:
181179860b2SJed Brown        self.writeLogLine('__getitem__: Checking parent type')
182179860b2SJed Brown        arg = self.send(key, operation = 'getType')
183179860b2SJed Brown        if not arg:
184179860b2SJed Brown          self.writeLogLine('__getitem__: Parent has no type')
185179860b2SJed Brown          arg = nargs.Arg(key)
186179860b2SJed Brown        try:
187179860b2SJed Brown          value = arg.getValue()
1885b6bfdb9SJed Brown        except AttributeError as e:
189179860b2SJed Brown          self.writeLogLine('__getitem__: Parent had invalid entry: '+str(e))
190179860b2SJed Brown          arg   = nargs.Arg(key)
191179860b2SJed Brown          value = arg.getValue()
192179860b2SJed Brown        self.writeLogLine('__getitem__: Setting parent value '+str(value))
193179860b2SJed Brown        self.send(key, value, operation = '__setitem__')
194179860b2SJed Brown        return value
195179860b2SJed Brown    else:
196179860b2SJed Brown      self.writeLogLine('__getitem__: Setting local type for '+key)
197179860b2SJed Brown      dict.__setitem__(self, key, nargs.Arg(key))
19831f4da61SSatish Balay      #self.save()
199179860b2SJed Brown    self.writeLogLine('__getitem__: Setting local value for '+key)
200179860b2SJed Brown    return dict.__getitem__(self, key).getValue()
201179860b2SJed Brown
202179860b2SJed Brown  def setType(self, key, value, forceLocal = 0):
203179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Sets the type for this key.
204179860b2SJed Brown       - If a value for the key already exists, it is converted to the new type'''
205179860b2SJed Brown    if not isinstance(value, nargs.Arg):
206179860b2SJed Brown      raise TypeError('An argument type must be a subclass of Arg')
207179860b2SJed Brown    value.setKey(key)
2083b049ba2SJed Brown    if forceLocal or self.parent is None or self.dict_has_key(key):
2093b049ba2SJed Brown      if self.dict_has_key(key):
210179860b2SJed Brown        v = dict.__getitem__(self, key)
211179860b2SJed Brown        if v.isValueSet():
212179860b2SJed Brown          try:
213179860b2SJed Brown            value.setValue(v.getValue())
214a5737712SSatish Balay          except TypeError:
2155b6bfdb9SJed Brown            print(value.__class__.__name__[3:])
2165b6bfdb9SJed Brown            print('-----------------------------------------------------------------------')
2175b6bfdb9SJed Brown            print('Warning! Incorrect argument type specified: -'+str(key)+'='+str(v.getValue())+' - expecting type '+value.__class__.__name__[3:]+'.')
2185b6bfdb9SJed Brown            print('-----------------------------------------------------------------------')
219a5737712SSatish Balay            pass
220179860b2SJed Brown      dict.__setitem__(self, key, value)
22131f4da61SSatish Balay      #self.save()
222179860b2SJed Brown    else:
223179860b2SJed Brown      return self.send(key, value)
224179860b2SJed Brown    return
225179860b2SJed Brown
226179860b2SJed Brown  def __setitem__(self, key, value):
227179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Sets the value of the Arg.'''
2283b049ba2SJed Brown    if not self.dict_has_key(key):
229179860b2SJed Brown      if not self.parent is None:
230179860b2SJed Brown        return self.send(key, value)
231179860b2SJed Brown      else:
232179860b2SJed Brown        dict.__setitem__(self, key, nargs.Arg(key))
233179860b2SJed Brown    dict.__getitem__(self, key).setValue(value)
234179860b2SJed Brown    self.writeLogLine('__setitem__: Set value for '+key+' to '+str(dict.__getitem__(self, key)))
23531f4da61SSatish Balay    #self.save()
236179860b2SJed Brown    return
237179860b2SJed Brown
238179860b2SJed Brown  def __delitem__(self, key):
239179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Deletes the Arg completely.'''
2403b049ba2SJed Brown    if self.dict_has_key(key):
241179860b2SJed Brown      dict.__delitem__(self, key)
24231f4da61SSatish Balay      #self.save()
243179860b2SJed Brown    elif not self.parent is None:
244179860b2SJed Brown      self.send(key)
245179860b2SJed Brown    return
246179860b2SJed Brown
247179860b2SJed Brown  def clear(self):
248179860b2SJed Brown    '''Clears both the local and parent dictionaries'''
249179860b2SJed Brown    if dict.__len__(self):
250179860b2SJed Brown      dict.clear(self)
25131f4da61SSatish Balay      #self.save()
252179860b2SJed Brown    if not self.parent is None:
253179860b2SJed Brown      self.send()
254179860b2SJed Brown    return
255179860b2SJed Brown
256179860b2SJed Brown  def __contains__(self, key):
257179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Then checks whether the value has been set'''
2583b049ba2SJed Brown    if self.dict_has_key(key):
259179860b2SJed Brown      if dict.__getitem__(self, key).isValueSet():
260179860b2SJed Brown        self.writeLogLine('has_key: Have value for '+key)
261179860b2SJed Brown      else:
262179860b2SJed Brown        self.writeLogLine('has_key: Do not have value for '+key)
263179860b2SJed Brown      return dict.__getitem__(self, key).isValueSet()
264179860b2SJed Brown    elif not self.parent is None:
265179860b2SJed Brown      return self.send(key)
266179860b2SJed Brown    return 0
267179860b2SJed Brown
268179860b2SJed Brown  def get(self, key, default=None):
2695b6bfdb9SJed Brown    if key in self:
270179860b2SJed Brown      return self.__getitem__(key)
271179860b2SJed Brown    else:
272179860b2SJed Brown      return default
273179860b2SJed Brown
274179860b2SJed Brown  def hasType(self, key):
275179860b2SJed Brown    '''Checks for the key locally, and if not found consults the parent. Then checks whether the type has been set'''
2763b049ba2SJed Brown    if self.dict_has_key(key):
277179860b2SJed Brown      return 1
278179860b2SJed Brown    elif not self.parent is None:
279179860b2SJed Brown      return self.send(key)
280179860b2SJed Brown    return 0
281179860b2SJed Brown
282179860b2SJed Brown  def items(self):
283179860b2SJed Brown    '''Return a list of all accessible items, as (key, value) pairs.'''
284179860b2SJed Brown    l = dict.items(self)
285179860b2SJed Brown    if not self.parent is None:
286179860b2SJed Brown      l.extend(self.send())
287179860b2SJed Brown    return l
288179860b2SJed Brown
289179860b2SJed Brown  def localitems(self):
290179860b2SJed Brown    '''Return a list of all the items stored locally, as (key, value) pairs.'''
291179860b2SJed Brown    return dict.items(self)
292179860b2SJed Brown
293179860b2SJed Brown  def keys(self):
294179860b2SJed Brown    '''Returns the list of keys in both the local and parent dictionaries'''
295bb3dd2f6SJed Brown    keyList = [key for key in dict.keys(self) if dict.__getitem__(self, key).isValueSet()]
296179860b2SJed Brown    if not self.parent is None:
297179860b2SJed Brown      keyList.extend(self.send())
298179860b2SJed Brown    return keyList
299179860b2SJed Brown
300179860b2SJed Brown  def types(self):
301179860b2SJed Brown    '''Returns the list of keys for which types are defined in both the local and parent dictionaries'''
302179860b2SJed Brown    keyList = dict.keys(self)
303179860b2SJed Brown    if not self.parent is None:
304179860b2SJed Brown      keyList.extend(self.send())
305179860b2SJed Brown    return keyList
306179860b2SJed Brown
307179860b2SJed Brown  def update(self, d):
308179860b2SJed Brown    '''Update the dictionary with the contents of d'''
309179860b2SJed Brown    for k in d:
310179860b2SJed Brown      self[k] = d[k]
311179860b2SJed Brown    return
312179860b2SJed Brown
313179860b2SJed Brown  def updateTypes(self, d):
314179860b2SJed Brown    '''Update types locally, which is equivalent to the dict.update() method'''
315179860b2SJed Brown    return dict.update(self, d)
316179860b2SJed Brown
317179860b2SJed Brown  def insertArg(self, key, value, arg):
318179860b2SJed Brown    '''Insert a (key, value) pair into the dictionary. If key is None, arg is put into the target list.'''
319179860b2SJed Brown    if not key is None:
320179860b2SJed Brown      self[key] = value
321179860b2SJed Brown    else:
322179860b2SJed Brown      if not self.target == ['default']:
323179860b2SJed Brown        self.target.append(arg)
324179860b2SJed Brown      else:
325179860b2SJed Brown        self.target = [arg]
326179860b2SJed Brown    return
327179860b2SJed Brown
328179860b2SJed Brown  def insertArgs(self, args):
329179860b2SJed Brown    '''Insert some text arguments into the dictionary (list and dictionaries are recognized)'''
330179860b2SJed Brown
331179860b2SJed Brown    if isinstance(args, list):
332179860b2SJed Brown      for arg in args:
333179860b2SJed Brown        (key, value) = nargs.Arg.parseArgument(arg)
334179860b2SJed Brown        self.insertArg(key, value, arg)
3356998aabbSVaclav Hapla    elif hasattr(args, 'keys'):
336179860b2SJed Brown      for key in args.keys():
337179860b2SJed Brown        if isinstance(args[key], str):
338179860b2SJed Brown          value = nargs.Arg.parseValue(args[key])
339179860b2SJed Brown        else:
340179860b2SJed Brown          value = args[key]
341179860b2SJed Brown        self.insertArg(key, value, None)
342179860b2SJed Brown    elif isinstance(args, str):
343179860b2SJed Brown        (key, value) = nargs.Arg.parseArgument(args)
344179860b2SJed Brown        self.insertArg(key, value, args)
345179860b2SJed Brown    return
346179860b2SJed Brown
347179860b2SJed Brown  def hasParent(self):
348179860b2SJed Brown    '''Return True if this RDict has a parent dictionary'''
349179860b2SJed Brown    return not self.parent is None
350179860b2SJed Brown
351179860b2SJed Brown  def getServerAddr(self, dir):
352179860b2SJed Brown    '''Read the server socket address (in pickled form) from a file, usually RDict.loc
353179860b2SJed Brown       - If we fail to connect to the server specified in the file, we spawn it using startServer()'''
354179860b2SJed Brown    filename = os.path.join(dir, self.addrFilename)
355179860b2SJed Brown    if not os.path.exists(filename):
356179860b2SJed Brown      self.startServer(filename)
357179860b2SJed Brown    if not os.path.exists(filename):
358179860b2SJed Brown      raise RuntimeError('Server address file does not exist: '+filename)
359179860b2SJed Brown    try:
360179860b2SJed Brown      f    = open(filename, 'r')
361492432c8SJed Brown      addr = pickle.load(f)
362179860b2SJed Brown      f.close()
363179860b2SJed Brown      return addr
3645b6bfdb9SJed Brown    except Exception as e:
365179860b2SJed Brown      self.writeLogLine('CLIENT: Exception during server address determination: '+str(e.__class__)+': '+str(e))
366179860b2SJed Brown    raise RuntimeError('Could not get server address in '+filename)
367179860b2SJed Brown
368179860b2SJed Brown  def writeServerAddr(self, server):
369179860b2SJed Brown    '''Write the server socket address (in pickled form) to a file, usually RDict.loc.'''
370c6ef1b5bSJed Brown    f = open(self.addrFilename, 'w')
371492432c8SJed Brown    pickle.dump(server.server_address, f)
372179860b2SJed Brown    f.close()
373179860b2SJed Brown    self.writeLogLine('SERVER: Wrote lock file '+os.path.abspath(self.addrFilename))
374179860b2SJed Brown    return
375179860b2SJed Brown
376179860b2SJed Brown  def startServer(self, addrFilename):
377179860b2SJed Brown    '''Spawn a new RDict server in the parent directory'''
378179860b2SJed Brown    import RDict # Need this to locate server script
379179860b2SJed Brown    import sys
380179860b2SJed Brown    import time
381becf0a19SJed Brown    import sysconfig
382179860b2SJed Brown
383179860b2SJed Brown    self.writeLogLine('CLIENT: Spawning a new server with lock file '+os.path.abspath(addrFilename))
384179860b2SJed Brown    if os.path.exists(addrFilename):
385179860b2SJed Brown      os.remove(addrFilename)
386179860b2SJed Brown    oldDir      = os.getcwd()
387179860b2SJed Brown    source      = os.path.join(os.path.dirname(os.path.abspath(sys.modules['RDict'].__file__)), 'RDict.py')
388becf0a19SJed Brown    interpreter = os.path.join(sysconfig.get_config_var('BINDIR'), sysconfig.get_config_var('PYTHON'))
389179860b2SJed Brown    if not os.path.isfile(interpreter):
390179860b2SJed Brown      interpreter = 'python'
391179860b2SJed Brown    os.chdir(os.path.dirname(addrFilename))
392179860b2SJed Brown    self.writeLogLine('CLIENT: Executing '+interpreter+' '+source+' server"')
393179860b2SJed Brown    try:
394179860b2SJed Brown      os.spawnvp(os.P_NOWAIT, interpreter, [interpreter, source, 'server'])
395179860b2SJed Brown    except:
396179860b2SJed Brown      self.writeLogLine('CLIENT: os.spawnvp failed.\n \
397179860b2SJed Brown      This is a typical problem on CYGWIN systems.  If you are using CYGWIN,\n \
398179860b2SJed Brown      you can fix this problem by running /bin/rebaseall.  If you do not have\n \
399179860b2SJed Brown      this program, you can install it with the CYGWIN installer in the package\n \
400179860b2SJed Brown      Rebase, under the category System.  You must run /bin/rebaseall after\n \
401179860b2SJed Brown      turning off all cygwin services -- in particular sshd, if any such services\n \
402179860b2SJed Brown      are running.  For more information about rebase, go to http://www.cygwin.com')
4035b6bfdb9SJed Brown      print('\n \
404179860b2SJed Brown      This is a typical problem on CYGWIN systems.  If you are using CYGWIN,\n \
405179860b2SJed Brown      you can fix this problem by running /bin/rebaseall.  If you do not have\n \
406179860b2SJed Brown      this program, you can install it with the CYGWIN installer in the package\n \
407179860b2SJed Brown      Rebase, under the category System.  You must run /bin/rebaseall after\n \
408179860b2SJed Brown      turning off all cygwin services -- in particular sshd, if any such services\n \
4095b6bfdb9SJed Brown      are running.  For more information about rebase, go to http://www.cygwin.com\n')
410179860b2SJed Brown      raise
411179860b2SJed Brown    os.chdir(oldDir)
412179860b2SJed Brown    timeout = 1
413179860b2SJed Brown    for i in range(10):
414179860b2SJed Brown      time.sleep(timeout)
415179860b2SJed Brown      timeout *= 2
416179860b2SJed Brown      if timeout > 100: timeout = 100
417179860b2SJed Brown      if os.path.exists(addrFilename): return
418179860b2SJed Brown    self.writeLogLine('CLIENT: Could not start server')
419179860b2SJed Brown    return
420179860b2SJed Brown
421179860b2SJed Brown  def connectParent(self, addr, dir):
422179860b2SJed Brown    '''Try to connect to a parent RDict server
423179860b2SJed Brown       - If addr and dir are both None, this operation fails
424179860b2SJed Brown       - If addr is None, check for an address file in dir'''
425179860b2SJed Brown    if addr is None:
426179860b2SJed Brown      if dir is None: return 0
427179860b2SJed Brown      addr = self.getServerAddr(dir)
428179860b2SJed Brown
429179860b2SJed Brown    import socket
430179860b2SJed Brown    import errno
431179860b2SJed Brown    connected = 0
432179860b2SJed Brown    s         = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
433179860b2SJed Brown    timeout   = 1
434179860b2SJed Brown    for i in range(10):
435179860b2SJed Brown      try:
436179860b2SJed Brown        self.writeLogLine('CLIENT: Trying to connect to '+str(addr))
437179860b2SJed Brown        s.connect(addr)
438179860b2SJed Brown        connected = 1
439179860b2SJed Brown        break
4405b6bfdb9SJed Brown      except socket.error as e:
441179860b2SJed Brown        self.writeLogLine('CLIENT: Failed to connect: '+str(e))
442179860b2SJed Brown        if e[0] == errno.ECONNREFUSED:
443179860b2SJed Brown          try:
444179860b2SJed Brown            import time
445179860b2SJed Brown            time.sleep(timeout)
446179860b2SJed Brown            timeout *= 2
447179860b2SJed Brown            if timeout > 100: timeout = 100
448179860b2SJed Brown          except KeyboardInterrupt:
449179860b2SJed Brown            break
450179860b2SJed Brown          # Try to spawn parent
451179860b2SJed Brown          if dir:
452179860b2SJed Brown            filename = os.path.join(dir, self.addrFilename)
453179860b2SJed Brown            if os.path.isfile(filename):
454179860b2SJed Brown              os.remove(filename)
455179860b2SJed Brown            self.startServer(filename)
4565b6bfdb9SJed Brown      except Exception as e:
457179860b2SJed Brown        self.writeLogLine('CLIENT: Failed to connect: '+str(e.__class__)+': '+str(e))
458179860b2SJed Brown    if not connected:
459179860b2SJed Brown      self.writeLogLine('CLIENT: Failed to connect to parent')
460179860b2SJed Brown      return 0
461179860b2SJed Brown    self.parent = s
462179860b2SJed Brown    self.writeLogLine('CLIENT: Connected to '+str(self.parent))
463179860b2SJed Brown    return 1
464179860b2SJed Brown
465179860b2SJed Brown  def sendPacket(self, s, packet, source = 'Unknown', isPickled = 0):
466179860b2SJed Brown    '''Pickle the input packet. Send first the size of the pickled string in 32-bit integer, and then the string itself'''
467179860b2SJed Brown    self.writeLogLine(source+': Sending packet '+str(packet))
468179860b2SJed Brown    if isPickled:
469179860b2SJed Brown      p = packet
470179860b2SJed Brown    else:
471492432c8SJed Brown      p = pickle.dumps(packet)
472179860b2SJed Brown    if hasattr(s, 'write'):
473179860b2SJed Brown      s.write(p)
474179860b2SJed Brown    else:
475179860b2SJed Brown      s.sendall(p)
476179860b2SJed Brown    self.writeLogLine(source+': Sent packet')
477179860b2SJed Brown    return
478179860b2SJed Brown
479179860b2SJed Brown  def recvPacket(self, s, source = 'Unknown'):
480179860b2SJed Brown    '''Receive first the size of the pickled string in a 32-bit integer, and then the string itself. Return the unpickled object'''
481179860b2SJed Brown    self.writeLogLine(source+': Receiving packet')
482179860b2SJed Brown    if hasattr(s, 'read'):
483179860b2SJed Brown      s.read(4)
484492432c8SJed Brown      value = pickle.load(s)
485179860b2SJed Brown    else:
486179860b2SJed Brown      objString = ''
487179860b2SJed Brown      while len(objString) < length:
488179860b2SJed Brown        objString += s.recv(length - len(objString))
489492432c8SJed Brown      value = pickle.loads(objString)
490179860b2SJed Brown    self.writeLogLine(source+': Received packet '+str(value))
491179860b2SJed Brown    return value
492179860b2SJed Brown
493179860b2SJed Brown  def send(self, key = None, value = None, operation = None):
494179860b2SJed Brown    '''Send a request to the parent'''
495179860b2SJed Brown    import inspect
496179860b2SJed Brown
497179860b2SJed Brown    objString = ''
498179860b2SJed Brown    for i in range(3):
499179860b2SJed Brown      try:
500179860b2SJed Brown        packet = []
501179860b2SJed Brown        if operation is None:
502179860b2SJed Brown          operation = inspect.stack()[1][3]
503179860b2SJed Brown        packet.append(operation)
504179860b2SJed Brown        if not key is None:
505179860b2SJed Brown          packet.append(key)
506179860b2SJed Brown          if not value is None:
507179860b2SJed Brown            packet.append(value)
508179860b2SJed Brown        self.sendPacket(self.parent, tuple(packet), source = 'CLIENT')
509179860b2SJed Brown        response = self.recvPacket(self.parent, source = 'CLIENT')
510179860b2SJed Brown        break
5115b6bfdb9SJed Brown      except IOError as e:
512179860b2SJed Brown        self.writeLogLine('CLIENT: IOError '+str(e))
513179860b2SJed Brown        if e.errno == 32:
514179860b2SJed Brown          self.connectParent(self.parentAddr, self.parentDirectory)
5155b6bfdb9SJed Brown      except Exception as e:
516179860b2SJed Brown        self.writeLogLine('CLIENT: Exception '+str(e)+' '+str(e.__class__))
517179860b2SJed Brown    try:
518179860b2SJed Brown      if isinstance(response, Exception):
519179860b2SJed Brown        self.writeLogLine('CLIENT: Got an exception '+str(response))
520179860b2SJed Brown        raise response
521179860b2SJed Brown      else:
522179860b2SJed Brown        self.writeLogLine('CLIENT: Received value '+str(response)+' '+str(type(response)))
523179860b2SJed Brown    except UnboundLocalError:
524179860b2SJed Brown      self.writeLogLine('CLIENT: Could not unpickle response')
525179860b2SJed Brown      response  = None
526179860b2SJed Brown    return response
527179860b2SJed Brown
528179860b2SJed Brown  def serve(self):
529179860b2SJed Brown    '''Start a server'''
530179860b2SJed Brown    import socket
5318f450857SSatish Balay    import SocketServer # novermin
532179860b2SJed Brown
533179860b2SJed Brown    if not useThreads:
534179860b2SJed Brown      raise RuntimeError('Cannot run a server if threads are disabled')
535179860b2SJed Brown
536179860b2SJed Brown    class ProcessHandler(SocketServer.StreamRequestHandler):
537179860b2SJed Brown      def handle(self):
538179860b2SJed Brown        import time
539179860b2SJed Brown
540179860b2SJed Brown        self.server.rdict.lastAccess = time.time()
541179860b2SJed Brown        self.server.rdict.writeLogLine('SERVER: Started new handler')
542179860b2SJed Brown        while 1:
543179860b2SJed Brown          try:
544179860b2SJed Brown            value = self.server.rdict.recvPacket(self.rfile, source = 'SERVER')
5455b6bfdb9SJed Brown          except EOFError as e:
546179860b2SJed Brown            self.server.rdict.writeLogLine('SERVER: EOFError receiving packet '+str(e)+' '+str(e.__class__))
547179860b2SJed Brown            return
5485b6bfdb9SJed Brown          except Exception as e:
549179860b2SJed Brown            self.server.rdict.writeLogLine('SERVER: Error receiving packet '+str(e)+' '+str(e.__class__))
550179860b2SJed Brown            self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER')
551179860b2SJed Brown            continue
552179860b2SJed Brown          if value[0] == 'stop': break
553179860b2SJed Brown          try:
554179860b2SJed Brown            response = getattr(self.server.rdict, value[0])(*value[1:])
5555b6bfdb9SJed Brown          except Exception as e:
556179860b2SJed Brown            self.server.rdict.writeLogLine('SERVER: Error executing operation '+str(e)+' '+str(e.__class__))
557179860b2SJed Brown            self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER')
558179860b2SJed Brown          else:
559179860b2SJed Brown            self.server.rdict.sendPacket(self.wfile, response, source = 'SERVER')
560179860b2SJed Brown        return
561179860b2SJed Brown
562179860b2SJed Brown    # check if server is running
563179860b2SJed Brown    if os.path.exists(self.addrFilename):
564179860b2SJed Brown      rdict     = RDict(parentDirectory = '.')
565179860b2SJed Brown      hasParent = rdict.hasParent()
566179860b2SJed Brown      del rdict
567179860b2SJed Brown      if hasParent:
568179860b2SJed Brown        self.writeLogLine('SERVER: Another server is already running')
569179860b2SJed Brown        raise RuntimeError('Server already running')
570179860b2SJed Brown
571179860b2SJed Brown    # Daemonize server
572179860b2SJed Brown    self.writeLogLine('SERVER: Daemonizing server')
573179860b2SJed Brown    if os.fork(): # Launch child
574179860b2SJed Brown      os._exit(0) # Kill off parent, so we are not a process group leader and get a new PID
575179860b2SJed Brown    os.setsid()   # Set session ID, so that we have no controlling terminal
576179860b2SJed Brown    # We choose to leave cwd at RDict.py: os.chdir('/') # Make sure root directory is not on a mounted drive
5775b6bfdb9SJed Brown    os.umask(0o77) # Fix creation mask
578179860b2SJed Brown    for i in range(3): # Crappy stopgap for closing descriptors
579179860b2SJed Brown      try:
580179860b2SJed Brown        os.close(i)
5815b6bfdb9SJed Brown      except OSError as e:
582179860b2SJed Brown        if e.errno != errno.EBADF:
583179860b2SJed Brown          raise RuntimeError('Could not close default descriptor '+str(i))
584179860b2SJed Brown
585179860b2SJed Brown    # wish there was a better way to get a usable socket
586179860b2SJed Brown    self.writeLogLine('SERVER: Establishing socket server')
587179860b2SJed Brown    basePort = 8000
588179860b2SJed Brown    flag     = 'nosocket'
589179860b2SJed Brown    p        = 1
590179860b2SJed Brown    while p < 1000 and flag == 'nosocket':
591179860b2SJed Brown      try:
592179860b2SJed Brown        server = SocketServer.ThreadingTCPServer((socket.gethostname(), basePort+p), ProcessHandler)
593179860b2SJed Brown        flag   = 'socket'
5945b6bfdb9SJed Brown      except Exception as e:
595179860b2SJed Brown        p = p + 1
596179860b2SJed Brown    if flag == 'nosocket':
597179860b2SJed Brown      p = 1
598179860b2SJed Brown      while p < 1000 and flag == 'nosocket':
599179860b2SJed Brown        try:
600179860b2SJed Brown          server = SocketServer.ThreadingTCPServer(('localhost', basePort+p), ProcessHandler)
601179860b2SJed Brown          flag   = 'socket'
6025b6bfdb9SJed Brown        except Exception as e:
603179860b2SJed Brown          p = p + 1
604179860b2SJed Brown    if flag == 'nosocket':
605179860b2SJed Brown      self.writeLogLine('SERVER: Could not established socket server on port '+str(basePort+p))
6065b6bfdb9SJed Brown      raise RuntimeError('Cannot get available socket')
607179860b2SJed Brown    self.writeLogLine('SERVER: Established socket server on port '+str(basePort+p))
608179860b2SJed Brown
609179860b2SJed Brown    self.isServer = 1
610179860b2SJed Brown    self.writeServerAddr(server)
611179860b2SJed Brown    self.serverShutdown(os.getpid())
612179860b2SJed Brown
613179860b2SJed Brown    server.rdict = self
614179860b2SJed Brown    self.writeLogLine('SERVER: Started server')
615179860b2SJed Brown    server.serve_forever()
616179860b2SJed Brown    return
617179860b2SJed Brown
618179860b2SJed Brown  def load(self):
619179860b2SJed Brown    '''Load the saved dictionary'''
620179860b2SJed Brown    if not self.parentDirectory is None and os.path.samefile(os.getcwd(), self.parentDirectory):
621179860b2SJed Brown      return
622179860b2SJed Brown    self.saveFilename = os.path.abspath(self.saveFilename)
623179860b2SJed Brown    if os.path.exists(self.saveFilename):
624179860b2SJed Brown      try:
625d2acaa84SFrancesco Ballarin        dbFile = open(self.saveFilename, 'rb')
626492432c8SJed Brown        data   = pickle.load(dbFile)
627179860b2SJed Brown        self.updateTypes(data)
628179860b2SJed Brown        dbFile.close()
629179860b2SJed Brown        self.writeLogLine('Loaded dictionary from '+self.saveFilename)
6305b6bfdb9SJed Brown      except Exception as e:
631179860b2SJed Brown        self.writeLogLine('Problem loading dictionary from '+self.saveFilename+'\n--> '+str(e))
632179860b2SJed Brown    else:
633179860b2SJed Brown      self.writeLogLine('No dictionary to load in this file: '+self.saveFilename)
634179860b2SJed Brown    return
635179860b2SJed Brown
636603e7a67SSatish Balay  def save(self, force = 1):
637179860b2SJed Brown    '''Save the dictionary after 5 seconds, ignoring all subsequent calls until the save
638179860b2SJed Brown       - Giving force = True will cause an immediate save'''
639179860b2SJed Brown    if self.readonly: return
64008eb64ffSMatthew G. Knepley    if force:
641179860b2SJed Brown      self.saveTimer = None
642179860b2SJed Brown      # This should be a critical section
643b8b3d021SJed Brown      dbFile = open(self.saveFilename, 'wb')
644bb3dd2f6SJed Brown      data   = dict([i for i in self.localitems() if not i[1].getTemporary()])
645492432c8SJed Brown      pickle.dump(data, dbFile)
646179860b2SJed Brown      dbFile.close()
647179860b2SJed Brown      self.writeLogLine('Saved local dictionary to '+os.path.abspath(self.saveFilename))
648179860b2SJed Brown    elif not self.saveTimer:
649179860b2SJed Brown      import threading
650179860b2SJed Brown      self.saveTimer = threading.Timer(5, self.save, [], {'force': 1})
651179860b2SJed Brown      self.saveTimer.setDaemon(1)
652179860b2SJed Brown      self.saveTimer.start()
653179860b2SJed Brown    return
654179860b2SJed Brown
655179860b2SJed Brown  def shutdown(self):
656179860b2SJed Brown    '''Shutdown the dictionary, writing out changes and notifying parent'''
657179860b2SJed Brown    if self.saveTimer:
658179860b2SJed Brown      self.saveTimer.cancel()
659179860b2SJed Brown      self.save(force = 1)
660179860b2SJed Brown    if self.isServer and os.path.isfile(self.addrFilename):
661179860b2SJed Brown      os.remove(self.addrFilename)
662179860b2SJed Brown    if not self.parent is None:
663179860b2SJed Brown      self.sendPacket(self.parent, self.stopCmd, isPickled = 1)
664179860b2SJed Brown      self.parent.close()
665179860b2SJed Brown      self.parent = None
666179860b2SJed Brown    self.writeLogLine('Shutting down')
667179860b2SJed Brown    self.logFile.close()
668179860b2SJed Brown    return
669179860b2SJed Brown
670179860b2SJed Brown  def serverShutdown(self, pid, delay = shutdownDelay):
671179860b2SJed Brown    if self.shutdownTimer is None:
672179860b2SJed Brown      import threading
673179860b2SJed Brown
674179860b2SJed Brown      self.shutdownTimer = threading.Timer(delay, self.serverShutdown, [pid], {'delay': 0})
675179860b2SJed Brown      self.shutdownTimer.setDaemon(1)
676179860b2SJed Brown      self.shutdownTimer.start()
677179860b2SJed Brown      self.writeLogLine('SERVER: Set shutdown timer for process '+str(pid)+' at '+str(delay)+' seconds')
678179860b2SJed Brown    else:
679179860b2SJed Brown      try:
680179860b2SJed Brown        import signal
681179860b2SJed Brown        import time
682179860b2SJed Brown
683179860b2SJed Brown        idleTime = time.time() - self.lastAccess
684179860b2SJed Brown        self.writeLogLine('SERVER: Last access '+str(self.lastAccess))
685179860b2SJed Brown        self.writeLogLine('SERVER: Idle time '+str(idleTime))
686179860b2SJed Brown        if idleTime < RDict.shutdownDelay:
687179860b2SJed Brown          self.writeLogLine('SERVER: Extending shutdown timer for '+str(pid)+' by '+str(RDict.shutdownDelay - idleTime)+' seconds')
688179860b2SJed Brown          self.shutdownTimer = None
689179860b2SJed Brown          self.serverShutdown(pid, RDict.shutdownDelay - idleTime)
690179860b2SJed Brown        else:
691179860b2SJed Brown          self.writeLogLine('SERVER: Killing server '+str(pid))
692179860b2SJed Brown          os.kill(pid, signal.SIGTERM)
6935b6bfdb9SJed Brown      except Exception as e:
694179860b2SJed Brown        self.writeLogLine('SERVER: Exception killing server: '+str(e))
695179860b2SJed Brown    return
696179860b2SJed Brown
697179860b2SJed Brownif __name__ ==  '__main__':
698179860b2SJed Brown  import sys
699179860b2SJed Brown  try:
700179860b2SJed Brown    if len(sys.argv) < 2:
7015b6bfdb9SJed Brown      print('RDict.py [server | client | clear | insert | remove] [parent]')
702179860b2SJed Brown    else:
703179860b2SJed Brown      action = sys.argv[1]
704179860b2SJed Brown      parent = None
705179860b2SJed Brown      if len(sys.argv) > 2:
706179860b2SJed Brown        if not sys.argv[2] == 'None': parent = sys.argv[2]
707179860b2SJed Brown      if action == 'server':
708179860b2SJed Brown        RDict(parentDirectory = parent).serve()
709179860b2SJed Brown      elif action == 'client':
7105b6bfdb9SJed Brown        print('Entries in server dictionary')
711179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
712179860b2SJed Brown        for key in rdict.types():
713179860b2SJed Brown          if not key.startswith('cacheKey') and not key.startswith('stamp-'):
7145b6bfdb9SJed Brown            print(str(key)+' '+str(rdict.getType(key)))
715179860b2SJed Brown      elif action == 'cacheClient':
7165b6bfdb9SJed Brown        print('Cache entries in server dictionary')
717179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
718179860b2SJed Brown        for key in rdict.types():
719179860b2SJed Brown          if key.startswith('cacheKey'):
7205b6bfdb9SJed Brown            print(str(key)+' '+str(rdict.getType(key)))
721179860b2SJed Brown      elif action == 'stampClient':
7225b6bfdb9SJed Brown        print('Stamp entries in server dictionary')
723179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
724179860b2SJed Brown        for key in rdict.types():
725179860b2SJed Brown          if key.startswith('stamp-'):
7265b6bfdb9SJed Brown            print(str(key)+' '+str(rdict.getType(key)))
727179860b2SJed Brown      elif action == 'clear':
7285b6bfdb9SJed Brown        print('Clearing all dictionaries')
729179860b2SJed Brown        RDict(parentDirectory = parent).clear()
730179860b2SJed Brown      elif action == 'insert':
731179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
732179860b2SJed Brown        rdict[sys.argv[3]] = sys.argv[4]
733179860b2SJed Brown      elif action == 'remove':
734179860b2SJed Brown        rdict = RDict(parentDirectory = parent)
735179860b2SJed Brown        del rdict[sys.argv[3]]
736179860b2SJed Brown      else:
737179860b2SJed Brown        sys.exit('Unknown action: '+action)
7385b6bfdb9SJed Brown  except Exception as e:
739179860b2SJed Brown    import traceback
7405b6bfdb9SJed Brown    print(traceback.print_tb(sys.exc_info()[2]))
741179860b2SJed Brown    sys.exit(str(e))
742179860b2SJed Brown  sys.exit(0)
743