Package Gnumed :: Package pycommon :: Module gmBackendListener |
[frames] | no frames]

# Source Code for Module Gnumed.pycommon.gmBackendListener

      1  """GNUmed database backend listener.
      2
      3  This module implements threaded listening for asynchronuous
      4  notifications from the database backend.
      5  """
      6  #=====================================================================
      7  __author__ = "H. Herb <hherb@gnumed.net>, K.Hilbert <karsten.hilbert@gmx.net>"
      8  __license__ = "GPL v2 or later"
      9
     10  import sys
     11  import time
     12  import threading
     13  import select
     14  import logging
     15
     16
     17  if __name__ == '__main__':
     18          sys.path.insert(0, '../../')
     19  from Gnumed.pycommon import gmDispatcher
     20  from Gnumed.pycommon import gmBorg
     21
     22
     23  _log = logging.getLogger('gm.db')
     24
     25
     26  signals2listen4 = [
     27          u'db_maintenance_warning',              # warns of impending maintenance and asks for disconnect
     28          u'db_maintenance_disconnect',   # announces a forced disconnect and disconnects
     29          u'gm_table_mod'                                 # sent for any (registered) table modification, payload contains details
     30  ]
     31
     32  #=====================================================================
     33 -class gmBackendListener(gmBorg.cBorg):
     34
     35 -        def __init__(self, conn=None, poll_interval=3):
     36
     37                  try:
     38                          self.already_inited
     39                          return
     40                  except AttributeError:
     41                          pass
     42
     43                  _log.info('starting backend notifications listener thread')
     44
     45                  # the listener thread will regularly try to acquire
     46                  # this lock, when it succeeds it will quit
     47                  self._quit_lock = threading.Lock()
     48                  # take the lock now so it cannot be taken by the worker
     49                  # thread until it is released in shutdown()
     50                  if not self._quit_lock.acquire(0):
     51                          _log.error('cannot acquire thread-quit lock, aborting')
     52                          raise EnvironmentError("cannot acquire thread-quit lock")
     53
     54                  self._conn = conn
     55                  self.backend_pid = self._conn.get_backend_pid()
     56                  _log.debug('connection has backend PID [%s]', self.backend_pid)
     57                  self._conn.set_isolation_level(0)               # autocommit mode = psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT
     58                  self._cursor = self._conn.cursor()
     59                  try:
     60                          self._conn_fd = self._conn.fileno()
     61                  except AttributeError:
     62                          self._conn_fd = self._cursor.fileno()
     63                  self._conn_lock = threading.Lock()              # lock for access to connection object
     64
     65                  self.__register_interests()
     66
     67                  # check for messages every 'poll_interval' seconds
     68                  self._poll_interval = poll_interval
     69                  self._listener_thread = None
     70                  self.__start_thread()
     71
     72                  self.already_inited = True
     73          #-------------------------------
     74          # public API
     75          #-------------------------------
     76 -        def shutdown(self):
     77                  if self._listener_thread is None:
     78                          self.__shutdown_connection()
     79                          return
     80
     81                  _log.info('stopping backend notifications listener thread')
     82                  self._quit_lock.release()
     83                  try:
     84                          # give the worker thread time to terminate
     85                          self._listener_thread.join(self._poll_interval+2.0)
     86                          try:
     87                                  if self._listener_thread.isAlive():
     88                                          _log.error('listener thread still alive after join()')
     89                                          _log.debug('active threads: %s' % threading.enumerate())
     90                          except:
     91                                  pass
     92                  except:
     93                          print sys.exc_info()
     94
     95                  self._listener_thread = None
     96
     97                  try:
     98                          self.__unregister_unspecific_notifications()
     99                  except:
    100                          _log.exception('unable to unregister unspecific notifications')
    101
    102                  self.__shutdown_connection()
    103
    104                  return
    105          #-------------------------------
    106          # event handlers
    107          #-------------------------------
    108          # internal helpers
    109          #-------------------------------
    110 -        def __register_interests(self):
    111                  # determine unspecific notifications
    112                  self.unspecific_notifications = signals2listen4
    113                  _log.info('configured unspecific notifications:')
    114                  _log.info('%s' % self.unspecific_notifications)
    115                  gmDispatcher.known_signals.extend(self.unspecific_notifications)
    116
    117                  # listen to unspecific notifications
    118                  self.__register_unspecific_notifications()
    119          #-------------------------------
    120 -        def __register_unspecific_notifications(self):
    121                  for sig in self.unspecific_notifications:
    122                          _log.info('starting to listen for [%s]' % sig)
    123                          cmd = 'LISTEN "%s"' % sig
    124                          self._conn_lock.acquire(1)
    125                          try:
    126                                  self._cursor.execute(cmd)
    127                          finally:
    128                                  self._conn_lock.release()
    129          #-------------------------------
    130 -        def __unregister_unspecific_notifications(self):
    131                  for sig in self.unspecific_notifications:
    132                          _log.info('stopping to listen for [%s]' % sig)
    133                          cmd = 'UNLISTEN "%s"' % sig
    134                          self._conn_lock.acquire(1)
    135                          try:
    136                                  self._cursor.execute(cmd)
    137                          finally:
    138                                  self._conn_lock.release()
    139          #-------------------------------
    140 -        def __shutdown_connection(self):
    141                  _log.debug('shutting down connection with backend PID [%s]', self.backend_pid)
    142                  self._conn_lock.acquire(1)
    143                  try:
    144                          self._conn.rollback()
    145                          self._conn.close()
    146                  except:
    147                          pass                            # connection can already be closed :-(
    148                  finally:
    149                          self._conn_lock.release()
    150          #-------------------------------
    151 -        def __start_thread(self):
    152                  if self._conn is None:
    153                          raise ValueError("no connection to backend available, useless to start thread")
    154
    155                  self._listener_thread = threading.Thread (
    156                          target = self._process_notifications,
    157                          name = self.__class__.__name__
    158                  )
    159                  self._listener_thread.setDaemon(True)
    160                  _log.info('starting listener thread')
    161                  self._listener_thread.start()
    162          #-------------------------------
    163          # the actual thread code
    164          #-------------------------------
    165 -        def _process_notifications(self):
    166
    167                  # loop until quitting
    168                  _have_quit_lock = None
    169                  while not _have_quit_lock:
    170
    171                          # quitting ?
    172                          if self._quit_lock.acquire(0):
    173                                  break
    174
    175                          # wait at most self._poll_interval for new data
    176                          self._conn_lock.acquire(1)
    177                          try:
    178                                  ready_input_sockets = select.select([self._conn_fd], [], [], self._poll_interval)[0]
    179                          finally:
    180                                  self._conn_lock.release()
    181
    182                          # any input available ?
    183                          if len(ready_input_sockets) == 0:
    184                                  # no, select.select() timed out
    185                                  # give others a chance to grab the conn lock (eg listen/unlisten)
    186                                  time.sleep(0.3)
    187                                  continue
    188
    189                          # data available, wait for it to fully arrive
    190                          self._conn_lock.acquire(1)
    191                          try:
    192                                  self._conn.poll()
    193                          finally:
    194                                  self._conn_lock.release()
    195
    196                          # any notifications ?
    197                          while len(self._conn.notifies) > 0:
    198                                  # if self._quit_lock can be acquired we may be in
    199                                  # __del__ in which case gmDispatcher is not
    200                                  # guaranteed to exist anymore
    201                                  if self._quit_lock.acquire(0):
    202                                          _have_quit_lock = 1
    203                                          break
    204
    205                                  self._conn_lock.acquire(1)
    206                                  try:
    207                                          notification = self._conn.notifies.pop()
    208                                  finally:
    209                                          self._conn_lock.release()
    210                                  # decode payload
    211                                  payload = notification.payload.split(u'::')
    212                                  operation = None
    213                                  table = None
    214                                  pk_column = None
    215                                  pk_row = None
    216                                  pk_identity = None
    217                                  for item in payload:
    218                                          if item.startswith(u'operation='):
    219                                                  operation = item.split(u'=')[1]
    220                                          if item.startswith(u'table='):
    221                                                  table = item.split(u'=')[1]
    222                                          if item.startswith(u'PK name='):
    223                                                  pk_column = item.split(u'=')[1]
    224                                          if item.startswith(u'row PK='):
    225                                                  pk_row = item.split(u'=')[1]
    226                                          if item.startswith(u'person PK='):
    227                                                  pk_identity = item.split(u'=')[1]
    228                                  # try sending intra-client signals:
    229                                  # 1) generic signal
    230                                  try:
    231                                          results = gmDispatcher.send (
    232                                                  signal = notification.channel,
    233                                                  originated_in_database = True,
    234                                                  listener_pid = self.backend_pid,
    235                                                  sending_backend_pid = notification.pid,
    236                                                  pk_identity = pk_identity,
    237                                                  operation = operation,
    238                                                  table = table,
    239                                                  pk_column = pk_column,
    240                                                  pk_row = pk_row
    241                                          )
    242                                  except:
    243                                          print "problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (notification.channel, notification.pid)
    244                                          print sys.exc_info()
    245                                  # 2) dynamically emulated old style table specific signals
    246                                  if table is not None:
    247                                          signal = u'%s_mod_db' % table
    248                                          try:
    249                                                  results = gmDispatcher.send (
    250                                                          signal = signal,
    251                                                          originated_in_database = True,
    252                                                          listener_pid = self.backend_pid,
    253                                                          sending_backend_pid = notification.pid,
    254                                                          pk_identity = pk_identity,
    255                                                          operation = operation,
    256                                                          table = table,
    257                                                          pk_column = pk_column,
    258                                                          pk_row = pk_row
    259                                                  )
    260                                          except:
    261                                                  print "problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (signal, notification.pid)
    262                                                  print sys.exc_info()
    263
    264                                  # there *may* be more pending notifications but
    265                                  # we don't care when quitting
    266                                  if self._quit_lock.acquire(0):
    267                                          _have_quit_lock = 1
    268                                          break
    269
    270                  # exit thread activity
    271                  return
    272  #=====================================================================
    273  # main
    274  #=====================================================================
    275  if __name__ == "__main__":
    276
    277          if len(sys.argv) < 2:
    278                  sys.exit()
    279
    280          if sys.argv[1] not in ['test', 'monitor']:
    281                  sys.exit()
    282
    283
    284          notifies = 0
    285
    286          from Gnumed.pycommon import gmPG2, gmI18N
    287          from Gnumed.business import gmPerson, gmPersonSearch
    288
    289          gmI18N.activate_locale()
    290          gmI18N.install_domain(domain='gnumed')
    291          #-------------------------------
    292 -        def run_test():
    293
    294                  #-------------------------------
    295                  def dummy(n):
    296                          return float(n)*n/float(1+n)
    297                  #-------------------------------
    298                  def OnPatientModified():
    299                          global notifies
    300                          notifies += 1
    301                          sys.stdout.flush()
    302                          print "\nBackend says: patient data has been modified (%s. notification)" % notifies
    303                  #-------------------------------
    304                  try:
    305                          n = int(sys.argv[2])
    306                  except:
    307                          print "You can set the number of iterations\nwith the second command line argument"
    308                          n = 100000
    309
    310                  # try loop without backend listener
    311                  print "Looping", n, "times through dummy function"
    312                  i = 0
    313                  t1 = time.time()
    314                  while i < n:
    315                          r = dummy(i)
    316                          i += 1
    317                  t2 = time.time()
    318                  t_nothreads = t2-t1
    319                  print "Without backend thread, it took", t_nothreads, "seconds"
    320
    321                  listener = gmBackendListener(conn = gmPG2.get_raw_connection())
    322
    323                  # now try with listener to measure impact
    324                  print "Now in a new shell connect psql to the"
    325                  print "database <gnumed_v9> on localhost, return"
    326                  print "here and hit <enter> to continue."
    327                  raw_input('hit <enter> when done starting psql')
    328                  print "You now have about 30 seconds to go"
    329                  print "to the psql shell and type"
    330                  print " notify patient_changed<enter>"
    331                  print "several times."
    332                  print "This should trigger our backend listening callback."
    333                  print "You can also try to stop the demo with Ctrl-C !"
    334
    335                  listener.register_callback('patient_changed', OnPatientModified)
    336
    337                  try:
    338                          counter = 0
    339                          while counter < 20:
    340                                  counter += 1
    341                                  time.sleep(1)
    342                                  sys.stdout.flush()
    343                                  print '.',
    344                          print "Looping",n,"times through dummy function"
    345                          i = 0
    346                          t1 = time.time()
    347                          while i < n:
    348                                  r = dummy(i)
    349                                  i += 1
    350                          t2 = time.time()
    351                          t_threaded = t2-t1
    352                          print "With backend thread, it took", t_threaded, "seconds"
    353                          print "Difference:", t_threaded-t_nothreads
    354                  except KeyboardInterrupt:
    355                          print "cancelled by user"
    356
    357                  listener.shutdown()
    358                  listener.unregister_callback('patient_changed', OnPatientModified)
    359          #-------------------------------
    360 -        def run_monitor():
    361
    362                  print "starting up backend notifications monitor"
    363
    364                  def monitoring_callback(*args, **kwargs):
    365                          try:
    366                                  kwargs['originated_in_database']
    367                                  print '==> got notification from database "%s":' % kwargs['signal']
    368                          except KeyError:
    369                                  print '==> received signal from client: "%s"' % kwargs['signal']
    370                          del kwargs['signal']
    371                          for key in kwargs.keys():
    372                                  print '    [%s]: %s' % (key, kwargs[key])
    373
    374                  gmDispatcher.connect(receiver = monitoring_callback)
    375
    376                  listener = gmBackendListener(conn = gmPG2.get_raw_connection())
    377                  print "listening for the following notifications:"
    378                  print "1) unspecific:"
    379                  for sig in listener.unspecific_notifications:
    380                          print '   - %s' % sig
    381
    382                  while True:
    383                          pat = gmPersonSearch.ask_for_patient()
    384                          if pat is None:
    385                                  break
    386                          print "found patient", pat
    387                          gmPerson.set_active_patient(patient=pat)
    388                          print "now waiting for notifications, hit <ENTER> to select another patient"
    389                          raw_input()
    390
    391                  print "cleanup"
    392                  listener.shutdown()
    393
    394                  print "shutting down backend notifications monitor"
    395
    396          #-------------------------------
    397          if sys.argv[1] == 'monitor':
    398                  run_monitor()
    399          else:
    400                  run_test()
    401
    402  #=====================================================================
    403