Package Gnumed :: Package pycommon :: Module gmPG2 |
[frames] | no frames]

# Source Code for Module Gnumed.pycommon.gmPG2

       1  """GNUmed PostgreSQL connection handling.
       2
       3  TODO: iterator/generator batch fetching:
       4          - http://groups-beta.google.com/group/comp.lang.python/msg/7ff516d7d9387dad
       5          - search Google for "Geneator/Iterator Nesting Problem - Any Ideas? 2.4"
       6
       7  winner:
       8  def resultset_functional_batchgenerator(cursor, size=100):
       9          for results in iter(lambda: cursor.fetchmany(size), []):
      10                  for rec in results:
      11                          yield rec
      12  """
      13  # =======================================================================
      14  __author__  = "K.Hilbert <Karsten.Hilbert@gmx.net>"
      15  __license__ = 'GPL v2 or later (details at http://www.gnu.org)'
      16
      17  # stdlib
      18  import time
      19  import sys
      20  import os
      21  import codecs
      22  import types
      23  import logging
      24  import datetime as pydt
      25  import re as regex
      26
      27
      28  # GNUmed
      29  if __name__ == '__main__':
      30          sys.path.insert(0, '../../')
      31  from Gnumed.pycommon import gmLoginInfo
      32  from Gnumed.pycommon import gmExceptions
      33  from Gnumed.pycommon import gmDateTime
      34  from Gnumed.pycommon import gmBorg
      35  from Gnumed.pycommon import gmI18N
      36  from Gnumed.pycommon import gmLog2
      37  from Gnumed.pycommon.gmTools import prompted_input, u_replacement_character
      38
      39  _log = logging.getLogger('gm.db')
      40
      41
      42  # 3rd party
      43  try:
      44          import psycopg2 as dbapi
      45  except ImportError:
      46          _log.exception("Python database adapter psycopg2 not found.")
      47          print "CRITICAL ERROR: Cannot find module psycopg2 for connecting to the database server."
      48          raise
      49
      50
      51  _log.info('psycopg2 version: %s' % dbapi.__version__)
      52  _log.info('PostgreSQL via DB-API module "%s": API level %s, thread safety %s, parameter style "%s"' % (dbapi, dbapi.apilevel, dbapi.threadsafety, dbapi.paramstyle))
      53  if not (float(dbapi.apilevel) >= 2.0):
      54          raise ImportError('gmPG2: supported DB-API level too low')
      55  if not (dbapi.threadsafety > 0):
      56          raise ImportError('gmPG2: lacking minimum thread safety in psycopg2')
      57  if not (dbapi.paramstyle == 'pyformat'):
      58          raise ImportError('gmPG2: lacking pyformat (%%(<name>)s style) placeholder support in psycopg2')
      59  try:
      60          dbapi.__version__.index('dt')
      61  except ValueError:
      62          raise ImportError('gmPG2: lacking datetime support in psycopg2')
      63  try:
      64          dbapi.__version__.index('ext')
      65  except ValueError:
      66          raise ImportError('gmPG2: lacking extensions support in psycopg2')
      67  try:
      68          dbapi.__version__.index('pq3')
      69  except ValueError:
      70          raise ImportError('gmPG2: lacking v3 backend protocol support in psycopg2')
      71
      72  import psycopg2.extras
      73  import psycopg2.extensions
      74  import psycopg2.pool
      75  import psycopg2.errorcodes as sql_error_codes
      76
      77  # =======================================================================
      78  _default_client_encoding = 'UTF8'
      79  _log.info('assuming default client encoding of [%s]' % _default_client_encoding)
      80
      81  # things timezone
      82  _default_client_timezone = None                 # default time zone for connections
      83  _sql_set_timezone = None
      84  _timestamp_template = "cast('%s' as timestamp with time zone)"          # MUST NOT be uniocde or else getquoted will not work
      85  FixedOffsetTimezone = dbapi.tz.FixedOffsetTimezone
      86
      87  _default_dsn = None
      88  _default_login = None
      89
      90  postgresql_version_string = None
      91  postgresql_version = None                       # accuracy: major.minor
      92
      93  __ro_conn_pool = None
      94
      95  auto_request_login_params = True
      96  # =======================================================================
      97  # global data
      98  # =======================================================================
      99
     100  known_schema_hashes = {
     101          0: 'not released, testing only',
     102          2: 'b09d50d7ed3f91ddf4c4ddb8ea507720',
     103          3: 'e73718eaf230d8f1d2d01afa8462e176',
     104          4: '4428ccf2e54c289136819e701bb095ea',
     105          5: '7e7b093af57aea48c288e76632a382e5',  # ... old (v1) style hashes
     106          6: '90e2026ac2efd236da9c8608b8685b2d',  # new (v2) style hashes ...
     107          7: '6c9f6d3981483f8e9433df99d1947b27',
     108          8: '89b13a7af83337c3aad153b717e52360',
     109          9: '641a9b2be3c378ffc2bb2f0b1c9f051d',
     110          10: '7ef42a8fb2bd929a2cdd0c63864b4e8a',
     111          11: '03042ae24f3f92877d986fb0a6184d76',
     112          12: '06183a6616db62257e22814007a8ed07',
     113          13: 'fab7c1ae408a6530c47f9b5111a0841e',
     114          14: 'e170d543f067d1ea60bfe9076b1560cf',
     115          15: '70012ff960b77ecdff4981c94b5b55b6',
     116          16: '0bcf44ca22c479b52976e5eda1de8161',
     117          17: '161428ee97a00e3bf56168c3a15b7b50',
     118          18: 'a0f9efcabdecfb4ddb6d8c0b69c02092'
     119  }
     120
     121  map_schema_hash2version = {
     122          'b09d50d7ed3f91ddf4c4ddb8ea507720': 2,
     123          'e73718eaf230d8f1d2d01afa8462e176': 3,
     124          '4428ccf2e54c289136819e701bb095ea': 4,
     125          '7e7b093af57aea48c288e76632a382e5': 5,
     126          '90e2026ac2efd236da9c8608b8685b2d': 6,
     127          '6c9f6d3981483f8e9433df99d1947b27': 7,
     128          '89b13a7af83337c3aad153b717e52360': 8,
     129          '641a9b2be3c378ffc2bb2f0b1c9f051d': 9,
     130          '7ef42a8fb2bd929a2cdd0c63864b4e8a': 10,
     131          '03042ae24f3f92877d986fb0a6184d76': 11,
     132          '06183a6616db62257e22814007a8ed07': 12,
     133          'fab7c1ae408a6530c47f9b5111a0841e': 13,
     134          'e170d543f067d1ea60bfe9076b1560cf': 14,
     135          '70012ff960b77ecdff4981c94b5b55b6': 15,
     136          '0bcf44ca22c479b52976e5eda1de8161': 16,
     137          '161428ee97a00e3bf56168c3a15b7b50': 17,
     138          'a0f9efcabdecfb4ddb6d8c0b69c02092': 18
     139  }
     140
     141  map_client_branch2required_db_version = {
     142          u'GIT tree': 0,
     143          u'0.3': 9,
     144          u'0.4': 10,
     145          u'0.5': 11,
     146          u'0.6': 12,
     147          u'0.7': 13,
     148          u'0.8': 14,
     149          u'0.9': 15,
     150          u'1.0': 16,             # intentional duplicate with 1.1
     151          u'1.1': 16,
     152          u'1.2': 17,
     153          u'1.3': 18
     154  }
     155
     156  # get columns and data types for a given table
     157  query_table_col_defs = u"""select
     158          cols.column_name,
     159          cols.udt_name
     160  from
     161          information_schema.columns cols
     162  where
     163          cols.table_schema = %s
     164                  and
     165          cols.table_name = %s
     166  order by
     167          cols.ordinal_position"""
     168
     169  query_table_attributes = u"""select
     170          cols.column_name
     171  from
     172          information_schema.columns cols
     173  where
     174          cols.table_schema = %s
     175                  and
     176          cols.table_name = %s
     177  order by
     178          cols.ordinal_position"""
     179
     180  # only works for single-column FKs but that's fine
     181  # needs gm-dbo, any-doc won't work
     182  SQL_foreign_key_name = u"""SELECT tc.constraint_schema, tc.constraint_name
     183  FROM
     184          information_schema.table_constraints tc
     185                  INNER JOIN information_schema.constraint_column_usage ccu USING (constraint_catalog, constraint_schema, constraint_name)
     186                          INNER JOIN information_schema.key_column_usage kcu USING (constraint_catalog, constraint_schema, constraint_name)
     187  WHERE
     188          tc.constraint_type = 'FOREIGN KEY'
     189                  AND
     190          kcu.table_schema = %(src_schema)s
     191                  AND
     192          kcu.table_name = %(src_tbl)s
     193                  AND
     194          kcu.column_name = %(src_col)s
     195                  AND
     196          ccu.table_schema = %(target_schema)s
     197                  AND
     198          ccu.table_name = %(target_tbl)s
     199                  AND
     200          ccu.column_name = %(target_col)s"""
     201
     202  # =======================================================================
     203  # module globals API
     204  # =======================================================================
     205 -def set_default_client_encoding(encoding = None):
     206          # check whether psycopg2 can handle this encoding
     207          if encoding not in psycopg2.extensions.encodings:
     208                  raise ValueError('psycopg2 does not know how to handle client (wire) encoding [%s]' % encoding)
     209          # check whether Python can handle this encoding
     210          py_enc = psycopg2.extensions.encodings[encoding]
     211          try:
     212                  codecs.lookup(py_enc)
     213          except LookupError:
     214                  _log.warning('<codecs> module can NOT handle encoding [psycopg2::<%s> -> Python::<%s>]' % (encoding, py_enc))
     215                  raise
     216          # FIXME: check encoding against the database
     217          # FIXME: - but we may not yet have access
     218          # FIXME: - psycopg2 will pull its encodings from the database eventually
     219          # it seems save to set it
     220          global _default_client_encoding
     221          _log.info('setting default client encoding from [%s] to [%s]' % (_default_client_encoding, str(encoding)))
     222          _default_client_encoding = encoding
     223          return True
     224  #---------------------------------------------------
     225 -def set_default_client_timezone(timezone = None):
     226
     227          # FIXME: use __validate
     228          global _default_client_timezone
     229          _log.info('setting default client time zone from [%s] to [%s]' % (_default_client_timezone, timezone))
     230          _default_client_timezone = timezone
     231
     232          global _sql_set_timezone
     233          _sql_set_timezone = u'set timezone to %s'
     234
     235          return True
     236  #---------------------------------------------------
     237 -def __validate_timezone(conn=None, timezone=None):
     238
     239          _log.debug(u'validating time zone [%s]', timezone)
     240
     241          cmd = u'set timezone to %(tz)s'
     242          args = {u'tz': timezone}
     243
     244          conn.commit()
     245          curs = conn.cursor()
     246          is_valid = False
     247          try:
     248                  curs.execute(cmd, args)
     249                  _log.info(u'time zone [%s] is settable', timezone)
     250                  # can we actually use it, though ?
     251                  cmd = u"""select '1920-01-19 23:00:00+01'::timestamp with time zone"""
     252                  try:
     253                          curs.execute(cmd)
     254                          curs.fetchone()
     255                          _log.info(u'time zone [%s] is usable', timezone)
     256                          is_valid = True
     257                  except:
     258                          _log.error('error using time zone [%s]', timezone)
     259          except dbapi.DataError:
     260                  _log.warning(u'time zone [%s] is not settable', timezone)
     261          except:
     262                  _log.error(u'failed to set time zone to [%s]', timezone)
     263                  _log.exception(u'')
     264
     265          curs.close()
     266          conn.rollback()
     267
     268          return is_valid
     269  #---------------------------------------------------
     270 -def __expand_timezone(conn=None, timezone=None):
     271          """some timezone defs are abbreviations so try to expand
     272          them because "set time zone" doesn't take abbreviations"""
     273
     274          cmd = u"""
     275  select distinct on (abbrev) name
     276  from pg_timezone_names
     277  where
     278          abbrev = %(tz)s and
     279          name ~ '^[^/]+/[^/]+$' and
     280          name !~ '^Etc/'
     281  """
     282          args = {u'tz': timezone}
     283
     284          conn.commit()
     285          curs = conn.cursor()
     286
     287          result = timezone
     288          try:
     289                  curs.execute(cmd, args)
     290                  rows = curs.fetchall()
     291                  if len(rows) > 0:
     292                          result = rows[0][0]
     293                          _log.debug(u'[%s] maps to [%s]', timezone, result)
     294          except:
     295                  _log.exception(u'cannot expand timezone abbreviation [%s]', timezone)
     296
     297          curs.close()
     298          conn.rollback()
     299
     300          return result
     301  #---------------------------------------------------
     302 -def __detect_client_timezone(conn=None):
     303          """This is run on the very first connection."""
     304
     305          # FIXME: check whether server.timezone is the same
     306          # FIXME: value as what we eventually detect
     307
     308          # we need gmDateTime to be initialized
     309          if gmDateTime.current_local_iso_numeric_timezone_string is None:
     310                  gmDateTime.init()
     311
     312          _log.debug('trying to detect timezone from system')
     313
     314          tz_candidates = []
     315          try:
     316                  tz = os.environ['TZ'].decode(gmI18N.get_encoding(), 'replace')
     317                  tz_candidates.append(tz)
     318                  expanded = __expand_timezone(conn = conn, timezone = tz)
     319                  if expanded != tz:
     320                          tz_candidates.append(expanded)
     321          except KeyError:
     322                  pass
     323
     324          tz_candidates.append(gmDateTime.current_local_timezone_name)
     325          expanded = __expand_timezone(conn = conn, timezone = gmDateTime.current_local_timezone_name)
     326          if expanded != gmDateTime.current_local_timezone_name:
     327                  tz_candidates.append(expanded)
     328
     329          _log.debug('candidates: %s', str(tz_candidates))
     330
     331          # find best among candidates
     332          global _default_client_timezone
     333          global _sql_set_timezone
     334          found = False
     335          for tz in tz_candidates:
     336                  if __validate_timezone(conn = conn, timezone = tz):
     337                          _default_client_timezone = tz
     338                          _sql_set_timezone = u'set timezone to %s'
     339                          found = True
     340                          break
     341
     342          if not found:
     343                  _default_client_timezone = gmDateTime.current_local_iso_numeric_timezone_string
     344                  _sql_set_timezone = u"set time zone interval %s hour to minute"
     345
     346          _log.info('client system time zone detected as equivalent to [%s]', _default_client_timezone)
     347  # =======================================================================
     348  # login API
     349  # =======================================================================
     350 -def __request_login_params_tui():
     351          """Text mode request of database login parameters"""
     352          import getpass
     353          login = gmLoginInfo.LoginInfo()
     354
     355          print "\nPlease enter the required login parameters:"
     356          try:
     357                  login.host = prompted_input(prompt = "host ('' = non-TCP/IP)", default = '')
     358                  login.database = prompted_input(prompt = "database", default = 'gnumed_v19')
     359                  login.user = prompted_input(prompt = "user name", default = '')
     360                  tmp = 'password for "%s" (not shown): ' % login.user
     361                  login.password = getpass.getpass(tmp)
     362                  login.port = prompted_input(prompt = "port", default = 5432)
     363          except KeyboardInterrupt:
     364                  _log.warning("user cancelled text mode login dialog")
     365                  print "user cancelled text mode login dialog"
     366                  raise gmExceptions.ConnectionError(_("Cannot connect to database without login information!"))
     367
     368          return login
     369  #---------------------------------------------------
     370 -def __request_login_params_gui_wx():
     371          """GUI (wx) input request for database login parameters.
     372
     373          Returns gmLoginInfo.LoginInfo object
     374          """
     375          import wx
     376          # OK, wxPython was already loaded. But has the main Application instance
     377          # been initialized yet ? if not, the exception will kick us out
     378          if wx.GetApp() is None:
     379                  raise gmExceptions.NoGuiError(_("The wxPython GUI framework hasn't been initialized yet!"))
     380
     381          # Let's launch the login dialog
     382          # if wx was not initialized /no main App loop, an exception should be raised anyway
     383          import gmAuthWidgets
     384          dlg = gmAuthWidgets.cLoginDialog(None, -1)
     385          dlg.ShowModal()
     386          login = dlg.panel.GetLoginInfo()
     387          dlg.Destroy()
     388
     389          #if user cancelled or something else went wrong, raise an exception
     390          if login is None:
     391                  raise gmExceptions.ConnectionError(_("Can't connect to database without login information!"))
     392
     393          return login
     394  #---------------------------------------------------
     395 -def request_login_params():
     396          """Request login parameters for database connection."""
     397          # do we auto-request parameters at all ?
     398          if not auto_request_login_params:
     399                  raise Exception('Cannot request login parameters.')
     400
     401          # are we inside X ?
     402          # (if we aren't wxGTK will crash hard at
     403          # C-level with "can't open Display")
     404          if os.environ.has_key('DISPLAY'):
     405                  # try wxPython GUI
     406                  try: return __request_login_params_gui_wx()
     407                  except: pass
     408
     409          # well, either we are on the console or
     410          # wxPython does not work, use text mode
     411          return __request_login_params_tui()
     412
     413  # =======================================================================
     414  # DSN API
     415  # -----------------------------------------------------------------------
     416 -def make_psycopg2_dsn(database=None, host=None, port=5432, user=None, password=None):
     417          dsn_parts = []
     418
     419          if (database is not None) and (database.strip() != ''):
     420                  dsn_parts.append('dbname=%s' % database)
     421
     422          if (host is not None) and (host.strip() != ''):
     423                  dsn_parts.append('host=%s' % host)
     424
     425          if (port is not None) and (str(port).strip() != ''):
     426                  dsn_parts.append('port=%s' % port)
     427
     428          if (user is not None) and (user.strip() != ''):
     429                  dsn_parts.append('user=%s' % user)
     430
     431          if (password is not None) and (password.strip() != ''):
     432                  dsn_parts.append('password=%s' % password)
     433
     434          dsn_parts.append('sslmode=prefer')
     435
     436          return ' '.join(dsn_parts)
     437  # ------------------------------------------------------
     438 -def get_default_login():
     439          # make sure we do have a login
     440          get_default_dsn()
     441          return _default_login
     442  # ------------------------------------------------------
     443 -def get_default_dsn():
     444          global _default_dsn
     445          if _default_dsn is not None:
     446                  return _default_dsn
     447
     448          login = request_login_params()
     449          set_default_login(login=login)
     450
     451          return _default_dsn
     452  # ------------------------------------------------------
     453 -def set_default_login(login=None):
     454          if login is None:
     455                  return False
     456
     457          if login.host is not None:
     458                  if login.host.strip() == u'':
     459                          login.host = None
     460
     461          global _default_login
     462          _default_login = login
     463          _log.info('setting default login from [%s] to [%s]' % (_default_login, login))
     464
     465          dsn = make_psycopg2_dsn(login.database, login.host, login.port, login.user, login.password)
     466
     467          global _default_dsn
     468          if _default_dsn is None:
     469                  old_dsn = u'None'
     470          else:
     471                  old_dsn = regex.sub(r'password=[^\s]+', u'password=%s' % u_replacement_character, _default_dsn)
     472          _log.info ('setting default DSN from [%s] to [%s]',
     473                  old_dsn,
     474                  regex.sub(r'password=[^\s]+', u'password=%s' % u_replacement_character, dsn)
     475          )
     476          _default_dsn = dsn
     477
     478          return True
     479  # =======================================================================
     480  # netadata API
     481  # =======================================================================
     482 -def database_schema_compatible(link_obj=None, version=None, verbose=True):
     483          expected_hash = known_schema_hashes[version]
     484          if version == 0:
     485                  args = {'ver': 9999}
     486          else:
     487                  args = {'ver': version}
     488          rows, idx = run_ro_queries (
     489                  link_obj = link_obj,
     490                  queries = [{
     491                          'cmd': u'select md5(gm.concat_table_structure(%(ver)s::integer)) as md5',
     492                          'args': args
     493                  }]
     494          )
     495          if rows[0]['md5'] != expected_hash:
     496                  _log.error('database schema version mismatch')
     497                  _log.error('expected: %s (%s)' % (version, expected_hash))
     498                  _log.error('detected: %s (%s)' % (get_schema_version(link_obj=link_obj), rows[0]['md5']))
     499                  if verbose:
     500                          _log.debug('schema dump follows:')
     501                          for line in get_schema_structure(link_obj = link_obj).split():
     502                                  _log.debug(line)
     503                          _log.debug('schema revision history dump follows:')
     504                          for line in get_schema_revision_history(link_obj = link_obj):
     505                                  _log.debug(u' - '.join(line))
     506                  return False
     507          _log.info('detected schema version [%s], hash [%s]' % (map_schema_hash2version[rows[0]['md5']], rows[0]['md5']))
     508          return True
     509  #------------------------------------------------------------------------
     510 -def get_schema_version(link_obj=None):
     511          rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': u'select md5(gm.concat_table_structure()) as md5'}])
     512          try:
     513                  return map_schema_hash2version[rows[0]['md5']]
     514          except KeyError:
     515                  return u'unknown database schema version, MD5 hash is [%s]' % rows[0]['md5']
     516  #------------------------------------------------------------------------
     517 -def get_schema_structure(link_obj=None):
     518          rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': u'select gm.concat_table_structure()'}])
     519          return rows[0][0]
     520  #------------------------------------------------------------------------
     521 -def get_schema_hash(link_obj=None):
     522          rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': u'select md5(gm.concat_table_structure()) as md5'}])
     523          return rows[0]['md5']
     524  #------------------------------------------------------------------------
     525 -def get_schema_revision_history(link_obj=None):
     526
     527          if table_exists(link_obj = link_obj, schema = 'gm', table = 'schema_revision'):
     528                  cmd = u"""
     529                          SELECT
     530                                  imported::text,
     531                                  version,
     532                                  filename
     533                          FROM gm.schema_revision
     534                          ORDER BY imported"""
     535          elif table_exists(link_obj = link_obj, schema = 'public', table = 'gm_schema_revision'):
     536                  cmd = u"""
     537                          SELECT
     538                                  imported::text,
     539                                  version,
     540                                  filename
     541                          FROM public.gm_schema_revision
     542                          ORDER BY imported"""
     543          else:
     544                  return []
     545
     546          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}])
     547          return rows
     548  #------------------------------------------------------------------------
     549 -def get_current_user():
     550          rows, idx = run_ro_queries(queries = [{'cmd': u'select CURRENT_USER'}])
     551          return rows[0][0]
     552  #------------------------------------------------------------------------
     553 -def get_foreign_keys2column(schema='public', table=None, column=None, link_obj=None):
     554          """Get the foreign keys pointing to schema.table.column.
     555
     556          Does not properly work with multi-column FKs.
     557          GNUmed doesn't use any, however.
     558          """
     559          cmd = u"""
     560  select
     561          %(schema)s as referenced_schema,
     562          %(tbl)s as referenced_table,
     563          %(col)s as referenced_column,
     564          pgc.confkey as referenced_column_list,
     565          pgc.conrelid::regclass as referencing_table,
     566          pgc.conkey as referencing_column_list,
     567          (select attname from pg_attribute where attnum = pgc.conkey[1] and attrelid = pgc.conrelid) as referencing_column
     568  from
     569          pg_constraint pgc
     570  where
     571          pgc.contype = 'f'
     572                  and
     573          pgc.confrelid = (
     574                  select oid from pg_class where relname = %(tbl)s and relnamespace = (
     575                          select oid from pg_namespace where nspname = %(schema)s
     576                   )
     577          )       and
     578          (
     579                  select attnum
     580                  from pg_attribute
     581                  where
     582                          attrelid = (select oid from pg_class where relname = %(tbl)s and relnamespace = (
     583                                  select oid from pg_namespace where nspname = %(schema)s
     584                          ))
     585                                  and
     586                          attname = %(col)s
     587          ) = any(pgc.confkey)
     588  """
     589
     590          args = {
     591                  'schema': schema,
     592                  'tbl': table,
     593                  'col': column
     594          }
     595
     596          rows, idx = run_ro_queries (
     597                  link_obj = link_obj,
     598                  queries = [
     599                          {'cmd': cmd, 'args': args}
     600                  ]
     601          )
     602
     603          return rows
     604
     605  #------------------------------------------------------------------------
     606 -def get_foreign_key_names(src_schema=None, src_table=None, src_column=None, target_schema=None, target_table=None, target_column=None, link_obj=None):
     607
     608          args = {
     609                  'src_schema': src_schema,
     610                  'src_tbl': src_table,
     611                  'src_col': src_column,
     612                  'target_schema': target_schema,
     613                  'target_tbl': target_table,
     614                  'target_col': target_column
     615          }
     616
     617          rows, idx = run_ro_queries (
     618                  link_obj = link_obj,
     619                  queries = [{'cmd': SQL_foreign_key_name, 'args': args}],
     620                  get_col_idx = False
     621          )
     622
     623          return rows
     624
     625  #------------------------------------------------------------------------
     626 -def get_child_tables(schema='public', table=None, link_obj=None):
     627          """Return child tables of <table>."""
     628          cmd = u"""
     629  select
     630          pgn.nspname as namespace,
     631          pgc.relname as table
     632  from
     633          pg_namespace pgn,
     634          pg_class pgc
     635  where
     636          pgc.relnamespace = pgn.oid
     637                  and
     638          pgc.oid in (
     639                  select inhrelid from pg_inherits where inhparent = (
     640                          select oid from pg_class where
     641                                  relnamespace = (select oid from pg_namespace where nspname = %(schema)s) and
     642                                  relname = %(table)s
     643                  )
     644          )"""
     645          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': {'schema': schema, 'table': table}}])
     646          return rows
     647  #------------------------------------------------------------------------
     648 -def schema_exists(link_obj=None, schema=u'gm'):
     649          cmd = u"""SELECT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = %(schema)s)"""
     650          args = {'schema': schema}
     651          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}])
     652          return rows[0][0]
     653  #------------------------------------------------------------------------
     654 -def table_exists(link_obj=None, schema=None, table=None):
     655          """Returns false, true."""
     656          cmd = u"""
     657  select exists (
     658          select 1 from information_schema.tables
     659          where
     660                  table_schema = %s and
     661                  table_name = %s and
     662                  table_type = 'BASE TABLE'
     663  )"""
     664          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': (schema, table)}])
     665          return rows[0][0]
     666  #------------------------------------------------------------------------
     667 -def get_col_indices(cursor = None):
     668          if cursor.description is None:
     669                  _log.error('no result description available: unused cursor or last query did not select rows')
     670                  return None
     671          col_indices = {}
     672          col_index = 0
     673          for col_desc in cursor.description:
     674                  col_name = col_desc[0]
     675                  # a query like "select 1,2;" will return two columns of the same name !
     676                  # hence adjust to that, note, however, that dict-style access won't work
     677                  # on results of such queries ...
     678                  if col_indices.has_key(col_name):
     679                          col_name = '%s_%s' % (col_name, col_index)
     680                  col_indices[col_name] = col_index
     681                  col_index += 1
     682
     683          return col_indices
     684  #------------------------------------------------------------------------
     685 -def get_col_defs(link_obj=None, schema='public', table=None):
     686          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_col_defs, 'args': (schema, table)}])
     687          col_names = []
     688          col_type = {}
     689          for row in rows:
     690                  col_names.append(row[0])
     691                  # map array types
     692                  if row[1].startswith('_'):
     693                          col_type[row[0]] = row[1][1:] + '[]'
     694                  else:
     695                          col_type[row[0]] = row[1]
     696          col_defs = []
     697          col_defs.append(col_names)
     698          col_defs.append(col_type)
     699          return col_defs
     700  #------------------------------------------------------------------------
     701 -def get_col_names(link_obj=None, schema='public', table=None):
     702          """Return column attributes of table"""
     703          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_attributes, 'args': (schema, table)}])
     704          cols = []
     705          for row in rows:
     706                  cols.append(row[0])
     707          return cols
     708
     709  #------------------------------------------------------------------------
     710  # i18n functions
     711  #------------------------------------------------------------------------
     712 -def export_translations_from_database(filename=None):
     713          tx_file = codecs.open(filename, 'wb', 'utf8')
     714          tx_file.write(u'-- GNUmed database string translations exported %s\n' % gmDateTime.pydt_now_here().strftime('%Y-%m-%d %H:%M'))
     715          tx_file.write(u'-- - contains translations for each of [%s]\n' % u', '.join(get_translation_languages()))
     716          tx_file.write(u'-- - user database language is set to [%s]\n\n' % get_current_user_language())
     717          tx_file.write(u'-- Please email this file to <gnumed-devel@gnu.org>.\n')
     718          tx_file.write(u'-- ----------------------------------------------------------------------------------------------\n\n')
     719          tx_file.write(u'set default_transaction_read_only to off;\n\n')
     720          tx_file.write(u"set client_encoding to 'utf-8';\n\n")
     721          tx_file.write(u'\\unset ON_ERROR_STOP\n\n')
     722
     723          cmd = u'SELECT lang, orig, trans FROM i18n.translations ORDER BY lang, orig'
     724          rows, idx = run_ro_queries(queries = [{'cmd': cmd}], get_col_idx = False)
     725          for row in rows:
     726                  line = u"select i18n.upd_tx(E'%s', E'%s', E'%s');\n" % (
     727                          row['lang'].replace("'", "\\'"),
     728                          row['orig'].replace("'", "\\'"),
     729                          row['trans'].replace("'", "\\'")
     730                  )
     731                  tx_file.write(line)
     732          tx_file.write(u'\n')
     733
     734          tx_file.write(u'\set ON_ERROR_STOP 1\n')
     735          tx_file.close()
     736
     737          return True
     738  #------------------------------------------------------------------------
     739 -def delete_translation_from_database(link_obj=None, language=None, original=None):
     740          cmd = u'DELETE FROM i18n.translations WHERE lang = %(lang)s AND orig = %(orig)s'
     741          args = {'lang': language, 'orig': original}
     742          run_rw_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}], return_data = False, end_tx = True)
     743          return True
     744
     745  #------------------------------------------------------------------------
     746 -def update_translation_in_database(language=None, original=None, translation=None):
     747          cmd = u'SELECT i18n.upd_tx(%(lang)s, %(orig)s, %(trans)s)'
     748          args = {'lang': language, 'orig': original, 'trans': translation}
     749          run_rw_queries(queries = [{'cmd': cmd, 'args': args}], return_data = False)
     750          return args
     751
     752  #------------------------------------------------------------------------
     753 -def get_translation_languages():
     754          rows, idx = run_ro_queries (
     755                  queries = [{'cmd': u'select distinct lang from i18n.translations'}]
     756          )
     757          return [ r[0] for r in rows ]
     758
     759  #------------------------------------------------------------------------
     760 -def get_database_translations(language=None, order_by=None):
     761
     762          args = {'lang': language}
     763          _log.debug('language [%s]', language)
     764
     765          if order_by is None:
     766                  order_by = u'ORDER BY %s' % order_by
     767          else:
     768                  order_by = u'ORDER BY lang, orig'
     769
     770          if language is None:
     771                  cmd = u"""
     772                  SELECT DISTINCT ON (orig, lang)
     773                          lang, orig, trans
     774                  FROM ((
     775
     776                          -- strings stored as translation keys whether translated or not
     777                          SELECT
     778                                  NULL as lang,
     779                                  ik.orig,
     780                                  NULL AS trans
     781                          FROM
     782                                  i18n.keys ik
     783
     784                  ) UNION ALL (
     785
     786                          -- already translated strings
     787                          SELECT
     788                                  it.lang,
     789                                  it.orig,
     790                                  it.trans
     791                          FROM
     792                                  i18n.translations it
     793
     794                  )) as translatable_strings
     795                  %s""" % order_by
     796          else:
     797                  cmd = u"""
     798                  SELECT DISTINCT ON (orig, lang)
     799                          lang, orig, trans
     800                  FROM ((
     801
     802                          -- strings stored as translation keys whether translated or not
     803                          SELECT
     804                                  %%(lang)s as lang,
     805                                  ik.orig,
     806                                  i18n._(ik.orig, %%(lang)s) AS trans
     807                          FROM
     808                                  i18n.keys ik
     809
     810                  ) UNION ALL (
     811
     812                          -- already translated strings
     813                          SELECT
     814                                  %%(lang)s as lang,
     815                                  it.orig,
     816                                  i18n._(it.orig, %%(lang)s) AS trans
     817                          FROM
     818                                  i18n.translations it
     819
     820                  )) AS translatable_strings
     821                  %s""" % order_by
     822
     823          rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': args}], get_col_idx = False)
     824
     825          if rows is None:
     826                  _log.error('no translatable strings found')
     827          else:
     828                  _log.debug('%s translatable strings found', len(rows))
     829
     830          return rows
     831
     832  #------------------------------------------------------------------------
     833 -def get_current_user_language():
     834          cmd = u'select i18n.get_curr_lang()'
     835          rows, idx = run_ro_queries(queries = [{'cmd': cmd}])
     836          return rows[0][0]
     837
     838  #------------------------------------------------------------------------
     839 -def set_user_language(user=None, language=None):
     840          """Set the user language in the database.
     841
     842          user = None: current db user
     843          language = None: unset
     844          """
     845          _log.info('setting database language for user [%s] to [%s]', user, language)
     846
     847          args = {
     848                  'usr': user,
     849                  'lang': language
     850          }
     851
     852          if language is None:
     853                  if user is None:
     854                          queries = [{'cmd': u'select i18n.unset_curr_lang()'}]
     855                  else:
     856                          queries = [{'cmd': u'select i18n.unset_curr_lang(%(usr)s)', 'args': args}]
     857                  queries.append({'cmd': u'select True'})
     858          else:
     859                  if user is None:
     860                          queries = [{'cmd': u'select i18n.set_curr_lang(%(lang)s)', 'args': args}]
     861                  else:
     862                          queries = [{'cmd': u'select i18n.set_curr_lang(%(lang)s, %(usr)s)', 'args': args}]
     863
     864          rows, idx = run_rw_queries(queries = queries, return_data = True)
     865
     866          if not rows[0][0]:
     867                  _log.error('cannot set database language to [%s] for user [%s]', language, user)
     868
     869          return rows[0][0]
     870  #------------------------------------------------------------------------
     871 -def force_user_language(language=None):
     872          """Set the user language in the database.
     873
     874          - regardless of whether there is any translation available.
     875          - only for the current user
     876          """
     877          _log.info('forcing database language for current db user to [%s]', language)
     878
     879          run_rw_queries(queries = [{
     880                  'cmd': u'select i18n.force_curr_lang(%(lang)s)',
     881                  'args': {'lang': language}
     882          }])
     883
     884  # =======================================================================
     885  # query runners and helpers
     886  # =======================================================================
     887 -def send_maintenance_notification():
     888          cmd = u'notify "db_maintenance_warning"'
     889          run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
     890  #------------------------------------------------------------------------
     891 -def send_maintenance_shutdown():
     892          cmd = u'notify "db_maintenance_disconnect"'
     893          run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
     894  #------------------------------------------------------------------------
     895 -def is_pg_interval(candidate=None):
     896          cmd = u'SELECT %(candidate)s::interval'
     897          try:
     898                  rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}])
     899                  return True
     900          except:
     901                  cmd = u'SELECT %(candidate)s::text::interval'
     902                  try:
     903                          rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}])
     904                          return True
     905                  except:
     906                          return False
     907
     908  #------------------------------------------------------------------------
     909 -def lock_row(link_obj=None, table=None, pk=None, exclusive=False):
     910          """Uses pg_advisory(_shared).
     911
     912          - locks stack and need one unlock per lock
     913          - same connection:
     914                  - all locks succeed
     915          - different connections:
     916                  - shared + shared succeed
     917                  - shared + exclusive fail
     918          """
     919          _log.debug('locking row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
     920          if exclusive:
     921                  cmd = u"""SELECT pg_try_advisory_lock('%s'::regclass::oid::int, %s)""" % (table, pk)
     922          else:
     923                  cmd = u"""SELECT pg_try_advisory_lock_shared('%s'::regclass::oid::int, %s)""" % (table, pk)
     924          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}], get_col_idx = False)
     925          if rows[0][0]:
     926                  return True
     927          _log.warning('cannot lock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
     928          return False
     929
     930  #------------------------------------------------------------------------
     931 -def unlock_row(link_obj=None, table=None, pk=None, exclusive=False):
     932          """Uses pg_advisory_unlock(_shared).
     933
     934          - each lock needs one unlock
     935          """
     936          _log.debug('trying to unlock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
     937          if exclusive:
     938                  cmd = u"SELECT pg_advisory_unlock('%s'::regclass::oid::int, %s)" % (table, pk)
     939          else:
     940                  cmd = u"SELECT pg_advisory_unlock_shared('%s'::regclass::oid::int, %s)" % (table, pk)
     941          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}], get_col_idx = False)
     942          if rows[0][0]:
     943                  return True
     944          _log.warning('cannot unlock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
     945          return False
     946
     947  #------------------------------------------------------------------------
     948 -def row_is_locked(table=None, pk=None):
     949          """Looks at pk_locks
     950
     951          - does not take into account locks other than 'advisory', however
     952          """
     953          cmd = u"""SELECT EXISTS (
     954                  SELECT 1 FROM pg_locks WHERE
     955                          classid = '%s'::regclass::oid::int
     956                                  AND
     957                          objid = %s
     958                                  AND
     959                          locktype = 'advisory'
     960          )""" % (table, pk)
     961          rows, idx = run_ro_queries(queries = [{'cmd': cmd}], get_col_idx = False)
     962          if rows[0][0]:
     963                  _log.debug('row is locked: [%s] [%s]', table, pk)
     964                  return True
     965          _log.debug('row is NOT locked: [%s] [%s]', table, pk)
     966          return False
     967
     968  #------------------------------------------------------------------------
     969 -def bytea2file(data_query=None, filename=None, chunk_size=0, data_size=None, data_size_query=None):
     970          outfile = file(filename, 'wb')
     971          result = bytea2file_object(data_query=data_query, file_obj=outfile, chunk_size=chunk_size, data_size=data_size, data_size_query=data_size_query)
     972          outfile.close()
     973          return result
     974  #------------------------------------------------------------------------
     975 -def bytea2file_object(data_query=None, file_obj=None, chunk_size=0, data_size=None, data_size_query=None):
     976          """Store data from a bytea field into a file.
     977
     978          <data_query>
     979          - dict {'cmd': ..., 'args': ...}
     980          - 'cmd' must be unicode containing "... substring(data from %(start)s for %(size)s) ..."
     981          - 'args' must be a dict
     982          - must return one row with one field of type bytea
     983          <file>
     984          - must be a file like Python object
     985          <data_size>
     986          - integer of the total size of the expected data or None
     987          <data_size_query>
     988          - dict {'cmd': ..., 'args': ...}
     989          - cmd must be unicode
     990          - must return one row with one field with the octet_length() of the data field
     991          - used only when <data_size> is None
     992          """
     993          if data_size == 0:
     994                  return True
     995
     996          # If the client sets an encoding other than the default we
     997          # will receive encoding-parsed data which isn't the binary
     998          # content we want. Hence we need to get our own connection.
     999          # It must be a read-write one so that we don't affect the
    1000          # encoding for other users of the shared read-only
    1001          # connections.
    1002          # Actually, encodings shouldn't be applied to binary data
    1003          # (eg. bytea types) in the first place but that is only
    1004          # reported to be fixed > v7.4.
    1005          # further tests reveal that at least on PG 8.0 this bug still
    1006          # manifests itself
    1007          conn = get_raw_connection(readonly=True)
    1008
    1009          if data_size is None:
    1010                  rows, idx = run_ro_queries(link_obj = conn, queries = [data_size_query])
    1011                  data_size = rows[0][0]
    1012                  if data_size in [None, 0]:
    1013                          conn.rollback()
    1014                          return True
    1015
    1016          max_chunk_size = 1024 * 1024 * 20                       # 20 MB, works for typical CR DICOMs
    1017          if chunk_size == 0:
    1018                  chunk_size = min(data_size, max_chunk_size)
    1019
    1020          _log.debug('expecting %s bytes of BYTEA data in chunks of %s bytes', data_size, chunk_size)
    1021
    1022          # Windoze sucks: it can't transfer objects of arbitrary size,
    1023          # anyways, we need to split the transfer,
    1024          # however, only possible if postgres >= 7.2
    1025          needed_chunks, remainder = divmod(data_size, chunk_size)
    1026          _log.debug('# of chunks: %s; remainder: %s bytes', needed_chunks, remainder)
    1027
    1028  #       # since we now require PG 9.1 we can disable this workaround:
    1029  #       # try setting "bytea_output"
    1030  #       # - fails if not necessary
    1031  #       # - succeeds if necessary
    1032  #       try:
    1033  #               run_ro_queries(link_obj = conn, queries = [{'cmd': u"set bytea_output to 'escape'"}])
    1034  #       except dbapi.ProgrammingError:
    1035  #               _log.debug('failed to set bytea_output to "escape", not necessary')
    1036
    1037          # retrieve chunks, skipped if data size < chunk size,
    1038          # does this not carry the danger of cutting up multi-byte escape sequences ?
    1039          # no, since bytea is binary,
    1040          # yes, since in bytea there are *some* escaped values, still
    1041          # no, since those are only escaped during *transfer*, not on-disk, hence
    1042          # only complete escape sequences are put on the wire
    1043          for chunk_id in range(needed_chunks):
    1044                  chunk_start = (chunk_id * chunk_size) + 1
    1045                  data_query['args']['start'] = chunk_start
    1046                  data_query['args']['size'] = chunk_size
    1047                  try:
    1048                          rows, idx = run_ro_queries(link_obj=conn, queries=[data_query])
    1049                  except:
    1050                          _log.error('cannot retrieve chunk [%s/%s], size [%s], try decreasing chunk size' % (chunk_id+1, needed_chunks, chunk_size))
    1051                          conn.rollback()
    1052                          raise
    1053                  # it would be a fatal error to see more than one result as ids are supposed to be unique
    1054                  file_obj.write(str(rows[0][0]))
    1055
    1056          # retrieve remainder
    1057          if remainder > 0:
    1058                  chunk_start = (needed_chunks * chunk_size) + 1
    1059                  data_query['args']['start'] = chunk_start
    1060                  data_query['args']['size'] = remainder
    1061                  try:
    1062                          rows, idx = run_ro_queries(link_obj=conn, queries=[data_query])
    1063                  except:
    1064                          _log.error('cannot retrieve remaining [%s] bytes' % remainder)
    1065                          conn.rollback()
    1066                          raise
    1067                  # it would be a fatal error to see more than one result as ids are supposed to be unique
    1068                  file_obj.write(str(rows[0][0]))
    1069
    1070          conn.rollback()
    1071          return True
    1072  #------------------------------------------------------------------------
    1073 -def file2bytea(query=None, filename=None, args=None, conn=None, file_md5=None):
    1074          """Store data from a file into a bytea field.
    1075
    1076          The query must:
    1077          - be in unicode
    1078          - contain a format spec identifying the row (eg a primary key)
    1079            matching <args> if it is an UPDATE
    1080          - contain a format spec " <field> = %(data)s::bytea"
    1081
    1082          The query CAN return the MD5 of the inserted data:
    1083                  RETURNING md5(<field>) AS md5
    1084          in which case it will compare it to the md5
    1085          of the file.
    1086          """
    1087          # read data from file
    1088          infile = file(filename, "rb")
    1089          data_as_byte_string = infile.read()
    1090          infile.close()
    1091          if args is None:
    1092                  args = {}
    1093          args['data'] = buffer(data_as_byte_string)
    1094          del(data_as_byte_string)
    1095
    1096          # insert the data
    1097          if conn is None:
    1098                  conn = get_raw_connection(readonly=False)
    1099                  close_conn = True
    1100          else:
    1101                  close_conn = False
    1102
    1103          rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = (file_md5 is not None))
    1104
    1105          success_status = True
    1106          if file_md5 is None:
    1107                  conn.commit()
    1108          else:
    1109                  db_md5 = rows[0]['md5']
    1110                  if file_md5 != db_md5:
    1111                          conn.rollback()
    1112                          success_status = False
    1113                          _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5)
    1114                  else:
    1115                          conn.commit()
    1116                          _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] <> [DB::%s]', file_md5, db_md5)
    1117
    1118          if close_conn:
    1119                  conn.close()
    1120
    1121          return success_status
    1122  #------------------------------------------------------------------------
    1123 -def sanitize_pg_regex(expression=None, escape_all=False):
    1124          """Escape input for use in a PostgreSQL regular expression.
    1125
    1126          If a fragment comes from user input and is to be used
    1127          as a regular expression we need to make sure it doesn't
    1128          contain invalid regex patterns such as unbalanced ('s.
    1129
    1130          <escape_all>
    1131                  True: try to escape *all* metacharacters
    1132                  False: only escape those which render the regex invalid
    1133          """
    1134          return expression.replace (
    1135                          '(', '\('
    1136                  ).replace (
    1137                          ')', '\)'
    1138                  ).replace (
    1139                          '[', '\['
    1140                  ).replace (
    1141                          '+', '\+'
    1142                  ).replace (
    1143                          '.', '\.'
    1144                  ).replace (
    1145                          '*', '\*'
    1146                  )
    1147                  #']', '\]',                     # not needed
    1148  #------------------------------------------------------------------------
    1149 -def run_ro_queries(link_obj=None, queries=None, verbose=False, return_data=True, get_col_idx=False):
    1150          """Run read-only queries.
    1151
    1152          <queries> must be a list of dicts:
    1153                  [
    1154                          {'cmd': <string>, 'args': <dict> or <tuple>},
    1155                          {...},
    1156                          ...
    1157                  ]
    1158          """
    1159          if isinstance(link_obj, dbapi._psycopg.cursor):
    1160                  curs = link_obj
    1161                  curs_close = __noop
    1162                  tx_rollback = __noop
    1163          elif isinstance(link_obj, dbapi._psycopg.connection):
    1164                  curs = link_obj.cursor()
    1165                  curs_close = curs.close
    1166                  tx_rollback = link_obj.rollback
    1167          elif link_obj is None:
    1168                  conn = get_connection(readonly=True, verbose=verbose)
    1169                  curs = conn.cursor()
    1170                  curs_close = curs.close
    1171                  tx_rollback = conn.rollback
    1172          else:
    1173                  raise ValueError('link_obj must be cursor, connection or None but not [%s]' % link_obj)
    1174
    1175          if verbose:
    1176                  _log.debug('cursor: %s', curs)
    1177
    1178          for query in queries:
    1179                  if type(query['cmd']) is not types.UnicodeType:
    1180                          print "run_ro_queries(): non-unicode query"
    1181                          print query['cmd']
    1182                  try:
    1183                          args = query['args']
    1184                  except KeyError:
    1185                          args = None
    1186                  try:
    1187                          curs.execute(query['cmd'], args)
    1188                          if verbose:
    1189                                  _log.debug('ran query: [%s]', curs.query)
    1190                                  if curs.statusmessage != u'':
    1191                                          _log.debug('PG status message: %s', curs.statusmessage)
    1192                                  _log.debug('cursor description: %s', str(curs.description))
    1193                  except dbapi.Error as pg_exc:
    1194                          _log.error('query failed: [%s]', curs.query)
    1195                          if curs.statusmessage != u'':
    1196                                  _log.error('PG status message: %s', curs.statusmessage)
    1197                          _log.error('PG error code: %s', pg_exc.pgcode)
    1198                          if pg_exc.pgerror is not None:
    1199                                  _log.error('PG error message: %s', pg_exc.pgerror.strip().strip(u'\n').strip().strip(u'\n'))
    1200                          try:
    1201                                  curs_close()
    1202                          except dbapi.InterfaceError:
    1203                                  _log.exception('cannot close cursor')
    1204                          tx_rollback()           # need to rollback so ABORT state isn't preserved in pooled conns
    1205                          if pg_exc.pgcode == sql_error_codes.INSUFFICIENT_PRIVILEGE:
    1206                                  details = u'Query: [%s]' % curs.query.strip().strip(u'\n').strip().strip(u'\n')
    1207                                  if curs.statusmessage != u'':
    1208                                          details = u'Status: %s\n%s' % (
    1209                                                  curs.statusmessage.strip().strip(u'\n').strip().strip(u'\n'),
    1210                                                  details
    1211                                          )
    1212                                  if pg_exc.pgerror is None:
    1213                                          msg = u'[%s]: %s' % (pg_exc.pgcode, pg_exc.pgerror)
    1214                                  else:
    1215                                          msg = u'[%s]: %s' % (pg_exc.pgcode, pg_exc.pgerror.strip().strip(u'\n').strip().strip(u'\n'))
    1216                                  raise gmExceptions.AccessDenied (
    1217                                          msg,
    1218                                          source = u'PostgreSQL',
    1219                                          code = pg_exc.pgcode,
    1220                                          details = details
    1221                                  )
    1222                          raise
    1223                  except:
    1224                          _log.error('query failed: [%s]', curs.query)
    1225                          if curs.statusmessage != u'':
    1226                                  _log.error('PG status message: %s', curs.statusmessage)
    1227                          try:
    1228                                  curs_close()
    1229                          except dbapi.InterfaceError:
    1230                                  _log.exception('cannot close cursor')
    1231                          tx_rollback()           # need to rollback so ABORT state isn't preserved in pooled conns
    1232                          raise
    1233
    1234          data = None
    1235          col_idx = None
    1236          if return_data:
    1237                  data = curs.fetchall()
    1238                  if verbose:
    1239                          _log.debug('last query returned [%s (%s)] rows', curs.rowcount, len(data))
    1240                          _log.debug('cursor description: %s', str(curs.description))
    1241                  if get_col_idx:
    1242                          col_idx = get_col_indices(curs)
    1243
    1244          curs_close()
    1245          tx_rollback()           # rollback just so that we don't stay IDLE IN TRANSACTION forever
    1246          return (data, col_idx)
    1247  #------------------------------------------------------------------------
    1248 -def run_rw_queries(link_obj=None, queries=None, end_tx=False, return_data=None, get_col_idx=False, verbose=False):
    1249          """Convenience function for running a transaction
    1250             that is supposed to get committed.
    1251
    1252          <link_obj>
    1253                  can be either:
    1254                  - a cursor
    1255                  - a connection
    1256
    1257          <queries>
    1258                  is a list of dicts [{'cmd': <string>, 'args': <dict> or <tuple>)
    1259                  to be executed as a single transaction, the last
    1260                  query may usefully return rows (such as a
    1261                  "SELECT currval('some_sequence')" statement)
    1262
    1263          <end_tx>
    1264                  - controls whether the transaction is finalized (eg.
    1265                    committed/rolled back) or not, this allows the
    1266                    call to run_rw_queries() to be part of a framing
    1267                    transaction
    1268                  - if link_obj is a connection then <end_tx> will
    1269                    default to False unless it is explicitly set to
    1270                    True which is taken to mean "yes, you do have full
    1271                    control over the transaction" in which case the
    1272                    transaction is properly finalized
    1273                  - if link_obj is a cursor we CANNOT finalize the
    1274                    transaction because we would need the connection for that
    1275                  - if link_obj is None <end_tx> will, of course, always be True
    1276
    1277          <return_data>
    1278                  - if true, the returned data will include the rows
    1279                    the last query selected
    1280                  - if false, it returns None instead
    1281
    1282          <get_col_idx>
    1283                  - if true, the returned data will include a dictionary
    1284                    mapping field names to column positions
    1285                  - if false, the returned data returns None instead
    1286
    1287          method result:
    1288                  - returns a tuple (data, idx)
    1289                  - <data>:
    1290                          * (None, None) if last query did not return rows
    1291                          * ("fetchall() result", <index>) if last query returned any rows
    1292                          * for <index> see <get_col_idx>
    1293          """
    1294          if isinstance(link_obj, dbapi._psycopg.cursor):
    1295                  conn_close = __noop
    1296                  conn_commit = __noop
    1297                  tx_rollback = __noop
    1298                  curs = link_obj
    1299                  curs_close = __noop
    1300          elif isinstance(link_obj, dbapi._psycopg.connection):
    1301                  conn_close = __noop
    1302                  if end_tx:
    1303                          conn_commit = link_obj.commit
    1304                          tx_rollback = link_obj.rollback
    1305                  else:
    1306                          conn_commit = __noop
    1307                          tx_rollback = __noop
    1308                  curs = link_obj.cursor()
    1309                  curs_close = curs.close
    1310          elif link_obj is None:
    1311                  conn = get_connection(readonly=False)
    1312                  conn_close = conn.close
    1313                  conn_commit = conn.commit
    1314                  tx_rollback = conn.rollback
    1315                  curs = conn.cursor()
    1316                  curs_close = curs.close
    1317          else:
    1318                  raise ValueError('link_obj must be cursor, connection or None and not [%s]' % link_obj)
    1319
    1320          for query in queries:
    1321                  if type(query['cmd']) is not types.UnicodeType:
    1322                          print "run_rw_queries(): non-unicode query"
    1323                          print query['cmd']
    1324                  try:
    1325                          args = query['args']
    1326                  except KeyError:
    1327                          args = None
    1328                  try:
    1329                          curs.execute(query['cmd'], args)
    1330                  except dbapi.Error as pg_exc:
    1331                          _log.error('RW query failed: [%s]', curs.query)
    1332                          if curs.statusmessage != u'':
    1333                                  _log.error('PG status message: %s', curs.statusmessage)
    1334                          _log.error('PG error code: %s', pg_exc.pgcode)
    1335                          if pg_exc.pgerror is not None:
    1336                                  _log.error('PG error message: %s', pg_exc.pgerror.strip().strip(u'\n').strip().strip(u'\n'))
    1337                          try:
    1338                                  curs_close()
    1339                                  tx_rollback()                   # just for good measure
    1340                                  conn_close()
    1341                          except dbapi.InterfaceError:
    1342                                  _log.exception('cannot cleanup')
    1343                          if pg_exc.pgcode == sql_error_codes.INSUFFICIENT_PRIVILEGE:
    1344                                  details = u'Query: [%s]' % curs.query.strip().strip(u'\n').strip().strip(u'\n')
    1345                                  if curs.statusmessage != u'':
    1346                                          details = u'Status: %s\n%s' % (
    1347                                                  curs.statusmessage.strip().strip(u'\n').strip().strip(u'\n'),
    1348                                                  details
    1349                                          )
    1350                                  if pg_exc.pgerror is None:
    1351                                          msg = u'[%s]: %s' % (pg_exc.pgcode, pg_exc.pgerror)
    1352                                  else:
    1353                                          msg = u'[%s]: %s' % (pg_exc.pgcode, pg_exc.pgerror.strip().strip(u'\n').strip().strip(u'\n'))
    1354                                  raise gmExceptions.AccessDenied (
    1355                                          msg,
    1356                                          source = u'PostgreSQL',
    1357                                          code = pg_exc.pgcode,
    1358                                          details = details
    1359                                  )
    1360                          raise
    1361                  except:
    1362                          _log.exception('error running RW query')
    1363                          gmLog2.log_stack_trace()
    1364                          try:
    1365                                  curs_close()
    1366                                  tx_rollback()
    1367                                  conn_close()
    1368                          except dbapi.InterfaceError:
    1369                                  _log.exception('cannot cleanup')
    1370                                  raise
    1371                          raise
    1372
    1373          data = None
    1374          col_idx = None
    1375          if return_data:
    1376                  try:
    1377                          data = curs.fetchall()
    1378                  except:
    1379                          _log.exception('error fetching data from RW query')
    1380                          gmLog2.log_stack_trace()
    1381                          try:
    1382                                  curs_close()
    1383                                  tx_rollback()
    1384                                  conn_close()
    1385                          except dbapi.InterfaceError:
    1386                                  _log.exception('cannot cleanup')
    1387                                  raise
    1388                          raise
    1389                  if get_col_idx:
    1390                          col_idx = get_col_indices(curs)
    1391
    1392          curs_close()
    1393          conn_commit()
    1394          conn_close()
    1395
    1396          return (data, col_idx)
    1397  #------------------------------------------------------------------------
    1398 -def run_insert(link_obj=None, schema=None, table=None, values=None, returning=None, end_tx=False, get_col_idx=False, verbose=False):
    1399          """Generates SQL for an INSERT query.
    1400
    1401          values: dict of values keyed by field to insert them into
    1402          """
    1403          if schema is None:
    1404                  schema = u'public'
    1405
    1406          fields = values.keys()          # that way val_snippets and fields really should end up in the same order
    1407          val_snippets = []
    1408          for field in fields:
    1409                  val_snippets.append(u'%%(%s)s' % field)
    1410
    1411          if returning is None:
    1412                  returning = u''
    1413                  return_data = False
    1414          else:
    1415                  returning = u'\n\tRETURNING\n\t\t%s' % u', '.join(returning)
    1416                  return_data = True
    1417
    1418          cmd = u"""\nINSERT INTO %s.%s (
    1419                  %s
    1420          ) VALUES (
    1421                  %s
    1422          )%s""" % (
    1423                  schema,
    1424                  table,
    1425                  u',\n\t\t'.join(fields),
    1426                  u',\n\t\t'.join(val_snippets),
    1427                  returning
    1428          )
    1429
    1430          _log.debug(u'running SQL: >>>%s<<<', cmd)
    1431
    1432          return run_rw_queries (
    1433                  link_obj = link_obj,
    1434                  queries = [{'cmd': cmd, 'args': values}],
    1435                  end_tx = end_tx,
    1436                  return_data = return_data,
    1437                  get_col_idx = get_col_idx,
    1438                  verbose = verbose
    1439          )
    1440  # =======================================================================
    1441  # connection handling API
    1442  # -----------------------------------------------------------------------
    1443 -class cConnectionPool(psycopg2.pool.PersistentConnectionPool):
    1444          """
    1445          GNUmed database connection pool.
    1446
    1447          Extends psycopg2's PersistentConnectionPool with
    1448          a custom _connect() function. Supports one connection
    1449          per thread - which also ties it to one particular DSN.
    1450          """
    1451          #--------------------------------------------------
    1452 -        def _connect(self, key=None):
    1453
    1454                  conn = get_raw_connection(dsn = self._kwargs['dsn'], verbose = self._kwargs['verbose'], readonly=True)
    1455
    1456                  conn.original_close = conn.close
    1457                  conn.close = _raise_exception_on_ro_conn_close
    1458
    1459                  if key is not None:
    1460                          self._used[key] = conn
    1461                          self._rused[id(conn)] = key
    1462                  else:
    1463                          self._pool.append(conn)
    1464
    1465                  return conn
    1466          #--------------------------------------------------
    1467 -        def shutdown(self):
    1468                  for conn_key in self._used.keys():
    1469                          _log.debug('closing pooled database connection, pool key: %s, backend PID: %s', conn_key, self._used[conn_key].get_backend_pid())
    1470                          self._used[conn_key].original_close()
    1471  # -----------------------------------------------------------------------
    1472 -def get_raw_connection(dsn=None, verbose=False, readonly=True):
    1473          """Get a raw, unadorned connection.
    1474
    1475          - this will not set any parameters such as encoding, timezone, datestyle
    1476          - the only requirement is a valid DSN
    1477          - hence it can be used for "service" connections
    1478            for verifying encodings etc
    1479          """
    1480          # FIXME: support verbose
    1481          if dsn is None:
    1482                  dsn = get_default_dsn()
    1483
    1484          if u'host=salaam.homeunix' in dsn:
    1485                  raise ValueError('The public database is not hosted by <salaam.homeunix.com> anymore.\n\nPlease point your configuration files to <publicdb.gnumed.de>.')
    1486
    1487          try:
    1488                  conn = dbapi.connect(dsn=dsn, connection_factory=psycopg2.extras.DictConnection)
    1489          except dbapi.OperationalError, e:
    1490
    1491                  t, v, tb = sys.exc_info()
    1492                  try:
    1493                          msg = e.args[0]
    1494                  except (AttributeError, IndexError, TypeError):
    1495                          raise
    1496
    1497                  msg = unicode(msg, gmI18N.get_encoding(), 'replace')
    1498
    1499                  if msg.find('fe_sendauth') != -1:
    1500                          raise cAuthenticationError, (dsn, msg), tb
    1501
    1502                  if regex.search('user ".*" does not exist', msg) is not None:
    1503                          raise cAuthenticationError, (dsn, msg), tb
    1504
    1505                  if msg.find('uthenti') != -1:
    1506                          raise cAuthenticationError, (dsn, msg), tb
    1507
    1508                  raise
    1509
    1510          _log.debug('new database connection, backend PID: %s, readonly: %s', conn.get_backend_pid(), readonly)
    1511
    1512          # do first-time stuff
    1513          global postgresql_version
    1514          if postgresql_version is None:
    1515                  curs = conn.cursor()
    1516                  curs.execute("""
    1517                          SELECT
    1518                                  substring(setting, E'^\\\\d{1,2}\\\\.\\\\d{1,2}')::numeric AS version
    1519                          FROM
    1520                                  pg_settings
    1521                          WHERE
    1522                                  name = 'server_version'
    1523                  """)
    1524                  postgresql_version = curs.fetchone()['version']
    1525                  _log.info('PostgreSQL version (numeric): %s' % postgresql_version)
    1526                  try:
    1527                          curs.execute("SELECT pg_size_pretty(pg_database_size(current_database()))")
    1528                          _log.info('database size: %s', curs.fetchone()[0])
    1529                  except:
    1530                          pass
    1531                  if verbose:
    1532                          __log_PG_settings(curs=curs)
    1533                  curs.close()
    1534                  conn.commit()
    1535
    1536          if _default_client_timezone is None:
    1537                  __detect_client_timezone(conn = conn)
    1538
    1539          curs = conn.cursor()
    1540
    1541          # set access mode
    1542          if readonly:
    1543                  _log.debug('access mode [READ ONLY]')
    1544                  cmd = 'set session characteristics as transaction READ ONLY'
    1545                  curs.execute(cmd)
    1546                  cmd = 'set default_transaction_read_only to on'
    1547                  curs.execute(cmd)
    1548          else:
    1549                  _log.debug('access mode [READ WRITE]')
    1550                  cmd = 'set session characteristics as transaction READ WRITE'
    1551                  curs.execute(cmd)
    1552                  cmd = 'set default_transaction_read_only to off'
    1553                  curs.execute(cmd)
    1554
    1555          curs.close()
    1556          conn.commit()
    1557
    1558          conn.is_decorated = False
    1559
    1560          return conn
    1561  # =======================================================================
    1562 -def get_connection(dsn=None, readonly=True, encoding=None, verbose=False, pooled=True):
    1563          """Get a new connection.
    1564
    1565          This assumes the locale system has been initialized
    1566          unless an encoding is specified.
    1567          """
    1568          # FIXME: support pooled on RW, too
    1569          # FIXME: for now, support the default DSN only
    1570          if pooled and readonly and (dsn is None):
    1571                  global __ro_conn_pool
    1572                  if __ro_conn_pool is None:
    1573                          __ro_conn_pool = cConnectionPool (
    1574                                  minconn = 1,
    1575                                  maxconn = 2,
    1576                                  dsn = dsn,
    1577                                  verbose = verbose
    1578                          )
    1579                  conn = __ro_conn_pool.getconn()
    1580          else:
    1581                  conn = get_raw_connection(dsn=dsn, verbose=verbose, readonly=False)
    1582
    1583          if conn.is_decorated:
    1584                  return conn
    1585
    1586          if encoding is None:
    1587                  encoding = _default_client_encoding
    1588          if encoding is None:
    1589                  encoding = gmI18N.get_encoding()
    1590                  _log.warning('client encoding not specified')
    1591                  _log.warning('the string encoding currently set in the active locale is used: [%s]' % encoding)
    1592                  _log.warning('for this to work properly the application MUST have called locale.setlocale() before')
    1593
    1594          # set connection properties
    1595          # - client encoding
    1596          try:
    1597                  conn.set_client_encoding(encoding)
    1598          except dbapi.OperationalError:
    1599                  t, v, tb = sys.exc_info()
    1600                  if str(v).find("can't set encoding to") != -1:
    1601                          raise cEncodingError, (encoding, v), tb
    1602                  raise
    1603
    1604          # - transaction isolation level
    1605          if readonly:
    1606                  # alter-database default, checked at connect, no need to set now
    1607                  iso_level = u'read committed'
    1608          else:
    1609                  conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE)
    1610                  iso_level = u'serializable'
    1611
    1612          _log.debug('client string encoding [%s], isolation level [%s], time zone [%s]', encoding, iso_level, _default_client_timezone)
    1613
    1614          curs = conn.cursor()
    1615
    1616          # - client time zone
    1617          curs.execute(_sql_set_timezone, [_default_client_timezone])
    1618
    1619          conn.commit()
    1620
    1621  #       # FIXME: remove this whole affair once either 9.0 is standard (Ubuntu 10 LTS is
    1622  #       # FIXME: PG 8.4, however!) or else when psycopg2 supports a workaround
    1623  #       #
    1624  #       # - bytea data format
    1625  #       # PG 9.0 switched to - by default - using "hex" rather than "escape",
    1626  #       # however, psycopg2's linked with a pre-9.0 libpq do assume "escape"
    1627  #       # as the transmission mode for bytea output,
    1628  #       # so try to set this setting back to "escape",
    1629  #       # if that's not possible the reason will be that PG < 9.0 does not support
    1630  #       # that setting - which also means we don't need it and can ignore the
    1631  #       # failure
    1632  #       cmd = "set bytea_output to 'escape'"
    1633  #       try:
    1634  #               curs.execute(cmd)
    1635  #       except dbapi.ProgrammingError:
    1636  #               _log.error('cannot set bytea_output format')
    1637
    1638          curs.close()
    1639          conn.commit()
    1640
    1641          conn.is_decorated = True
    1642
    1643          return conn
    1644  #-----------------------------------------------------------------------
    1645 -def shutdown():
    1646          if __ro_conn_pool is None:
    1647                  return
    1648          __ro_conn_pool.shutdown()
    1649  # ======================================================================
    1650  # internal helpers
    1651  #-----------------------------------------------------------------------
    1652 -def __noop():
    1653          pass
    1654  #-----------------------------------------------------------------------
    1655 -def _raise_exception_on_ro_conn_close():
    1656          raise TypeError(u'close() called on read-only connection')
    1657  #-----------------------------------------------------------------------
    1658 -def log_database_access(action=None):
    1659          run_insert (
    1660                  schema = u'gm',
    1661                  table = u'access_log',
    1662                  values = {u'user_action': action},
    1663                  end_tx = True
    1664          )
    1665  #-----------------------------------------------------------------------
    1666 -def sanity_check_time_skew(tolerance=60):
    1667          """Check server time and local time to be within
    1668          the given tolerance of each other.
    1669
    1670          tolerance: seconds
    1671          """
    1672          _log.debug('maximum skew tolerance (seconds): %s', tolerance)
    1673
    1674          cmd = u"SELECT now() at time zone 'UTC'"
    1675          conn = get_raw_connection(readonly=True)
    1676          curs = conn.cursor()
    1677
    1678          start = time.time()
    1679          rows, idx = run_ro_queries(link_obj = curs, queries = [{'cmd': cmd}])
    1680          end = time.time()
    1681          client_now_as_utc = pydt.datetime.utcnow()
    1682
    1683          curs.close()
    1684          conn.commit()
    1685
    1686          server_now_as_utc = rows[0][0]
    1687          query_duration = end - start
    1688          _log.info('server "now" (UTC): %s', server_now_as_utc)
    1689          _log.info('client "now" (UTC): %s', client_now_as_utc)
    1690          _log.debug('wire roundtrip (seconds): %s', query_duration)
    1691
    1692          if query_duration > tolerance:
    1693                  _log.error('useless to check client/server time skew, wire roundtrip > tolerance')
    1694                  return False
    1695
    1696          if server_now_as_utc > client_now_as_utc:
    1697                  real_skew = server_now_as_utc - client_now_as_utc
    1698          else:
    1699                  real_skew = client_now_as_utc - server_now_as_utc
    1700
    1701          _log.debug('client/server time skew: %s', real_skew)
    1702
    1703          if real_skew > pydt.timedelta(seconds = tolerance):
    1704                  _log.error('client/server time skew > tolerance')
    1705                  return False
    1706
    1707          return True
    1708  #-----------------------------------------------------------------------
    1709 -def sanity_check_database_settings():
    1710          """Checks database settings.
    1711
    1712          returns (status, message)
    1713          status:
    1714                  0: no problem
    1715                  1: non-fatal problem
    1716                  2: fatal problem
    1717          """
    1718          _log.debug('checking database settings')
    1719
    1720          conn = get_connection()
    1721
    1722          # - version string
    1723          global postgresql_version_string
    1724          if postgresql_version_string is None:
    1725                  curs = conn.cursor()
    1726                  curs.execute('SELECT version()')
    1727                  postgresql_version_string = curs.fetchone()['version']
    1728                  curs.close()
    1729                  _log.info('PostgreSQL version (string): "%s"' % postgresql_version_string)
    1730
    1731          options2check = {
    1732                  # setting: [expected value, risk, fatal?]
    1733                  u'allow_system_table_mods': [u'off', u'system breakage', False],
    1734                  u'check_function_bodies': [u'on', u'suboptimal error detection', False],
    1735                  u'datestyle': [u'ISO', u'faulty timestamp parsing', True],
    1736                  u'default_transaction_isolation': [u'read committed', u'faulty database reads', True],
    1737                  u'default_transaction_read_only': [u'on', u'accidental database writes', False],
    1738                  u'fsync': [u'on', u'data loss/corruption', True],
    1739                  u'full_page_writes': [u'on', u'data loss/corruption', False],
    1740                  u'lc_messages': [u'C', u'suboptimal error detection', False],
    1741                  u'password_encryption': [u'on', u'breach of confidentiality', False],
    1742                  #u'regex_flavor': [u'advanced', u'query breakage', False],                                      # 9.0 doesn't support this anymore, default now advanced anyway
    1743                  u'synchronous_commit': [u'on', u'data loss/corruption', False],
    1744                  u'sql_inheritance': [u'on', u'query breakage, data loss/corruption', True],
    1745                  u'ignore_checksum_failure': [u'off', u'data loss/corruption', False]            # starting with PG 9.3
    1746          }
    1747
    1748          from Gnumed.pycommon import gmCfg2
    1749          _cfg = gmCfg2.gmCfgData()
    1750          if _cfg.get(option = u'hipaa'):
    1751                  options2check[u'log_connections'] = [u'on', u'non-compliance with HIPAA', True]
    1752                  options2check[u'log_disconnections'] = [u'on', u'non-compliance with HIPAA', True]
    1753          else:
    1754                  options2check[u'log_connections'] = [u'on', u'non-compliance with HIPAA', None]
    1755                  options2check[u'log_disconnections'] = [u'on', u'non-compliance with HIPAA', None]
    1756
    1757          cmd = u"SELECT name, setting from pg_settings where name in %(settings)s"
    1758          rows, idx = run_ro_queries (
    1759                  link_obj = conn,
    1760                  queries = [{'cmd': cmd, 'args': {'settings': tuple(options2check.keys())}}],
    1761                  get_col_idx = False
    1762          )
    1763
    1764          found_error = False
    1765          found_problem = False
    1766          msg = []
    1767          for row in rows:
    1768                  option = row['name']
    1769                  value_found = row['setting']
    1770                  value_expected = options2check[option][0]
    1771                  risk = options2check[option][1]
    1772                  fatal_setting = options2check[option][2]
    1773                  if value_found != value_expected:
    1774                          if fatal_setting is True:
    1775                                  found_error = True
    1776                          elif fatal_setting is False:
    1777                                  found_problem = True
    1778                          elif fatal_setting is None:
    1779                                  pass
    1780                          else:
    1781                                  _log.error(options2check[option])
    1782                                  raise ValueError(u'invalid database configuration sanity check')
    1783                          msg.append(_(' option [%s]: %s') % (option, value_found))
    1784                          msg.append(_('  risk: %s') % risk)
    1785                          _log.warning('PG option [%s] set to [%s], expected [%s], risk: <%s>' % (option, value_found, value_expected, risk))
    1786
    1787          if found_error:
    1788                  return 2, u'\n'.join(msg)
    1789
    1790          if found_problem:
    1791                  return 1, u'\n'.join(msg)
    1792
    1793          return 0, u''
    1794  #------------------------------------------------------------------------
    1795 -def __log_PG_settings(curs=None):
    1796          # don't use any of the run_*()s since that might
    1797          # create a loop if we fail here
    1798          # FIXME: use pg_settings
    1799          try:
    1800                  curs.execute(u'show all')
    1801          except:
    1802                  _log.exception(u'cannot log PG settings (>>>show all<<< failed)')
    1803                  return False
    1804          settings = curs.fetchall()
    1805          if settings is None:
    1806                  _log.error(u'cannot log PG settings (>>>show all<<< did not return rows)')
    1807                  return False
    1808          for setting in settings:
    1809                  _log.debug(u'PG option [%s]: %s', setting[0], setting[1])
    1810          return True
    1811  # =======================================================================
    1812 -def extract_msg_from_pg_exception(exc=None):
    1813
    1814          try:
    1815                  msg = exc.args[0]
    1816          except (AttributeError, IndexError, TypeError):
    1817                  return u'cannot extract message from exception'
    1818
    1819          return unicode(msg, gmI18N.get_encoding(), 'replace')
    1820  # =======================================================================
    1821 -class cAuthenticationError(dbapi.OperationalError):
    1822
    1823 -        def __init__(self, dsn=None, prev_val=None):
    1824                  self.dsn = dsn
    1825                  self.prev_val = prev_val
    1826
    1827 -        def __str__(self):
    1828                  _log.warning('%s.__str__() called', self.__class__.__name__)
    1829                  tmp = u'PostgreSQL: %sDSN: %s' % (self.prev_val, self.dsn)
    1830                  _log.error(tmp)
    1831                  return tmp.encode(gmI18N.get_encoding(), 'replace')
    1832
    1833 -        def __unicode__(self):
    1834                  return u'PostgreSQL: %sDSN: %s' % (self.prev_val, self.dsn)
    1835
    1836  # =======================================================================
    1837  # custom psycopg2 extensions
    1838  # =======================================================================
    1839 -class cEncodingError(dbapi.OperationalError):
    1840
    1841 -        def __init__(self, encoding=None, prev_val=None):
    1842                  self.encoding = encoding
    1843                  self.prev_val = prev_val
    1844
    1845 -        def __str__(self):
    1846                  _log.warning('%s.__str__() called', self.__class__.__name__)
    1847                  return 'PostgreSQL: %s\nencoding: %s' % (self.prev_val.encode(gmI18N.get_encoding(), 'replace'), self.encoding.encode(gmI18N.get_encoding(), 'replace'))
    1848
    1849 -        def __unicode__(self):
    1850                  return u'PostgreSQL: %s\nencoding: %s' % (self.prev_val, self.encoding)
    1851
    1852  # -----------------------------------------------------------------------
    1853  # Python -> PostgreSQL
    1854  # -----------------------------------------------------------------------
    1855  # test when Squeeze (and thus psycopg2 2.2 becomes Stable
    1856 -class cAdapterPyDateTime(object):
    1857
    1858 -        def __init__(self, dt):
    1859                  if dt.tzinfo is None:
    1860                          raise ValueError(u'datetime.datetime instance is lacking a time zone: [%s]' % _timestamp_template % dt.isoformat())
    1861                  self.__dt = dt
    1862
    1863 -        def getquoted(self):
    1864                  return _timestamp_template % self.__dt.isoformat()
    1865
    1866  ## remove for 0.9
    1867  ## ----------------------------------------------------------------------
    1868  ##class cAdapterMxDateTime(object):
    1869  ##
    1870  ##      def __init__(self, dt):
    1871  ##              if dt.tz == '???':
    1872  ##                      _log.info('[%s]: no time zone string available in (%s), assuming local time zone', self.__class__.__name__, dt)
    1873  ##              self.__dt = dt
    1874  ##
    1875  ##      def getquoted(self):
    1876  ##              # under some locale settings the mx.DateTime ISO formatter
    1877  ##              # will insert "," into the ISO string,
    1878  ##              # while this is allowed per the ISO8601 spec PostgreSQL
    1879  ##              # cannot currently handle that,
    1880  ##              # so map those "," to "." to make things work:
    1881  ##              return mxDT.ISO.str(self.__dt).replace(',', '.')
    1882  ##
    1883  ## ----------------------------------------------------------------------
    1884  ## PostgreSQL -> Python
    1885  ## ----------------------------------------------------------------------
    1886
    1887  #=======================================================================
    1888  #  main
    1889  #-----------------------------------------------------------------------
    1890
    1891  # make sure psycopg2 knows how to handle unicode ...
    1892  # intended to become standard
    1893  # test when Squeeze (and thus psycopg2 2.2 becomes Stable
    1894  psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
    1895  psycopg2.extensions.register_type(psycopg2._psycopg.UNICODEARRAY)
    1896
    1897  # tell psycopg2 how to adapt datetime types with timestamps when locales are in use
    1898  # check in 0.9:
    1899  psycopg2.extensions.register_adapter(pydt.datetime, cAdapterPyDateTime)
    1900
    1901  # do NOT adapt *lists* to "... IN (*) ..." syntax because we want
    1902  # them adapted to "... ARRAY[]..." so we can support PG arrays
    1903
    1904  #=======================================================================
    1905  if __name__ == "__main__":
    1906
    1907          if len(sys.argv) < 2:
    1908                  sys.exit()
    1909
    1910          if sys.argv[1] != 'test':
    1911                  sys.exit()
    1912
    1913          logging.basicConfig(level=logging.DEBUG)
    1914          #--------------------------------------------------------------------
    1915 -        def test_file2bytea():
    1916                  run_rw_queries(queries = [
    1917                          {'cmd': u'create table test_bytea (data bytea)'}
    1918                  ])
    1919
    1920                  cmd = u'insert into test_bytea values (%(data)s::bytea)'
    1921                  try:
    1922                          file2bytea(query = cmd, filename = sys.argv[2])
    1923                  except:
    1924                          _log.exception('error')
    1925
    1926                  run_rw_queries(queries = [
    1927                          {'cmd': u'drop table test_bytea'}
    1928                  ])
    1929          #--------------------------------------------------------------------
    1930 -        def test_get_connection():
    1931                  print "testing get_connection()"
    1932
    1933                  dsn = 'foo'
    1934                  try:
    1935                          conn = get_connection(dsn=dsn)
    1936                  except dbapi.OperationalError, e:
    1937                          print "SUCCESS: get_connection(%s) failed as expected" % dsn
    1938                          t, v = sys.exc_info()[:2]
    1939                          print ' ', t
    1940                          print ' ', v
    1941
    1942                  dsn = 'dbname=gnumed_v9'
    1943                  try:
    1944                          conn = get_connection(dsn=dsn)
    1945                  except cAuthenticationError:
    1946                          print "SUCCESS: get_connection(%s) failed as expected" % dsn
    1947                          t, v = sys.exc_info()[:2]
    1948                          print ' ', t
    1949                          print ' ', v
    1950
    1951                  dsn = 'dbname=gnumed_v9 user=abc'
    1952                  try:
    1953                          conn = get_connection(dsn=dsn)
    1954                  except cAuthenticationError:
    1955                          print "SUCCESS: get_connection(%s) failed as expected" % dsn
    1956                          t, v = sys.exc_info()[:2]
    1957                          print ' ', t
    1958                          print ' ', v
    1959
    1960                  dsn = 'dbname=gnumed_v9 user=any-doc'
    1961                  try:
    1962                          conn = get_connection(dsn=dsn)
    1963                  except cAuthenticationError:
    1964                          print "SUCCESS: get_connection(%s) failed as expected" % dsn
    1965                          t, v = sys.exc_info()[:2]
    1966                          print ' ', t
    1967                          print ' ', v
    1968
    1969                  dsn = 'dbname=gnumed_v9 user=any-doc password=abc'
    1970                  try:
    1971                          conn = get_connection(dsn=dsn)
    1972                  except cAuthenticationError:
    1973                          print "SUCCESS: get_connection(%s) failed as expected" % dsn
    1974                          t, v = sys.exc_info()[:2]
    1975                          print ' ', t
    1976                          print ' ', v
    1977
    1978                  dsn = 'dbname=gnumed_v9 user=any-doc password=any-doc'
    1979                  conn = get_connection(dsn=dsn, readonly=True)
    1980
    1981                  dsn = 'dbname=gnumed_v9 user=any-doc password=any-doc'
    1982                  conn = get_connection(dsn=dsn, readonly=False)
    1983
    1984                  dsn = 'dbname=gnumed_v9 user=any-doc password=any-doc'
    1985                  encoding = 'foo'
    1986                  try:
    1987                          conn = get_connection(dsn=dsn, encoding=encoding)
    1988                  except cEncodingError:
    1989                          print "SUCCESS: get_connection(%s, %s) failed as expected" % (dsn, encoding)
    1990                          t, v = sys.exc_info()[:2]
    1991                          print ' ', t
    1992                          print ' ', v
    1993          #--------------------------------------------------------------------
    1994 -        def test_exceptions():
    1995                  print "testing exceptions"
    1996
    1997                  try:
    1998                          raise cAuthenticationError('no dsn', 'no previous exception')
    1999                  except cAuthenticationError:
    2000                          t, v, tb = sys.exc_info()
    2001                          print t
    2002                          print v
    2003                          print tb
    2004
    2005                  try:
    2006                          raise cEncodingError('no dsn', 'no previous exception')
    2007                  except cEncodingError:
    2008                          t, v, tb = sys.exc_info()
    2009                          print t
    2010                          print v
    2011                          print tb
    2012          #--------------------------------------------------------------------
    2013 -        def test_ro_queries():
    2014                  print "testing run_ro_queries()"
    2015
    2016                  dsn = 'dbname=gnumed_v9 user=any-doc password=any-doc'
    2017                  conn = get_connection(dsn, readonly=True)
    2018
    2019                  data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': u'SELECT version()'}], return_data=True, get_col_idx=True, verbose=True)
    2020                  print data
    2021                  print idx
    2022                  data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': u'SELECT 1'}], return_data=True, get_col_idx=True)
    2023                  print data
    2024                  print idx
    2025
    2026                  curs = conn.cursor()
    2027
    2028                  data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': u'SELECT version()'}], return_data=True, get_col_idx=True, verbose=True)
    2029                  print data
    2030                  print idx
    2031
    2032                  data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': u'SELECT 1'}], return_data=True, get_col_idx=True, verbose=True)
    2033                  print data
    2034                  print idx
    2035
    2036                  try:
    2037                          data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': u'selec 1'}], return_data=True, get_col_idx=True, verbose=True)
    2038                          print data
    2039                          print idx
    2040                  except psycopg2.ProgrammingError:
    2041                          print 'SUCCESS: run_ro_queries("selec 1") failed as expected'
    2042                          t, v = sys.exc_info()[:2]
    2043                          print ' ', t
    2044                          print ' ', v
    2045
    2046                  curs.close()
    2047          #--------------------------------------------------------------------
    2048 -        def test_request_dsn():
    2049                  conn = get_connection()
    2050                  print conn
    2051                  conn.close()
    2052          #--------------------------------------------------------------------
    2053 -        def test_set_encoding():
    2054                  print "testing set_default_client_encoding()"
    2055
    2056                  enc = 'foo'
    2057                  try:
    2058                          set_default_client_encoding(enc)
    2059                          print "SUCCESS: encoding [%s] worked" % enc
    2060                  except ValueError:
    2061                          print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc
    2062                          t, v = sys.exc_info()[:2]
    2063                          print ' ', t
    2064                          print ' ', v
    2065
    2066                  enc = ''
    2067                  try:
    2068                          set_default_client_encoding(enc)
    2069                          print "SUCCESS: encoding [%s] worked" % enc
    2070                  except ValueError:
    2071                          print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc
    2072                          t, v = sys.exc_info()[:2]
    2073                          print ' ', t
    2074                          print ' ', v
    2075
    2076                  enc = 'latin1'
    2077                  try:
    2078                          set_default_client_encoding(enc)
    2079                          print "SUCCESS: encoding [%s] worked" % enc
    2080                  except ValueError:
    2081                          print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc
    2082                          t, v = sys.exc_info()[:2]
    2083                          print ' ', t
    2084                          print ' ', v
    2085
    2086                  enc = 'utf8'
    2087                  try:
    2088                          set_default_client_encoding(enc)
    2089                          print "SUCCESS: encoding [%s] worked" % enc
    2090                  except ValueError:
    2091                          print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc
    2092                          t, v = sys.exc_info()[:2]
    2093                          print ' ', t
    2094                          print ' ', v
    2095
    2096                  enc = 'unicode'
    2097                  try:
    2098                          set_default_client_encoding(enc)
    2099                          print "SUCCESS: encoding [%s] worked" % enc
    2100                  except ValueError:
    2101                          print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc
    2102                          t, v = sys.exc_info()[:2]
    2103                          print ' ', t
    2104                          print ' ', v
    2105
    2106                  enc = 'UNICODE'
    2107                  try:
    2108                          set_default_client_encoding(enc)
    2109                          print "SUCCESS: encoding [%s] worked" % enc
    2110                  except ValueError:
    2111                          print "SUCCESS: set_default_client_encoding(%s) failed as expected" % enc
    2112                          t, v = sys.exc_info()[:2]
    2113                          print ' ', t
    2114                          print ' ', v
    2115          #--------------------------------------------------------------------
    2116 -        def test_connection_pool():
    2117                  dsn = get_default_dsn()
    2118                  pool = cConnectionPool(minconn=1, maxconn=2, dsn=None, verbose=False)
    2119                  print pool
    2120                  print pool.getconn()
    2121                  print pool.getconn()
    2122                  print pool.getconn()
    2123                  print type(pool.getconn())
    2124          #--------------------------------------------------------------------
    2125 -        def test_list_args():
    2126                  dsn = get_default_dsn()
    2127                  conn = get_connection(dsn, readonly=True)
    2128                  curs = conn.cursor()
    2129                  curs.execute('SELECT * from clin.clin_narrative where narrative = %s', ['a'])
    2130          #--------------------------------------------------------------------
    2131 -        def test_sanitize_pg_regex():
    2132                  tests = [
    2133                          ['(', '\\(']
    2134                          , ['[', '\\[']
    2135                          , [')', '\\)']
    2136                  ]
    2137                  for test in tests:
    2138                          result = sanitize_pg_regex(test[0])
    2139                          if result != test[1]:
    2140                                  print 'ERROR: sanitize_pg_regex(%s) returned "%s", expected "%s"' % (test[0], result, test[1])
    2141          #--------------------------------------------------------------------
    2142 -        def test_is_pg_interval():
    2143                  status = True
    2144                  tests = [
    2145                          [None, True],           # None == NULL == succeeds !
    2146                          [1, True],
    2147                          ['1', True],
    2148                          ['abc', False]
    2149                  ]
    2150
    2151                  if not is_pg_interval():
    2152                          print 'ERROR: is_pg_interval() returned "False", expected "True"'
    2153                          status = False
    2154
    2155                  for test in tests:
    2156                          result = is_pg_interval(test[0])
    2157                          if result != test[1]:
    2158                                  print 'ERROR: is_pg_interval(%s) returned "%s", expected "%s"' % (test[0], result, test[1])
    2159                                  status = False
    2160
    2161                  return status
    2162          #--------------------------------------------------------------------
    2163 -        def test_sanity_check_time_skew():
    2164                  sanity_check_time_skew()
    2165          #--------------------------------------------------------------------
    2166 -        def test_get_foreign_key_details():
    2167                  for row in get_foreign_keys2column (
    2168                          schema = u'dem',
    2169                          table = u'identity',
    2170                          column = u'pk'
    2171                  ):
    2172                          print '%s.%s references %s.%s.%s' % (
    2173                                  row['referencing_table'],
    2174                                  row['referencing_column'],
    2175                                  row['referenced_schema'],
    2176                                  row['referenced_table'],
    2177                                  row['referenced_column']
    2178                          )
    2179          #--------------------------------------------------------------------
    2180 -        def test_set_user_language():
    2181                  # (user, language, result, exception type)
    2182                  tests = [
    2183                          # current user
    2184                          [None, 'de_DE', True],
    2185                          [None, 'lang_w/o_tx', False],
    2186                          [None, None, True],
    2187                          # valid user
    2188                          ['any-doc', 'de_DE', True],
    2189                          ['any-doc', 'lang_w/o_tx', False],
    2190                          ['any-doc', None, True],
    2191                          # invalid user
    2192                          ['invalid user', 'de_DE', None],
    2193                          ['invalid user', 'lang_w/o_tx', False], # lang checking happens before user checking
    2194                          ['invalid user', None, True]
    2195                  ]
    2196                  for test in tests:
    2197                          try:
    2198                                  result = set_user_language(user = test[0], language = test[1])
    2199                                  if result != test[2]:
    2200                                          print "test:", test
    2201                                          print "result:", result, "expected:", test[2]
    2202                          except psycopg2.IntegrityError, e:
    2203                                  if test[2] is None:
    2204                                          continue
    2205                                  print "test:", test
    2206                                  print "expected exception"
    2207                                  print "result:", e
    2208          #--------------------------------------------------------------------
    2209 -        def test_get_schema_revision_history():
    2210                  for line in get_schema_revision_history():
    2211                          print u' - '.join(line)
    2212          #--------------------------------------------------------------------
    2213 -        def test_run_query():
    2214                  gmDateTime.init()
    2215                  args = {'dt': gmDateTime.pydt_max_here()}
    2216                  cmd = u"SELECT %(dt)s"
    2217
    2218                  #cmd = u"SELECT 'infinity'::timestamp with time zone"
    2219
    2220                  cmd = u"""
    2221  SELECT to_timestamp (foofoo,'YYMMDD.HH24MI') FROM (
    2222          SELECT REGEXP_REPLACE (
    2223                  't1.130729.0902.tif',                   -- string
    2224                  E'(.1)\.([0-9\.]+)(\.tif)',             -- pattern
    2225                  E'\\\\2'                                                -- replacement
    2226          ) AS foofoo
    2227  ) AS foo"""
    2228                  rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': args}], get_col_idx = False)
    2229                  print rows
    2230                  print rows[0]
    2231                  print rows[0][0]
    2232          #--------------------------------------------------------------------
    2233 -        def test_schema_exists():
    2234                  print schema_exists()
    2235          #--------------------------------------------------------------------
    2236 -        def test_row_locks():
    2237                  row_is_locked(table = 'dem.identity', pk = 12)
    2238
    2239                  print "1st connection:"
    2240                  print " locked:", row_is_locked(table = 'dem.identity', pk = 12)
    2241                  print " 1st shared lock succeeded:", lock_row(table = 'dem.identity', pk = 12, exclusive = False)
    2242                  print " locked:", row_is_locked(table = 'dem.identity', pk = 12)
    2243
    2244                  print "   2nd shared lock should succeed:", lock_row(table = 'dem.identity', pk = 12, exclusive = False)
    2245                  print "   `-> unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = False)
    2246                  print " locked:", row_is_locked(table = 'dem.identity', pk = 12)
    2247                  print "   exclusive lock should succeed:", lock_row(table = 'dem.identity', pk = 12, exclusive = True)
    2248                  print "   `-> unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = True)
    2249                  print " locked:", row_is_locked(table = 'dem.identity', pk = 12)
    2250
    2251                  print "2nd connection:"
    2252                  conn = get_raw_connection(readonly=True)
    2253                  print " shared lock should succeed:", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False)
    2254                  print " `-> unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False)
    2255                  print " locked:", row_is_locked(table = 'dem.identity', pk = 12)
    2256                  print " exclusive lock succeeded ?", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = True), "(should fail)"
    2257                  print " locked:", row_is_locked(table = 'dem.identity', pk = 12)
    2258
    2259                  print "1st connection:"
    2260                  print " unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = False)
    2261                  print " locked:", row_is_locked(table = 'dem.identity', pk = 12)
    2262
    2263                  print "2nd connection:"
    2264                  print " exclusive lock should succeed", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = True)
    2265                  print " locked:", row_is_locked(table = 'dem.identity', pk = 12)
    2266                  print "  shared lock should succeed:", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False)
    2267                  print "  `-> unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False)
    2268                  print " locked:", row_is_locked(table = 'dem.identity', pk = 12)
    2269                  print " unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False)
    2270                  print " locked:", row_is_locked(table = 'dem.identity', pk = 12)
    2271
    2272                  conn.close()
    2273          #--------------------------------------------------------------------
    2274 -        def test_get_foreign_key_names():
    2275                  print get_foreign_key_names (
    2276                          src_schema = 'dem',
    2277                          src_table = 'names',
    2278                          src_column = 'id_identity',
    2279                          target_schema = 'dem',
    2280                          target_table = 'identity',
    2281                          target_column = 'pk'
    2282                  )
    2283
    2284          #--------------------------------------------------------------------
    2285          # run tests
    2286          #test_file2bytea()
    2287          #test_get_connection()
    2288          #test_exceptions()
    2289          #test_ro_queries()
    2290          #test_request_dsn()
    2291          #test_set_encoding()
    2292          #test_connection_pool()
    2293          #test_list_args()
    2294          #test_sanitize_pg_regex()
    2295          #test_is_pg_interval()
    2296          #test_sanity_check_time_skew()
    2297          #test_get_foreign_key_details()
    2298          #test_set_user_language()
    2299          #test_get_schema_revision_history()
    2300          #test_run_query()
    2301          #test_schema_exists()
    2302          #test_get_foreign_key_names()
    2303          test_row_locks()
    2304
    2305  # ======================================================================
    2306