dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better support for google datastore

Browse Source
This commit is contained in:
mdipierro
2014-03-30 23:36:51 -05:00
parent 74b0bd9c3a
commit 0caae1f8d4
3 changed files with 46 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+3
View File
@@ -11,6 +11,7 @@ env:
- DB=mysql://root:@localhost/test_w2p - DB=mysql://root:@localhost/test_w2p
- DB=postgres://postgres:@localhost/test_w2p - DB=postgres://postgres:@localhost/test_w2p
- DB=google:datastore - DB=google:datastore
- DB=google:datastore+ndb
- DB=mongodb://mongodb:mongodb@localhost/test_w2p - DB=mongodb://mongodb:mongodb@localhost/test_w2p
- DB=imap://imap:imap@localhost:993 - DB=imap://imap:imap@localhost:993
before_script: before_script:
@@ -43,6 +44,8 @@ matrix:
env: DB=google:datastore env: DB=google:datastore
- python: '2.6' - python: '2.6'
env: DB=google:datastore env: DB=google:datastore
- python: '2.6'
env: DB=google:datastore+ndb
script: export COVERAGE_PROCESS_START=gluon/tests/coverage.ini; ./web2py.py --run_system_tests --with_coverage script: export COVERAGE_PROCESS_START=gluon/tests/coverage.ini; ./web2py.py --run_system_tests --with_coverage
VERSION
+1 -1
View File
@@ -1 +1 @@
Version 2.9.5-trunk+timestamp.2014.03.29.17.42.51 Version 2.9.5-trunk+timestamp.2014.03.30.23.35.50
gluon/dal.py
+42 -39
View File
@@ -122,6 +122,7 @@ Supported DAL URI strings::
'informixu://user:password@server:3050/database' # unicode informix 'informixu://user:password@server:3050/database' # unicode informix
'ingres://database' # or use an ODBC connection string, e.g. 'ingres://dsn=dsn_name' 'ingres://database' # or use an ODBC connection string, e.g. 'ingres://dsn=dsn_name'
'google:datastore' # for google app engine datastore 'google:datastore' # for google app engine datastore
'google:datastore+ndb' # for google app engine datastore + ndb
'google:sql' # for google app engine with sql (mysql compatible) 'google:sql' # for google app engine with sql (mysql compatible)
'teradata://DSN=dsn;UID=user;PWD=pass; DATABASE=database' # experimental 'teradata://DSN=dsn;UID=user;PWD=pass; DATABASE=database' # experimental
'imap://user:password@server:port' # experimental 'imap://user:password@server:port' # experimental
@@ -4832,6 +4833,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
See: https://developers.google.com/appengine/docs/python/ndb/cache See: https://developers.google.com/appengine/docs/python/ndb/cache
""" """
MAX_FETCH_LIMIT = 1000000
uploads_in_blob = True uploads_in_blob = True
types = {} types = {}
# reconnect is not required for Datastore dbs # reconnect is not required for Datastore dbs
@@ -4846,7 +4848,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8', def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
credential_decoder=IDENTITY, driver_args={}, credential_decoder=IDENTITY, driver_args={},
adapter_args={}, do_connect=True, after_connection=None): adapter_args={}, do_connect=True, after_connection=None):
self.use_ndb = ('use_ndb' in adapter_args) and adapter_args['use_ndb'] self.use_ndb = adapter_args.get('use_ndb',uri.startswith('google:datastore+ndb'))
if self.use_ndb is True: if self.use_ndb is True:
self.types.update({ self.types.update({
'boolean': ndb.BooleanProperty, 'boolean': ndb.BooleanProperty,
@@ -5095,11 +5097,13 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
attributes = attributes or {} attributes = attributes or {}
args_get = attributes.get args_get = attributes.get
new_fields = [] new_fields = []
for item in fields: for item in fields:
if isinstance(item,SQLALL): if isinstance(item,SQLALL):
new_fields += item._table new_fields += item._table
else: else:
new_fields.append(item) new_fields.append(item)
fields = new_fields fields = new_fields
if query: if query:
tablename = self.get_table(query) tablename = self.get_table(query)
@@ -5129,6 +5133,7 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
"text and blob field types not allowed in projection queries") "text and blob field types not allowed in projection queries")
else: else:
projection.append(f.name) projection.append(f.name)
elif args_get('filterfields') == True: elif args_get('filterfields') == True:
projection = [] projection = []
for f in fields: for f in fields:
@@ -5142,22 +5147,19 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
args_get('projection') == True\ args_get('projection') == True\
else None else None
cursor = None cursor = args_get('reusecursor')
if isinstance(args_get('reusecursor'), str): cursor = cursor if isinstance(cursor, str) else None
cursor = args_get('reusecursor')
if self.use_ndb: if self.use_ndb:
qo = ndb.QueryOptions(projection=query_projection, cursor=cursor) qo = ndb.QueryOptions(projection=query_projection, cursor=cursor)
items = tableobj.query(default_options=qo) items = tableobj.query(default_options=qo)
else: else:
items = gae.Query(tableobj, projection=query_projection, items = gae.Query(tableobj, projection=query_projection, cursor=cursor)
cursor=cursor)
for filter in filters: for filter in filters:
if args_get('projection') == True and \ if (args_get('projection') == True and
filter.name in query_projection and \ filter.name in query_projection and
filter.op in ['=', '<=', '>=']: filter.op in ('=', '<=', '>=')):
raise SyntaxError( raise SyntaxError("projection fields cannot have equality filters")
"projection fields cannot have equality filters")
if filter.name=='__key__' and filter.op=='>' and filter.value==0: if filter.name=='__key__' and filter.op=='>' and filter.value==0:
continue continue
elif filter.name=='__key__' and filter.op=='=': elif filter.name=='__key__' and filter.op=='=':
@@ -5168,29 +5170,26 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
# can't use projection # can't use projection
# extra values will be ignored in post-processing later # extra values will be ignored in post-processing later
item = filter.value.get() if self.use_ndb else tableobj.get(filter.value) item = filter.value.get() if self.use_ndb else tableobj.get(filter.value)
items = (item and [item]) or [] items = [item] if item else []
else: else:
# key qeuries return a class instance, # key qeuries return a class instance,
# can't use projection # can't use projection
# extra values will be ignored in post-processing later # extra values will be ignored in post-processing later
item = tableobj.get_by_id(filter.value) item = tableobj.get_by_id(filter.value)
items = (item and [item]) or [] items = [item] if item else []
elif isinstance(items,list): # i.e. there is a single record! elif isinstance(items,list): # i.e. there is a single record!
items = [i for i in items if filter.apply( items = [i for i in items if filter.apply(
getattr(item,filter.name),filter.value)] getattr(item,filter.name),filter.value)]
else: else:
if filter.name=='__key__' and filter.op != 'in': if filter.name=='__key__' and filter.op != 'in':
if self.use_ndb: items.order(tableobj._key) if self.use_ndb else items.order('__key__')
items.order(tableobj._key) if self.use_ndb:
else: items = self.filter(items, tableobj, filter.name, filter.op, filter.value)
items.order('__key__') else:
items = self.filter(items, tableobj, filter.name, items = items.filter('%s %s' % (filter.name,filter.op), filter.value)
filter.op, filter.value) \
if self.use_ndb else \
items.filter('%s %s' % (filter.name,filter.op),
filter.value)
if not isinstance(items,list): if not isinstance(items,list):
query = items
if args_get('left', None): if args_get('left', None):
raise SyntaxError('Set: no left join in appengine') raise SyntaxError('Set: no left join in appengine')
if args_get('groupby', None): if args_get('groupby', None):
@@ -5214,26 +5213,29 @@ class GoogleDatastoreAdapter(NoSQLAdapter):
_order = {'-id':-tableobj._key,'id':tableobj._key}.get(order) _order = {'-id':-tableobj._key,'id':tableobj._key}.get(order)
if _order is None: if _order is None:
_order = make_order(order) _order = make_order(order)
items = items.order(_order) query = query.order(_order)
else: else:
order={'-id':'-__key__','id':'__key__'}.get(order,order) order={'-id':'-__key__','id':'__key__'}.get(order,order)
items = items.order(order) query = query.order(order)
if args_get('limitby', None): if args_get('limitby', None):
(lmin, lmax) = attributes['limitby'] (lmin, lmax) = attributes['limitby']
(limit, offset) = (lmax - lmin, lmin) limit, fetch_args = lmax-lmin, {'offset':lmin,'keys_only':True}
if self.use_ndb: else:
rows, cursor, more = items.fetch_page(limit,offset=offset,keys_only=True) limit, fetch_args = self.MAX_FETCH_LIMIT, {'offset':0,'keys_only':True}
else:
rows = items.fetch(limit,offset=offset,keys_only=True) if self.use_ndb:
keys, cursor, more = query.fetch_page(limit,**fetch_args)
rows = ndb.get_multi(rows) if self.use_ndb else gae.get(rows) items = ndb.get_multi(keys)
#cursor is only useful if there was a limit and we didn't return else:
# all results keys = query.fetch(limit, **fetch_args)
if args_get('reusecursor'): items = gae.get(keys)
db['_lastcursor'] = cursor if self.use_ndb else items.cursor() cursor = query.cursor()
items = rows #cursor is only useful if there was a limit and we didn't return
# all results
if args_get('reusecursor'):
db['_lastcursor'] = cursor
return (items, tablename, projection or db[tablename].fields) return (items, tablename, projection or db[tablename].fields)
def select(self,query,fields,attributes): def select(self,query,fields,attributes):
@@ -7177,6 +7179,7 @@ ADAPTERS = {
'jdbc:postgres': JDBCPostgreSQLAdapter, 'jdbc:postgres': JDBCPostgreSQLAdapter,
'gae': GoogleDatastoreAdapter, # discouraged, for backward compatibility 'gae': GoogleDatastoreAdapter, # discouraged, for backward compatibility
'google:datastore': GoogleDatastoreAdapter, 'google:datastore': GoogleDatastoreAdapter,
'google:datastore+ndb': GoogleDatastoreAdapter,
'google:sql': GoogleSQLAdapter, 'google:sql': GoogleSQLAdapter,
'couchdb': CouchDBAdapter, 'couchdb': CouchDBAdapter,
'mongodb': MongoDBAdapter, 'mongodb': MongoDBAdapter,