From 5a23ca6230c0ea5176fb6905b3a0c5bbdf6afb8d Mon Sep 17 00:00:00 2001 From: Massimo Di Pierro Date: Mon, 16 Jan 2012 21:34:07 -0600 Subject: [PATCH] http://code.google.com/p/web2py/issues/detail?id=610 DAL IMAP suppoer, thanks spametki --- VERSION | 2 +- gluon/dal.py | 577 ++++++++++++++++++++++++++++++++++++++++++++++- gluon/sqlhtml.py | 8 +- 3 files changed, 571 insertions(+), 16 deletions(-) diff --git a/VERSION b/VERSION index 493dafef..61729010 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -Version 1.99.4 (2012-01-13 23:00:13) stable +Version 1.99.4 (2012-01-16 21:33:37) stable diff --git a/gluon/dal.py b/gluon/dal.py index b73e8b59..19b967ab 100644 --- a/gluon/dal.py +++ b/gluon/dal.py @@ -31,6 +31,7 @@ including: - MongoDB (in progress) - Google:nosql - Google:sql +- IMAP (experimental) Example of usage: @@ -108,6 +109,7 @@ Supported DAL URI strings: 'google:datastore' # for google app engine datastore 'google:sql' # for google app engine with sql (mysql compatible) 'teradata://DSN=dsn;UID=user;PWD=pass' # experimental +'imap://user:password@server:port' # experimental For more info: help(DAL) @@ -319,6 +321,12 @@ if not 'google' in drivers: except: logger.debug('no mongoDB driver') + try: + import imaplib + drivers.append('IMAP') + except: + logger.debug('could not import imaplib') + PLURALIZE_RULES = [ (re.compile('child$'), re.compile('child$'), 'children'), (re.compile('oot$'), re.compile('oot$'), 'eet'), @@ -3936,7 +3944,7 @@ class MongoDBAdapter(NoSQLAdapter): limitby = attributes.get('limitby', False) #distinct = attributes.get('distinct', False) if orderby: - #print "in if orderby %s" % orderby + print "in if orderby %s" % orderby if isinstance(orderby, (list, tuple)): print "in xorify" orderby = xorify(orderby) @@ -3948,7 +3956,7 @@ class MongoDBAdapter(NoSQLAdapter): mongosort_list.append((f[1:],-1)) else: mongosort_list.append((f,1)) - print "mongosort_list = %s" % mongosort_list + print "mongosort_list = %s" % mongosort_list if limitby: # a tuple @@ -3958,7 +3966,7 @@ class MongoDBAdapter(NoSQLAdapter): limitby_limit = 0 #if distinct: - #print "in distinct %s" % distinct + # print "in distinct %s" % distinct mongofields_dict = son.SON() mongoqry_dict = {} @@ -3989,15 +3997,15 @@ class MongoDBAdapter(NoSQLAdapter): print "mongoqry_dict=%s" % mongoqry_dict except: pass - print "mongofields_dict=%s" % mongofields_dict + # print "mongofields_dict=%s" % mongofields_dict ctable = self.connection[tablename] mongo_list_dicts = ctable.find(mongoqry_dict,mongofields_dict,skip=limitby_skip, limit=limitby_limit, sort=mongosort_list) # pymongo cursor object - print "mongo_list_dicts=%s" % mongo_list_dicts + print "mongo_list_dicts=%s" % mongo_list_dicts #if mongo_list_dicts.count() > 0: # #colnames = mongo_list_dicts[0].keys() # assuming all docs have same "shape", grab colnames from first dictionary (aka row) #else: #colnames = mongofields_dict.keys() - #print "colnames = %s" % colnames + print "colnames = %s" % colnames #rows = [row.values() for row in mongo_list_dicts] rows = mongo_list_dicts return self.parse(rows, fields, mongofields_dict.keys(), False, tablename) @@ -4094,7 +4102,7 @@ class MongoDBAdapter(NoSQLAdapter): return rowsobj def INVERT(self,first): - #print "in invert first=%s" % first + print "in invert first=%s" % first return '-%s' % self.expand(first) def drop(self, table, mode=''): @@ -4241,6 +4249,555 @@ class MongoDBAdapter(NoSQLAdapter): return '%s, %s' % (self.expand(first), self.expand(second)) +class IMAPAdapter(NoSQLAdapter): + """ IMAP server adapter + + This class is intended as an interface with + email IMAP servers to perform simple queries in the + web2py DAL query syntax, so email read, search and + other related IMAP mail services (as those implemented + by brands like Google(r), Hotmail(r) and Yahoo!(r) + can be managed from web2py applications. + + The code uses examples by Yuji Tomita on this post: + http://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/#comment-1137 + + And IMAP docs for Python imaplib and IETF's RFC2060 + + This adapter was tested with a small set of operations with Gmail(r). Other + services requests could raise command syntax and response data issues. + + + """ + types = { + 'string': str, + 'text': str, + 'date': datetime.date, + 'datetime': datetime.datetime, + 'id': long, + 'boolean': bool, + 'integer': int, + 'blob': str, + } + + dbengine = 'imap' + + def __init__(self, + db, + uri, + pool_size=0, + folder=None, + db_codec ='UTF-8', + credential_decoder=lambda x:x, + driver_args={}, + adapter_args={}): + + # db uri: user@example.com:password@imap.server.com:123 + uri = uri.split("://")[1] + self.db = db + self.uri = uri + self.pool_size=0 + self.folder = folder + self.db_codec = db_codec + self.credential_decoder = credential_decoder + self.driver_args = driver_args + self.adapter_args = adapter_args + self.mailbox_size = None + self.mailbox_names = dict() + self.encoding = sys.getfilesystemencoding() + """ MESSAGE is an identifier for sequence number""" + + self.search_fields = { + 'id': 'MESSAGE', + 'created': 'DATE', + 'uid': 'UID', + 'sender': 'FROM', + 'to': 'TO', + 'content': 'TEXT', + 'deleted': '\\Deleted', + 'draft': '\\Draft', + 'flagged': '\\Flagged', + 'recent': '\\Recent', + 'seen': '\\Seen', + 'subject': 'SUBJECT', + 'answered': '\\Answered', + 'mime': None, + 'email': None, + 'attachments': None + } + + db['_lastsql'] = '' + + m = re.compile('^(?P[^:]+)(\:(?P[^@]*))?@(?P[^\:@/]+)(\:(?P[0-9]+))?$').match(uri) + user = m.group('user') + password = m.group('password') + host = m.group('host') + port = int(m.group('port')) + over_ssl = False + if port==993: + over_ssl = True + + driver_args.update(dict(host=host,port=port, password=password, user=user)) + def connect(driver_args=driver_args): + # it is assumed sucessful authentication alLways + # TODO: support direct connection and login tests + if over_ssl: + imap4 = imaplib.IMAP4_SSL + else: + imap4 = imaplib.IMAP4 + connection = imap4(driver_args["host"], driver_args["port"]) + connection.login(driver_args["user"], driver_args["password"]) + return connection + + self.pool_connection(connect,cursor=False) + self.db.define_tables = self.define_tables + + def get_last_message(self, tablename): + last_message = None + # request mailbox list to the server + # if needed + if not len(self.mailbox_names.keys()) > 0: + self.get_mailboxes() + try: + result = self.connection.select(self.mailbox_names[tablename]) + last_message = int(result[1][0]) + except (IndexError, ValueError, TypeError, KeyError), e: + logger.debug("Error retrieving the last mailbox sequence number. %s" % str(e)) + return last_message + + def get_uid_bounds(self, tablename): + if not len(self.mailbox_names.keys()) > 0: + self.get_mailboxes() + # fetch first and last messages + # return (first, last) messages uid's + last_message = self.get_last_message(tablename) + result, data = self.connection.uid("search", None, "(ALL)") + uid_list = data[0].strip().split() + if len(uid_list) <= 0: + return None + else: + return (uid_list[0], uid_list[-1]) + + def convert_date(self, date, add=None): + if add is None: + add = datetime.timedelta() + """ Convert a date object to a string + with d-Mon-Y style for IMAP or the inverse + case + + add adds to the date object + """ + months = [None, "Jan","Feb","Mar","Apr","May","Jun", + "Jul", "Aug","Sep","Oct","Nov","Dec"] + if isinstance(date, basestring): + # Prevent unexpected date response format + try: + dayname, datestring = date.split(",") + except (ValueError): + logger.debug("Could not parse date text: %s" % date) + return None + date_list = datestring.strip().split() + year = int(date_list[2]) + month = months.index(date_list[1]) + day = int(date_list[0]) + hms = [int(value) for value in date_list[3].split(":")] + return datetime.datetime(year, month, day, + hms[0], hms[1], hms[2]) + add + elif isinstance(date, (datetime.datetime, datetime.date)): + return (date + add).strftime("%d-%b-%Y") + + else: + return None + + def decode_text(self): + """ translate encoded text for mail to unicode""" + # not implemented + pass + + def get_charset(self, message): + charset = message.get_content_charset() + return charset + + def get_mailboxes(self): + mailboxes_list = self.connection.list() + mailboxes = list() + for item in mailboxes_list[1]: + item = item.strip() + if not "NOSELECT" in item.upper(): + sub_items = item.split("\"") + sub_items = [sub_item for sub_item in sub_items if len(sub_item.strip()) > 0] + mailbox = sub_items[len(sub_items) - 1] + # remove unwanted characters and store original names + mailbox_name = mailbox.replace("[", "").replace("]", "").replace("/", "_") + mailboxes.append(mailbox_name) + self.mailbox_names[mailbox_name] = mailbox + return mailboxes + + def define_tables(self): + """ + Auto create common IMAP fileds + + This function creates fields definitions "statically" + meaning that custom fields as in other adapters should + not be supported and definitions handled on a service/mode + basis (local syntax for Gmail(r), Ymail(r) + """ + mailboxes = self.get_mailboxes() + for mailbox_name in mailboxes: + self.db.define_table("%s" % mailbox_name, + Field("uid", "string", writable=False), + Field("answered", "boolean", writable=False), + Field("created", "datetime", writable=False), + Field("content", "list:text", writable=False), + Field("to", "string", writable=False), + Field("deleted", "boolean", writable=False), + Field("draft", "boolean", writable=False), + Field("flagged", "boolean", writable=False), + Field("sender", "string", writable=False), + Field("recent", "boolean", writable=False), + Field("seen", "boolean", writable=False), + Field("subject", "string", writable=False), + Field("mime", "string", writable=False), + Field("email", "text", writable=False), + Field("attachments", "list:text", writable=False), + ) + + def create_table(self, *args, **kwargs): + # not implemented + logger.debug("Create table feature is not implemented for %s" % type(self)) + + def _select(self,query,fields,attributes): + """ Search and Fetch records and return web2py + rows + """ + + # move this statement elsewhere (upper-level) + import email + import email.header + decode_header = email.header.decode_header + # get records from imap server with search + fetch + # convert results to a dictionary + tablename = None + if isinstance(query, (Expression, Query)): + tablename = self.get_table(query) + mailbox = self.mailbox_names.get(tablename, None) + if isinstance(query, Expression): + pass + elif isinstance(query, Query): + if mailbox is not None: + # select with readonly + selected = self.connection.select(mailbox, True) + self.mailbox_size = int(selected[1][0]) + search_query = "(%s)" % str(query).strip() + search_result = self.connection.uid("search", None, search_query) + # Normal IMAP response OK is assumed (change this) + if search_result[0] == "OK": + fetch_results = list() + # For "light" remote server responses just get the first + # ten records (change for non-experimental implementation) + # However, light responses are not guaranteed with this + # approach, just fewer messages. + messages_set = search_result[1][0].split()[:10] + # Partial fetches are not used since the email + # library does not seem to support it (it converts + # partial messages to mangled message instances) + imap_fields = "(RFC822)" + if len(messages_set) > 0: + # create fetch results object list + # fetch each remote message and store it in memmory + # (change to multi-fetch command syntax for faster + # transactions) + for uid in messages_set: + typ, data = self.connection.uid("fetch", uid, imap_fields) + fr = {"message": int(data[0][0].split()[0]), + "uid": int(uid), + "email": email.message_from_string(data[0][1]) + } + fr["multipart"] = fr["email"].is_multipart() + fetch_results.append(fr) + + elif isinstance(query, basestring): + pass + else: + pass + + imapqry_dict = {} + imapfields_dict = {} + + if len(fields) == 1 and isinstance(fields[0], SQLALL): + allfields = True + elif len(fields) == 0: + allfields = True + else: + allfields = False + if allfields: + fieldnames = ["%s.%s" % (tablename, field) for field in self.search_fields.keys()] + else: + fieldnames = ["%s.%s" % (tablename, field.name) for field in fields] + + for k in fieldnames: + imapfields_dict[k] = k + + imapqry_list = list() + imapqry_array = list() + for fr in fetch_results: + n = int(fr["message"]) + item_dict = dict() + message = fr["email"] + uid = fr["uid"] + charset = self.get_charset(message) + # Return messages data mapping static fields + # and fetched results. Mapping should be made + # outside the select function (with auxiliary + # instance methods) + + # pending: search flags states trough the email message + # instances for correct output + + if "%s.id" % tablename in fieldnames: + item_dict["%s.id" % tablename] = n + if "%s.created" % tablename in fieldnames: + item_dict["%s.created" % tablename] = self.convert_date(message["Date"]) + if "%s.uid" % tablename in fieldnames: + item_dict["%s.uid" % tablename] = uid + if "%s.sender" % tablename in fieldnames: + # If there is no encoding found in the message header + # force utf-8 replacing characters (change this to + # module's defaults). Applies to .sender and .to fields + if charset is not None: + item_dict["%s.sender" % tablename] = unicode(message["From"], charset, "replace") + else: + item_dict["%s.sender" % tablename] = unicode(message["From"], "utf-8", "replace") + if "%s.to" % tablename in fieldnames: + if charset is not None: + item_dict["%s.to" % tablename] = unicode(message["To"], charset, "replace") + else: + item_dict["%s.to" % tablename] = unicode(message["To"], "utf-8", "replace") + if "%s.content" % tablename in fieldnames: + content = [] + for part in message.walk(): + if "text" in part.get_content_maintype(): + payload = part.get_payload(decode=True) + content.append(payload) + item_dict["%s.content" % tablename] = content + if "%s.deleted" % tablename in fieldnames: + item_dict["%s.deleted" % tablename] = None + if "%s.draft" % tablename in fieldnames: + item_dict["%s.draft" % tablename] = None + if "%s.flagged" % tablename in fieldnames: + item_dict["%s.flagged" % tablename] = None + if "%s.recent" % tablename in fieldnames: + item_dict["%s.recent" % tablename] = None + if "%s.seen" % tablename in fieldnames: + item_dict["%s.seen" % tablename] = None + if "%s.subject" % tablename in fieldnames: + subject = message["Subject"] + decoded_subject = decode_header(subject) + text = decoded_subject[0][0] + encoding = decoded_subject[0][1] + if encoding is not None: + text = unicode(text, encoding) + item_dict["%s.subject" % tablename] = text + if "%s.answered" % tablename in fieldnames: + item_dict["%s.answered" % tablename] = None + if "%s.mime" % tablename in fieldnames: + item_dict["%s.mime" % tablename] = message.get_content_type() + + # here goes the whole RFC822 body as an email instance + # for controller side custom processing + if "%s.email" % tablename in fieldnames: + item_dict["%s.email" % tablename] = message + + if "%s.attachments" % tablename in fieldnames: + attachments = [] + for part in message.walk(): + if not "text" in part.get_content_maintype(): + attachments.append(part.get_payload(decode=True)) + item_dict["%s.attachments" % tablename] = attachments + imapqry_list.append(item_dict) + + # extra object mapping for the sake of rows object + # creation (sends an array or lists) + for item_dict in imapqry_list: + imapqry_array_item = list() + for fieldname in fieldnames: + imapqry_array_item.append(item_dict[fieldname]) + imapqry_array.append(imapqry_array_item) + + return tablename, imapqry_array, fieldnames + + def select(self,query,fields,attributes): + tablename, imapqry_array , fieldnames = self._select(query,fields,attributes) + # parse result and return a rows object + colnames = fieldnames + result = self.parse(imapqry_array, colnames) + return result + + def count(self,query,distinct=None): + # not implemented + # (count search results without select call) + pass + + def BELONGS(self, first, second): + result = None + name = self.search_fields[first.name] + if name == "MESSAGE": + values = [str(val) for val in second if str(val).isdigit()] + result = "%s" % ",".join(values).strip() + + elif name == "UID": + values = [str(val) for val in second if str(val).isdigit()] + result = "UID %s" % ",".join(values).strip() + + else: + raise Exception("Operation not supported") + # result = "(%s %s)" % (self.expand(first), self.expand(second)) + return result + + def CONTAINS(self, first, second): + result = None + name = self.search_fields[first.name] + + if name in ("FROM", "TO", "SUBJECT", "TEXT"): + result = "%s \"%s\"" % (name, self.expand(second)) + else: + if first.name in ("cc", "bcc"): + result = "%s \"%s\"" % (first.name.upper(), self.expand(second)) + elif first.name == "mime": + result = "HEADER Content-Type \"%s\"" % self.expand(second) + else: + raise Exception("Operation not supported") + return result + + def GT(self, first, second): + result = None + name = self.search_fields[first.name] + if name == "MESSAGE": + last_message = self.get_last_message(first.tablename) + result = "%d:%d" % (int(self.expand(second)) + 1, last_message) + elif name == "UID": + # GT and LT may not return + # expected sets depending on + # the uid format implemented + try: + pedestal, threshold = self.get_uid_bounds(first.tablename) + except TypeError, e: + logger.debug("Error requesting uid bounds: %s", str(e)) + return "" + try: + lower_limit = int(self.expand(second)) + 1 + except (ValueError, TypeError), e: + raise Exception("Operation not supported (non integer UID)") + result = "UID %s:%s" % (lower_limit, threshold) + elif name == "DATE": + result = "SINCE %s" % self.convert_date(second, add=datetime.timedelta(1)) + else: + raise Exception("Operation not supported") + return result + + def GE(self, first, second): + result = None + name = self.search_fields[first.name] + if name == "MESSAGE": + last_message = self.get_last_message(first.tablename) + result = "%s:%s" % (self.expand(second), last_message) + elif name == "UID": + # GT and LT may not return + # expected sets depending on + # the uid format implemented + try: + pedestal, threshold = self.get_uid_bounds(first.tablename) + except TypeError, e: + logger.debug("Error requesting uid bounds: %s", str(e)) + return "" + lower_limit = self.expand(second) + result = "UID %s:%s" % (lower_limit, threshold) + elif name == "DATE": + result = "SINCE %s" % self.convert_date(second) + else: + raise Exception("Operation not supported") + return result + + def LT(self, first, second): + result = None + name = self.search_fields[first.name] + if name == "MESSAGE": + result = "%s:%s" % (1, int(self.expand(second)) - 1) + elif name == "UID": + try: + pedestal, threshold = self.get_uid_bounds(first.tablename) + except TypeError, e: + logger.debug("Error requesting uid bounds: %s", str(e)) + return "" + try: + upper_limit = int(self.expand(second)) - 1 + except (ValueError, TypeError), e: + raise Exception("Operation not supported (non integer UID)") + result = "UID %s:%s" % (pedestal, upper_limit) + elif name == "DATE": + result = "BEFORE %s" % self.convert_date(second) + else: + raise Exception("Operation not supported") + return result + + def LE(self, first, second): + result = None + name = self.search_fields[first.name] + if name == "MESSAGE": + result = "%s:%s" % (1, self.expand(second)) + elif name == "UID": + try: + pedestal, threshold = self.get_uid_bounds(first.tablename) + except TypeError, e: + logger.debug("Error requesting uid bounds: %s", str(e)) + return "" + upper_limit = int(self.expand(second)) + result = "UID %s:%s" % (pedestal, upper_limit) + elif name == "DATE": + result = "BEFORE %s" % self.convert_date(second, add=datetime.timedelta(1)) + else: + raise Exception("Operation not supported") + return result + + def NE(self, first, second): + result = self.NOT(self.EQ(first, second)) + result = result.replace("NOT NOT", "").strip() + return result + + def EQ(self,first,second): + name = self.search_fields[first.name] + result = None + if name is not None: + if name == "MESSAGE": + # query by message sequence number + result = "%s" % self.expand(second) + elif name == "UID": + result = "UID %s" % self.expand(second) + elif name == "DATE": + result = "ON %s" % self.convert_date(second) + + elif name in ('\\Deleted', '\\Draft', '\\Flagged', '\\Recent', '\\Seen', '\\Answered'): + if second: + result = "%s" % (name.upper()[1:]) + else: + result = "NOT %s" % (name.upper()[1:]) + else: + raise Exception("Operation not supported") + else: + raise Exception("Operation not supported") + return result + + def AND(self, first, second): + result = "%s %s" % (self.expand(first), self.expand(second)) + return result + + def OR(self, first, second): + result = "OR %s %s" % (self.expand(first), self.expand(second)) + return "%s" % result.replace("OR OR", "OR") + + def NOT(self, first): + result = "NOT %s" % self.expand(first) + return result ######################################################################## # end of adapters @@ -4271,6 +4828,7 @@ ADAPTERS = { 'google:sql': GoogleSQLAdapter, 'couchdb': CouchDBAdapter, 'mongodb': MongoDBAdapter, + 'imap': IMAPAdapter } @@ -6935,8 +7493,3 @@ if __name__ == '__main__': import doctest doctest.testmod() - - - - - diff --git a/gluon/sqlhtml.py b/gluon/sqlhtml.py index d7e5ecf6..65431dbb 100644 --- a/gluon/sqlhtml.py +++ b/gluon/sqlhtml.py @@ -702,7 +702,7 @@ class SQLFORM(FORM): labels = None, col3 = {}, submit_button = 'Submit', - delete_label = 'Check to delete:', + delete_label = 'Check to delete', showid = True, readonly = False, comments = True, @@ -919,7 +919,7 @@ class SQLFORM(FORM): ) xfields.append((self.FIELDKEY_DELETE_RECORD+SQLFORM.ID_ROW_SUFFIX, LABEL( - delete_label, + delete_label,separator, _for=self.FIELDKEY_DELETE_RECORD, _id=self.FIELDKEY_DELETE_RECORD+SQLFORM.ID_LABEL_SUFFIX), widget, @@ -1577,7 +1577,9 @@ class SQLFORM(FORM): record = table(request.args[-1]) or redirect(URL('error')) edit_form = SQLFORM(table,record,upload=upload,ignore_rw=ignore_rw, deletable=deletable, - _class='web2py_form') + _class='web2py_form', + submit_button = T('Submit'), + delete_label = T('Check to delete')) edit_form.process(formname=formname, onvalidation=onvalidation, onsuccess=onupdate,