Remove submodule, just put Dependencies in ./libs

libs/imdb/parser/__init__.py

@@ -0,0 +1,28 @@
+"""
+parser package (imdb package).
+
+This package provides various parsers to access IMDb data (e.g.: a
+parser for the web/http interface, a parser for the SQL database
+interface, etc.).
+So far, the http/httpThin, mobile and sql parsers are implemented.
+
+Copyright 2004-2009 Davide Alberani <da@erlug.linux.it>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+__all__ = ['http', 'mobile', 'sql']
+
+

libs/imdb/parser/http/__init__.py

@@ -0,0 +1,771 @@
+"""
+parser.http package (imdb package).
+
+This package provides the IMDbHTTPAccessSystem class used to access
+IMDb's data through the web interface.
+the imdb.IMDb function will return an instance of this class when
+called with the 'accessSystem' argument set to "http" or "web"
+or "html" (this is the default).
+
+Copyright 2004-2010 Davide Alberani <da@erlug.linux.it>
+               2008 H. Turgut Uyar <uyar@tekir.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import sys
+import logging
+from urllib import FancyURLopener, quote_plus
+from codecs import lookup
+
+from imdb import IMDbBase, imdbURL_movie_main, imdbURL_person_main, \
+                imdbURL_character_main, imdbURL_company_main, \
+                imdbURL_keyword_main, imdbURL_find, imdbURL_top250, \
+                imdbURL_bottom100
+from imdb.utils import analyze_title
+from imdb._exceptions import IMDbDataAccessError, IMDbParserError
+
+import searchMovieParser
+import searchPersonParser
+import searchCharacterParser
+import searchCompanyParser
+import searchKeywordParser
+import movieParser
+import personParser
+import characterParser
+import companyParser
+import topBottomParser
+
+# Logger for miscellaneous functions.
+_aux_logger = logging.getLogger('imdbpy.parser.http.aux')
+
+IN_GAE = False
+try:
+    import google.appengine
+    IN_GAE = True
+    _aux_logger.info('IMDbPY is running in the Google App Engine environment')
+except ImportError:
+    pass
+
+
+class _ModuleProxy:
+    """A proxy to instantiate and access parsers."""
+    def __init__(self, module, defaultKeys=None, oldParsers=False,
+                useModule=None, fallBackToNew=False):
+        """Initialize a proxy for the given module; defaultKeys, if set,
+        muste be a dictionary of values to set for instanced objects."""
+        if oldParsers or fallBackToNew:
+            _aux_logger.warn('The old set of parsers was removed; falling ' \
+                            'back to the new parsers.')
+        self.useModule = useModule
+        if defaultKeys is None:
+            defaultKeys = {}
+        self._defaultKeys = defaultKeys
+        self._module = module
+
+    def __getattr__(self, name):
+        """Called only when no look-up is found."""
+        _sm = self._module
+        # Read the _OBJECTS dictionary to build the asked parser.
+        if name in _sm._OBJECTS:
+            _entry = _sm._OBJECTS[name]
+            # Initialize the parser.
+            kwds = {}
+            if self.useModule:
+                kwds = {'useModule': self.useModule}
+            parserClass = _entry[0][0]
+            obj = parserClass(**kwds)
+            attrsToSet = self._defaultKeys.copy()
+            attrsToSet.update(_entry[1] or {})
+            # Set attribute to the object.
+            for key in attrsToSet:
+                setattr(obj, key, attrsToSet[key])
+            setattr(self, name, obj)
+            return obj
+        return getattr(_sm, name)
+
+
+PY_VERSION = sys.version_info[:2]
+
+
+# The cookies for the "adult" search.
+# Please don't mess with these account.
+# Old 'IMDbPY' account.
+_old_cookie_id = 'boM2bYxz9MCsOnH9gZ0S9QHs12NWrNdApxsls1Vb5/NGrNdjcHx3dUas10UASoAjVEvhAbGagERgOpNkAPvxdbfKwaV2ikEj9SzXY1WPxABmDKQwdqzwRbM+12NSeJFGUEx3F8as10WwidLzVshDtxaPIbP13NdjVS9UZTYqgTVGrNcT9vyXU1'
+_old_cookie_uu = '3M3AXsquTU5Gur/Svik+ewflPm5Rk2ieY3BIPlLjyK3C0Dp9F8UoPgbTyKiGtZp4x1X+uAUGKD7BM2g+dVd8eqEzDErCoYvdcvGLvVLAen1y08hNQtALjVKAe+1hM8g9QbNonlG1/t4S82ieUsBbrSIQbq1yhV6tZ6ArvSbA7rgHc8n5AdReyAmDaJ5Wm/ee3VDoCnGj/LlBs2ieUZNorhHDKK5Q=='
+# New 'IMDbPYweb' account.
+_cookie_id = 'rH1jNAkjTlNXvHolvBVBsgaPICNZbNdjVjzFwzas9JRmusdjVoqBs/Hs12NR+1WFxEoR9bGKEDUg6sNlADqXwkas12N131Rwdb+UQNGKN8PWrNdjcdqBQVLq8mbGDHP3hqzxhbD692NQi9D0JjpBtRaPIbP1zNdjUOqENQYv1ADWrNcT9vyXU1'
+_cookie_uu = 'su4/m8cho4c6HP+W1qgq6wchOmhnF0w+lIWvHjRUPJ6nRA9sccEafjGADJ6hQGrMd4GKqLcz2X4z5+w+M4OIKnRn7FpENH7dxDQu3bQEHyx0ZEyeRFTPHfQEX03XF+yeN1dsPpcXaqjUZAw+lGRfXRQEfz3RIX9IgVEffdBAHw2wQXyf9xdMPrQELw0QNB8dsffsqcdQemjPB0w+moLcPh0JrKrHJ9hjBzdMPpcXTH7XRwwOk='
+
+
+class _FakeURLOpener(object):
+    """Fake URLOpener object, used to return empty strings instead of
+    errors.
+    """
+    def __init__(self, url, headers):
+        self.url = url
+        self.headers = headers
+    def read(self, *args, **kwds): return ''
+    def close(self, *args, **kwds): pass
+    def info(self, *args, **kwds): return self.headers
+
+
+class IMDbURLopener(FancyURLopener):
+    """Fetch web pages and handle errors."""
+    _logger = logging.getLogger('imdbpy.parser.http.urlopener')
+
+    def __init__(self, *args, **kwargs):
+        self._last_url = u''
+        FancyURLopener.__init__(self, *args, **kwargs)
+        # Headers to add to every request.
+        # XXX: IMDb's web server doesn't like urllib-based programs,
+        #      so lets fake to be Mozilla.
+        #      Wow!  I'm shocked by my total lack of ethic! <g>
+        for header in ('User-Agent', 'User-agent', 'user-agent'):
+            self.del_header(header)
+        self.set_header('User-Agent', 'Mozilla/5.0')
+        # XXX: This class is used also to perform "Exact Primary
+        #      [Title|Name]" searches, and so by default the cookie is set.
+        c_header = 'id=%s; uu=%s' % (_cookie_id, _cookie_uu)
+        self.set_header('Cookie', c_header)
+
+    def get_proxy(self):
+        """Return the used proxy, or an empty string."""
+        return self.proxies.get('http', '')
+
+    def set_proxy(self, proxy):
+        """Set the proxy."""
+        if not proxy:
+            if self.proxies.has_key('http'):
+                del self.proxies['http']
+        else:
+            if not proxy.lower().startswith('http://'):
+                proxy = 'http://%s' % proxy
+            self.proxies['http'] = proxy
+
+    def set_header(self, header, value, _overwrite=True):
+        """Set a default header."""
+        if _overwrite:
+            self.del_header(header)
+        self.addheaders.append((header, value))
+
+    def del_header(self, header):
+        """Remove a default header."""
+        for index in xrange(len(self.addheaders)):
+            if self.addheaders[index][0] == header:
+                del self.addheaders[index]
+                break
+
+    def retrieve_unicode(self, url, size=-1):
+        """Retrieves the given URL, and returns a unicode string,
+        trying to guess the encoding of the data (assuming latin_1
+        by default)"""
+        encode = None
+        try:
+            if size != -1:
+                self.set_header('Range', 'bytes=0-%d' % size)
+            uopener = self.open(url)
+            kwds = {}
+            if PY_VERSION > (2, 3) and not IN_GAE:
+                kwds['size'] = size
+            content = uopener.read(**kwds)
+            self._last_url = uopener.url
+            # Maybe the server is so nice to tell us the charset...
+            server_encode = uopener.info().getparam('charset')
+            # Otherwise, look at the content-type HTML meta tag.
+            if server_encode is None and content:
+                first_bytes = content[:512]
+                begin_h = first_bytes.find('text/html; charset=')
+                if begin_h != -1:
+                    end_h = first_bytes[19+begin_h:].find('"')
+                    if end_h != -1:
+                        server_encode = first_bytes[19+begin_h:19+begin_h+end_h]
+            if server_encode:
+                try:
+                    if lookup(server_encode):
+                        encode = server_encode
+                except (LookupError, ValueError, TypeError):
+                    pass
+            uopener.close()
+            if size != -1:
+                self.del_header('Range')
+            self.close()
+        except IOError, e:
+            if size != -1:
+                # Ensure that the Range header is removed.
+                self.del_header('Range')
+            raise IMDbDataAccessError, {'errcode': e.errno,
+                                        'errmsg': str(e.strerror),
+                                        'url': url,
+                                        'proxy': self.get_proxy(),
+                                        'exception type': 'IOError',
+                                        'original exception': e}
+        if encode is None:
+            encode = 'latin_1'
+            # The detection of the encoding is error prone...
+            self._logger.warn('Unable to detect the encoding of the retrieved '
+                        'page [%s]; falling back to default latin1.', encode)
+        ##print unicode(content, encode, 'replace').encode('utf8')
+        return unicode(content, encode, 'replace')
+
+    def http_error_default(self, url, fp, errcode, errmsg, headers):
+        if errcode == 404:
+            self._logger.warn('404 code returned for %s: %s (headers: %s)',
+                                url, errmsg, headers)
+            return _FakeURLOpener(url, headers)
+        raise IMDbDataAccessError, {'url': 'http:%s' % url,
+                                    'errcode': errcode,
+                                    'errmsg': errmsg,
+                                    'headers': headers,
+                                    'error type': 'http_error_default',
+                                    'proxy': self.get_proxy()}
+
+    def open_unknown(self, fullurl, data=None):
+        raise IMDbDataAccessError, {'fullurl': fullurl,
+                                    'data': str(data),
+                                    'error type': 'open_unknown',
+                                    'proxy': self.get_proxy()}
+
+    def open_unknown_proxy(self, proxy, fullurl, data=None):
+        raise IMDbDataAccessError, {'proxy': str(proxy),
+                                    'fullurl': fullurl,
+                                    'error type': 'open_unknown_proxy',
+                                    'data': str(data)}
+
+
+class IMDbHTTPAccessSystem(IMDbBase):
+    """The class used to access IMDb's data through the web."""
+
+    accessSystem = 'http'
+    _http_logger = logging.getLogger('imdbpy.parser.http')
+
+    def __init__(self, isThin=0, adultSearch=1, proxy=-1, oldParsers=False,
+                fallBackToNew=False, useModule=None, cookie_id=-1,
+                cookie_uu=None, *arguments, **keywords):
+        """Initialize the access system."""
+        IMDbBase.__init__(self, *arguments, **keywords)
+        self.urlOpener =  IMDbURLopener()
+        # When isThin is set, we're parsing the "maindetails" page
+        # of a movie (instead of the "combined" page) and movie/person
+        # references are not collected if no defaultModFunct is provided.
+        self.isThin = isThin
+        self._getRefs = True
+        self._mdparse = False
+        if isThin:
+            if self.accessSystem == 'http':
+                self.accessSystem = 'httpThin'
+            self._mdparse = True
+            if self._defModFunct is None:
+                self._getRefs = False
+                from imdb.utils import modNull
+                self._defModFunct = modNull
+        self.do_adult_search(adultSearch)
+        if cookie_id != -1:
+            if cookie_id is None:
+                self.del_cookies()
+            elif cookie_uu is not None:
+                self.set_cookies(cookie_id, cookie_uu)
+        if proxy != -1:
+            self.set_proxy(proxy)
+        if useModule is not None:
+            if not isinstance(useModule, (list, tuple)) and ',' in useModule:
+                useModule = useModule.split(',')
+        _def = {'_modFunct': self._defModFunct, '_as': self.accessSystem}
+        # Proxy objects.
+        self.smProxy = _ModuleProxy(searchMovieParser, defaultKeys=_def,
+                                    oldParsers=oldParsers, useModule=useModule,
+                                    fallBackToNew=fallBackToNew)
+        self.spProxy = _ModuleProxy(searchPersonParser, defaultKeys=_def,
+                                    oldParsers=oldParsers, useModule=useModule,
+                                    fallBackToNew=fallBackToNew)
+        self.scProxy = _ModuleProxy(searchCharacterParser, defaultKeys=_def,
+                                    oldParsers=oldParsers, useModule=useModule,
+                                    fallBackToNew=fallBackToNew)
+        self.scompProxy = _ModuleProxy(searchCompanyParser, defaultKeys=_def,
+                                    oldParsers=oldParsers, useModule=useModule,
+                                    fallBackToNew=fallBackToNew)
+        self.skProxy = _ModuleProxy(searchKeywordParser, defaultKeys=_def,
+                                    oldParsers=oldParsers, useModule=useModule,
+                                    fallBackToNew=fallBackToNew)
+        self.mProxy = _ModuleProxy(movieParser, defaultKeys=_def,
+                                    oldParsers=oldParsers, useModule=useModule,
+                                    fallBackToNew=fallBackToNew)
+        self.pProxy = _ModuleProxy(personParser, defaultKeys=_def,
+                                    oldParsers=oldParsers, useModule=useModule,
+                                    fallBackToNew=fallBackToNew)
+        self.cProxy = _ModuleProxy(characterParser, defaultKeys=_def,
+                                    oldParsers=oldParsers, useModule=useModule,
+                                    fallBackToNew=fallBackToNew)
+        self.compProxy = _ModuleProxy(companyParser, defaultKeys=_def,
+                                    oldParsers=oldParsers, useModule=useModule,
+                                    fallBackToNew=fallBackToNew)
+        self.topBottomProxy = _ModuleProxy(topBottomParser, defaultKeys=_def,
+                                    oldParsers=oldParsers, useModule=useModule,
+                                    fallBackToNew=fallBackToNew)
+
+    def _normalize_movieID(self, movieID):
+        """Normalize the given movieID."""
+        try:
+            return '%07d' % int(movieID)
+        except ValueError, e:
+            raise IMDbParserError, 'invalid movieID "%s": %s' % (movieID, e)
+
+    def _normalize_personID(self, personID):
+        """Normalize the given personID."""
+        try:
+            return '%07d' % int(personID)
+        except ValueError, e:
+            raise IMDbParserError, 'invalid personID "%s": %s' % (personID, e)
+
+    def _normalize_characterID(self, characterID):
+        """Normalize the given characterID."""
+        try:
+            return '%07d' % int(characterID)
+        except ValueError, e:
+            raise IMDbParserError, 'invalid characterID "%s": %s' % \
+                    (characterID, e)
+
+    def _normalize_companyID(self, companyID):
+        """Normalize the given companyID."""
+        try:
+            return '%07d' % int(companyID)
+        except ValueError, e:
+            raise IMDbParserError, 'invalid companyID "%s": %s' % \
+                    (companyID, e)
+
+    def get_imdbMovieID(self, movieID):
+        """Translate a movieID in an imdbID; in this implementation
+        the movieID _is_ the imdbID.
+        """
+        return movieID
+
+    def get_imdbPersonID(self, personID):
+        """Translate a personID in an imdbID; in this implementation
+        the personID _is_ the imdbID.
+        """
+        return personID
+
+    def get_imdbCharacterID(self, characterID):
+        """Translate a characterID in an imdbID; in this implementation
+        the characterID _is_ the imdbID.
+        """
+        return characterID
+
+    def get_imdbCompanyID(self, companyID):
+        """Translate a companyID in an imdbID; in this implementation
+        the companyID _is_ the imdbID.
+        """
+        return companyID
+
+    def get_proxy(self):
+        """Return the used proxy or an empty string."""
+        return self.urlOpener.get_proxy()
+
+    def set_proxy(self, proxy):
+        """Set the web proxy to use.
+
+        It should be a string like 'http://localhost:8080/'; if the
+        string is empty, no proxy will be used.
+        If set, the value of the environment variable HTTP_PROXY is
+        automatically used.
+        """
+        self.urlOpener.set_proxy(proxy)
+
+    def set_cookies(self, cookie_id, cookie_uu):
+        """Set a cookie to access an IMDb's account."""
+        c_header = 'id=%s; uu=%s' % (cookie_id, cookie_uu)
+        self.urlOpener.set_header('Cookie', c_header)
+
+    def del_cookies(self):
+        """Remove the used cookie."""
+        self.urlOpener.del_header('Cookie')
+
+    def do_adult_search(self, doAdult,
+                        cookie_id=_cookie_id, cookie_uu=_cookie_uu):
+        """If doAdult is true, 'adult' movies are included in the
+        search results; cookie_id and cookie_uu are optional
+        parameters to select a specific account (see your cookie
+        or cookies.txt file."""
+        if doAdult:
+            self.set_cookies(cookie_id, cookie_uu)
+            #c_header = 'id=%s; uu=%s' % (cookie_id, cookie_uu)
+            #self.urlOpener.set_header('Cookie', c_header)
+        else:
+            self.urlOpener.del_header('Cookie')
+
+    def _retrieve(self, url, size=-1):
+        """Retrieve the given URL."""
+        ##print url
+        self._http_logger.debug('fetching url %s (size: %d)', url, size)
+        return self.urlOpener.retrieve_unicode(url, size=size)
+
+    def _get_search_content(self, kind, ton, results):
+        """Retrieve the web page for a given search.
+        kind can be 'tt' (for titles), 'nm' (for names),
+        'char' (for characters) or 'co' (for companies).
+        ton is the title or the name to search.
+        results is the maximum number of results to be retrieved."""
+        if isinstance(ton, unicode):
+            ton = ton.encode('utf-8')
+        ##params = 'q=%s&%s=on&mx=%s' % (quote_plus(ton), kind, str(results))
+        params = 's=%s;mx=%s;q=%s' % (kind, str(results), quote_plus(ton))
+        if kind == 'ep':
+            params = params.replace('s=ep;', 's=tt;ttype=ep;', 1)
+        cont = self._retrieve(imdbURL_find % params)
+        #print 'URL:', imdbURL_find % params
+        if cont.find('Your search returned more than') == -1 or \
+                cont.find("displayed the exact matches") == -1:
+            return cont
+        # The retrieved page contains no results, because too many
+        # titles or names contain the string we're looking for.
+        params = 's=%s;q=%s;lm=0' % (kind, quote_plus(ton))
+        size = 22528 + results * 512
+        return self._retrieve(imdbURL_find % params, size=size)
+
+    def _search_movie(self, title, results):
+        # The URL of the query.
+        # XXX: To retrieve the complete results list:
+        #      params = urllib.urlencode({'more': 'tt', 'q': title})
+        ##params = urllib.urlencode({'tt': 'on','mx': str(results),'q': title})
+        ##params = 'q=%s&tt=on&mx=%s' % (quote_plus(title), str(results))
+        ##cont = self._retrieve(imdbURL_find % params)
+        cont = self._get_search_content('tt', title, results)
+        return self.smProxy.search_movie_parser.parse(cont, results=results)['data']
+
+    def _search_episode(self, title, results):
+        t_dict = analyze_title(title)
+        if t_dict['kind'] == 'episode':
+            title = t_dict['title']
+        cont = self._get_search_content('ep', title, results)
+        return self.smProxy.search_movie_parser.parse(cont, results=results)['data']
+
+    def get_movie_main(self, movieID):
+        if not self.isThin:
+            cont = self._retrieve(imdbURL_movie_main % movieID + 'combined')
+        else:
+            cont = self._retrieve(imdbURL_movie_main % movieID + 'maindetails')
+        return self.mProxy.movie_parser.parse(cont, mdparse=self._mdparse)
+
+    def get_movie_full_credits(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'fullcredits')
+        return self.mProxy.movie_parser.parse(cont)
+
+    def get_movie_plot(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'plotsummary')
+        return self.mProxy.plot_parser.parse(cont, getRefs=self._getRefs)
+
+    def get_movie_awards(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'awards')
+        return self.mProxy.movie_awards_parser.parse(cont)
+
+    def get_movie_taglines(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'taglines')
+        return self.mProxy.taglines_parser.parse(cont)
+
+    def get_movie_keywords(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'keywords')
+        return self.mProxy.keywords_parser.parse(cont)
+
+    def get_movie_alternate_versions(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'alternateversions')
+        return self.mProxy.alternateversions_parser.parse(cont,
+                                                        getRefs=self._getRefs)
+
+    def get_movie_crazy_credits(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'crazycredits')
+        return self.mProxy.crazycredits_parser.parse(cont,
+                                                    getRefs=self._getRefs)
+
+    def get_movie_goofs(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'goofs')
+        return self.mProxy.goofs_parser.parse(cont, getRefs=self._getRefs)
+
+    def get_movie_quotes(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'quotes')
+        return self.mProxy.quotes_parser.parse(cont, getRefs=self._getRefs)
+
+    def get_movie_release_dates(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'releaseinfo')
+        ret = self.mProxy.releasedates_parser.parse(cont)
+        ret['info sets'] = ('release dates', 'akas')
+        return ret
+    get_movie_akas = get_movie_release_dates
+
+    def get_movie_vote_details(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'ratings')
+        return self.mProxy.ratings_parser.parse(cont)
+
+    def get_movie_official_sites(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'officialsites')
+        return self.mProxy.officialsites_parser.parse(cont)
+
+    def get_movie_trivia(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'trivia')
+        return self.mProxy.trivia_parser.parse(cont, getRefs=self._getRefs)
+
+    def get_movie_connections(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'movieconnections')
+        return self.mProxy.connections_parser.parse(cont)
+
+    def get_movie_technical(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'technical')
+        return self.mProxy.tech_parser.parse(cont)
+
+    def get_movie_business(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'business')
+        return self.mProxy.business_parser.parse(cont, getRefs=self._getRefs)
+
+    def get_movie_literature(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'literature')
+        return self.mProxy.literature_parser.parse(cont)
+
+    def get_movie_locations(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'locations')
+        return self.mProxy.locations_parser.parse(cont)
+
+    def get_movie_soundtrack(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'soundtrack')
+        return self.mProxy.soundtrack_parser.parse(cont)
+
+    def get_movie_dvd(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'dvd')
+        return self.mProxy.dvd_parser.parse(cont, getRefs=self._getRefs)
+
+    def get_movie_recommendations(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'recommendations')
+        return self.mProxy.rec_parser.parse(cont)
+
+    def get_movie_external_reviews(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'externalreviews')
+        return self.mProxy.externalrev_parser.parse(cont)
+
+    def get_movie_newsgroup_reviews(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'newsgroupreviews')
+        return self.mProxy.newsgrouprev_parser.parse(cont)
+
+    def get_movie_misc_sites(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'miscsites')
+        return self.mProxy.misclinks_parser.parse(cont)
+
+    def get_movie_sound_clips(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'soundsites')
+        return self.mProxy.soundclips_parser.parse(cont)
+
+    def get_movie_video_clips(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'videosites')
+        return self.mProxy.videoclips_parser.parse(cont)
+
+    def get_movie_photo_sites(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'photosites')
+        return self.mProxy.photosites_parser.parse(cont)
+
+    def get_movie_news(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'news')
+        return self.mProxy.news_parser.parse(cont, getRefs=self._getRefs)
+
+    def get_movie_amazon_reviews(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'amazon')
+        return self.mProxy.amazonrev_parser.parse(cont)
+
+    def get_movie_guests(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'epcast')
+        return self.mProxy.episodes_cast_parser.parse(cont)
+    get_movie_episodes_cast = get_movie_guests
+
+    def get_movie_merchandising_links(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'sales')
+        return self.mProxy.sales_parser.parse(cont)
+
+    def get_movie_episodes(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'episodes')
+        data_d = self.mProxy.episodes_parser.parse(cont)
+        # set movie['episode of'].movieID for every episode of the series.
+        if data_d.get('data', {}).has_key('episodes'):
+            nr_eps = 0
+            for season in data_d['data']['episodes'].values():
+                for episode in season.values():
+                    episode['episode of'].movieID = movieID
+                    nr_eps += 1
+            # Number of episodes.
+            if nr_eps:
+                data_d['data']['number of episodes'] = nr_eps
+        return data_d
+
+    def get_movie_episodes_rating(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'epdate')
+        data_d = self.mProxy.eprating_parser.parse(cont)
+        # set movie['episode of'].movieID for every episode.
+        if data_d.get('data', {}).has_key('episodes rating'):
+            for item in data_d['data']['episodes rating']:
+                episode = item['episode']
+                episode['episode of'].movieID = movieID
+        return data_d
+
+    def get_movie_faqs(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'faq')
+        return self.mProxy.movie_faqs_parser.parse(cont, getRefs=self._getRefs)
+
+    def get_movie_airing(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'tvschedule')
+        return self.mProxy.airing_parser.parse(cont)
+
+    get_movie_tv_schedule = get_movie_airing
+
+    def get_movie_synopsis(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'synopsis')
+        return self.mProxy.synopsis_parser.parse(cont)
+
+    def get_movie_parents_guide(self, movieID):
+        cont = self._retrieve(imdbURL_movie_main % movieID + 'parentalguide')
+        return self.mProxy.parentsguide_parser.parse(cont)
+
+    def _search_person(self, name, results):
+        # The URL of the query.
+        # XXX: To retrieve the complete results list:
+        #      params = urllib.urlencode({'more': 'nm', 'q': name})
+        ##params = urllib.urlencode({'nm': 'on', 'mx': str(results), 'q': name})
+        #params = 'q=%s&nm=on&mx=%s' % (quote_plus(name), str(results))
+        #cont = self._retrieve(imdbURL_find % params)
+        cont = self._get_search_content('nm', name, results)
+        return self.spProxy.search_person_parser.parse(cont, results=results)['data']
+
+    def get_person_main(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'maindetails')
+        ret = self.pProxy.maindetails_parser.parse(cont)
+        ret['info sets'] = ('main', 'filmography')
+        return ret
+
+    def get_person_filmography(self, personID):
+        return self.get_person_main(personID)
+
+    def get_person_biography(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'bio')
+        return self.pProxy.bio_parser.parse(cont, getRefs=self._getRefs)
+
+    def get_person_awards(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'awards')
+        return self.pProxy.person_awards_parser.parse(cont)
+
+    def get_person_other_works(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'otherworks')
+        return self.pProxy.otherworks_parser.parse(cont, getRefs=self._getRefs)
+
+    #def get_person_agent(self, personID):
+    #    cont = self._retrieve(imdbURL_person_main % personID + 'agent')
+    #    return self.pProxy.agent_parser.parse(cont)
+
+    def get_person_publicity(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'publicity')
+        return self.pProxy.publicity_parser.parse(cont)
+
+    def get_person_official_sites(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'officialsites')
+        return self.pProxy.person_officialsites_parser.parse(cont)
+
+    def get_person_news(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'news')
+        return self.pProxy.news_parser.parse(cont)
+
+    def get_person_episodes(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'filmoseries')
+        return self.pProxy.person_series_parser.parse(cont)
+
+    def get_person_merchandising_links(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'forsale')
+        return self.pProxy.sales_parser.parse(cont)
+
+    def get_person_genres_links(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'filmogenre')
+        return self.pProxy.person_genres_parser.parse(cont)
+
+    def get_person_keywords_links(self, personID):
+        cont = self._retrieve(imdbURL_person_main % personID + 'filmokey')
+        return self.pProxy.person_keywords_parser.parse(cont)
+
+    def _search_character(self, name, results):
+        cont = self._get_search_content('char', name, results)
+        return self.scProxy.search_character_parser.parse(cont, results=results)['data']
+
+    def get_character_main(self, characterID):
+        cont = self._retrieve(imdbURL_character_main % characterID)
+        ret = self.cProxy.character_main_parser.parse(cont)
+        ret['info sets'] = ('main', 'filmography')
+        return ret
+
+    get_character_filmography = get_character_main
+
+    def get_character_biography(self, characterID):
+        cont = self._retrieve(imdbURL_character_main % characterID + 'bio')
+        return self.cProxy.character_bio_parser.parse(cont,
+                                                    getRefs=self._getRefs)
+
+    def get_character_episodes(self, characterID):
+        cont = self._retrieve(imdbURL_character_main % characterID +
+                                'filmoseries')
+        return self.cProxy.character_series_parser.parse(cont)
+
+    def get_character_quotes(self, characterID):
+        cont = self._retrieve(imdbURL_character_main % characterID + 'quotes')
+        return self.cProxy.character_quotes_parser.parse(cont,
+                                                    getRefs=self._getRefs)
+
+    def _search_company(self, name, results):
+        cont = self._get_search_content('co', name, results)
+        url = self.urlOpener._last_url
+        return self.scompProxy.search_company_parser.parse(cont, url=url,
+                                                    results=results)['data']
+
+    def get_company_main(self, companyID):
+        cont = self._retrieve(imdbURL_company_main % companyID)
+        ret = self.compProxy.company_main_parser.parse(cont)
+        return ret
+
+    def _search_keyword(self, keyword, results):
+        # XXX: the IMDb web server seems to have some serious problem with
+        #      non-ascii keyword.
+        #      E.g.: http://akas.imdb.com/keyword/fianc%E9/
+        #      will return a 500 Internal Server Error: Redirect Recursion.
+        keyword = keyword.encode('utf8', 'ignore')
+        try:
+            cont = self._get_search_content('kw', keyword, results)
+        except IMDbDataAccessError:
+            self._http_logger.warn('unable to search for keyword %s', keyword,
+                                    exc_info=True)
+            return []
+        return self.skProxy.search_keyword_parser.parse(cont, results=results)['data']
+
+    def _get_keyword(self, keyword, results):
+        keyword = keyword.encode('utf8', 'ignore')
+        try:
+            cont = self._retrieve(imdbURL_keyword_main % keyword)
+        except IMDbDataAccessError:
+            self._http_logger.warn('unable to get keyword %s', keyword,
+                                    exc_info=True)
+            return []
+        return self.skProxy.search_moviekeyword_parser.parse(cont, results=results)['data']
+
+    def _get_top_bottom_movies(self, kind):
+        if kind == 'top':
+            parser = self.topBottomProxy.top250_parser
+            url = imdbURL_top250
+        elif kind == 'bottom':
+            parser = self.topBottomProxy.bottom100_parser
+            url = imdbURL_bottom100
+        else:
+            return []
+        cont = self._retrieve(url)
+        return parser.parse(cont)['data']
+
+

libs/imdb/parser/http/bsouplxml/bsoupxpath.py

@@ -0,0 +1,394 @@
+"""
+parser.http.bsoupxpath module (imdb.parser.http package).
+
+This module provides XPath support for BeautifulSoup.
+
+Copyright 2008 H. Turgut Uyar <uyar@tekir.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+__author__ = 'H. Turgut Uyar <uyar@tekir.org>'
+__docformat__ = 'restructuredtext'
+
+
+import re
+import string
+import _bsoup as BeautifulSoup
+
+
+# XPath related enumerations and constants
+
+AXIS_ANCESTOR          = 'ancestor'
+AXIS_ATTRIBUTE         = 'attribute'
+AXIS_CHILD             = 'child'
+AXIS_DESCENDANT        = 'descendant'
+AXIS_FOLLOWING         = 'following'
+AXIS_FOLLOWING_SIBLING = 'following-sibling'
+AXIS_PRECEDING_SIBLING = 'preceding-sibling'
+
+AXES = (AXIS_ANCESTOR, AXIS_ATTRIBUTE, AXIS_CHILD, AXIS_DESCENDANT,
+        AXIS_FOLLOWING, AXIS_FOLLOWING_SIBLING, AXIS_PRECEDING_SIBLING)
+
+XPATH_FUNCTIONS = ('starts-with', 'string-length')
+
+
+def tokenize_path(path):
+    """Tokenize a location path into location steps. Return the list of steps.
+
+    If two steps are separated by a double slash, the double slashes are part of
+    the second step. If they are separated by only one slash, the slash is not
+    included in any of the steps.
+    """
+    # form a list of tuples that mark the start and end positions of steps
+    separators = []
+    last_position = 0
+    i = -1
+    in_string = False
+    while i < len(path) - 1:
+        i = i + 1
+        if path[i] == "'":
+            in_string = not in_string
+        if in_string:
+            # slashes within strings are not step separators
+            continue
+        if path[i] == '/':
+            if i > 0:
+                separators.append((last_position, i))
+            if (path[i+1] == '/'):
+                last_position = i
+                i = i + 1
+            else:
+                last_position = i + 1
+    separators.append((last_position, len(path)))
+
+    steps = []
+    for start, end in separators:
+        steps.append(path[start:end])
+    return steps
+
+
+class Path:
+    """A location path.
+    """
+
+    def __init__(self, path, parse=True):
+        self.path = path
+        self.steps = []
+        if parse:
+            if (path[0] == '/') and (path[1] != '/'):
+                # if not on the descendant axis, remove the leading slash
+                path = path[1:]
+            steps = tokenize_path(path)
+            for step in steps:
+                self.steps.append(PathStep(step))
+
+    def apply(self, node):
+        """Apply the path to a node. Return the resulting list of nodes.
+
+        Apply the steps in the path sequentially by sending the output of each
+        step as input to the next step.
+        """
+        # FIXME: this should return a node SET, not a node LIST
+        # or at least a list with no duplicates
+        if self.path[0] == '/':
+            # for an absolute path, start from the root
+            if not isinstance(node, BeautifulSoup.Tag) \
+               or (node.name != '[document]'):
+                node = node.findParent('[document]')
+        nodes = [node]
+        for step in self.steps:
+            nodes = step.apply(nodes)
+        return nodes
+
+
+class PathStep:
+    """A location step in a location path.
+    """
+
+    AXIS_PATTERN          = r"""(%s)::|@""" % '|'.join(AXES)
+    NODE_TEST_PATTERN     = r"""\w+(\(\))?"""
+    PREDICATE_PATTERN     = r"""\[(.*?)\]"""
+    LOCATION_STEP_PATTERN = r"""(%s)?(%s)((%s)*)""" \
+                          % (AXIS_PATTERN, NODE_TEST_PATTERN, PREDICATE_PATTERN)
+
+    _re_location_step = re.compile(LOCATION_STEP_PATTERN)
+
+    PREDICATE_NOT_PATTERN = r"""not\((.*?)\)"""
+    PREDICATE_AXIS_PATTERN = r"""(%s)?(%s)(='(.*?)')?""" \
+                           % (AXIS_PATTERN, NODE_TEST_PATTERN)
+    PREDICATE_FUNCTION_PATTERN = r"""(%s)\(([^,]+(,\s*[^,]+)*)?\)(=(.*))?""" \
+                               % '|'.join(XPATH_FUNCTIONS)
+
+    _re_predicate_not = re.compile(PREDICATE_NOT_PATTERN)
+    _re_predicate_axis = re.compile(PREDICATE_AXIS_PATTERN)
+    _re_predicate_function = re.compile(PREDICATE_FUNCTION_PATTERN)
+
+    def __init__(self, step):
+        self.step = step
+        if (step == '.') or (step == '..'):
+            return
+
+        if step[:2] == '//':
+            default_axis = AXIS_DESCENDANT
+            step = step[2:]
+        else:
+            default_axis = AXIS_CHILD
+
+        step_match = self._re_location_step.match(step)
+
+        # determine the axis
+        axis = step_match.group(1)
+        if axis is None:
+            self.axis = default_axis
+        elif axis == '@':
+            self.axis = AXIS_ATTRIBUTE
+        else:
+            self.axis = step_match.group(2)
+
+        self.soup_args = {}
+        self.index = None
+
+        self.node_test = step_match.group(3)
+        if self.node_test == 'text()':
+            self.soup_args['text'] = True
+        else:
+            self.soup_args['name'] = self.node_test
+
+        self.checkers = []
+        predicates = step_match.group(5)
+        if predicates is not None:
+            predicates = [p for p in predicates[1:-1].split('][') if p]
+            for predicate in predicates:
+                checker = self.__parse_predicate(predicate)
+                if checker is not None:
+                    self.checkers.append(checker)
+
+    def __parse_predicate(self, predicate):
+        """Parse the predicate. Return a callable that can be used to filter
+        nodes. Update `self.soup_args` to take advantage of BeautifulSoup search
+        features.
+        """
+        try:
+            position = int(predicate)
+            if self.axis == AXIS_DESCENDANT:
+                return PredicateFilter('position', value=position)
+            else:
+                # use the search limit feature instead of a checker
+                self.soup_args['limit'] = position
+                self.index = position - 1
+                return None
+        except ValueError:
+            pass
+
+        if predicate == "last()":
+            self.index = -1
+            return None
+
+        negate = self._re_predicate_not.match(predicate)
+        if negate:
+            predicate = negate.group(1)
+
+        function_match = self._re_predicate_function.match(predicate)
+        if function_match:
+            name = function_match.group(1)
+            arguments = function_match.group(2)
+            value = function_match.group(4)
+            if value is not None:
+                value = function_match.group(5)
+            return PredicateFilter(name, arguments, value)
+
+        axis_match = self._re_predicate_axis.match(predicate)
+        if axis_match:
+            axis = axis_match.group(1)
+            if axis is None:
+                axis = AXIS_CHILD
+            elif axis == '@':
+                axis = AXIS_ATTRIBUTE
+            if axis == AXIS_ATTRIBUTE:
+                # use the attribute search feature instead of a checker
+                attribute_name = axis_match.group(3)
+                if axis_match.group(5) is not None:
+                    attribute_value = axis_match.group(6)
+                elif not negate:
+                    attribute_value = True
+                else:
+                    attribute_value = None
+                if not self.soup_args.has_key('attrs'):
+                    self.soup_args['attrs'] = {}
+                self.soup_args['attrs'][attribute_name] = attribute_value
+                return None
+            elif axis == AXIS_CHILD:
+                node_test = axis_match.group(3)
+                node_value = axis_match.group(6)
+                return PredicateFilter('axis', node_test, value=node_value,
+                                       negate=negate)
+
+        raise NotImplementedError("This predicate is not implemented")
+
+    def apply(self, nodes):
+        """Apply the step to a list of nodes. Return the list of nodes for the
+        next step.
+        """
+        if self.step == '.':
+            return nodes
+        elif self.step == '..':
+            return [node.parent for node in nodes]
+
+        result = []
+        for node in nodes:
+            if self.axis == AXIS_CHILD:
+                found = node.findAll(recursive=False, **self.soup_args)
+            elif self.axis == AXIS_DESCENDANT:
+                found = node.findAll(recursive=True, **self.soup_args)
+            elif self.axis == AXIS_ATTRIBUTE:
+                try:
+                    found = [node[self.node_test]]
+                except KeyError:
+                    found = []
+            elif self.axis == AXIS_FOLLOWING_SIBLING:
+                found = node.findNextSiblings(**self.soup_args)
+            elif self.axis == AXIS_PRECEDING_SIBLING:
+                # TODO: make sure that the result is reverse ordered
+                found = node.findPreviousSiblings(**self.soup_args)
+            elif self.axis == AXIS_FOLLOWING:
+                # find the last descendant of this node
+                last = node
+                while (not isinstance(last, BeautifulSoup.NavigableString)) \
+                      and (len(last.contents) > 0):
+                    last = last.contents[-1]
+                found = last.findAllNext(**self.soup_args)
+            elif self.axis == AXIS_ANCESTOR:
+                found = node.findParents(**self.soup_args)
+
+            # this should only be active if there is a position predicate
+            # and the axis is not 'descendant'
+            if self.index is not None:
+                if found:
+                    if len(found) > self.index:
+                        found = [found[self.index]]
+                    else:
+                        found = []
+
+            if found:
+                for checker in self.checkers:
+                    found = filter(checker, found)
+                result.extend(found)
+
+        return result
+
+
+class PredicateFilter:
+    """A callable class for filtering nodes.
+    """
+
+    def __init__(self, name, arguments=None, value=None, negate=False):
+        self.name = name
+        self.arguments = arguments
+        self.negate = negate
+
+        if name == 'position':
+            self.__filter = self.__position
+            self.value = value
+        elif name == 'axis':
+            self.__filter = self.__axis
+            self.node_test = arguments
+            self.value = value
+        elif name == 'starts-with':
+            self.__filter = self.__starts_with
+            args = map(string.strip, arguments.split(','))
+            if args[0][0] == '@':
+                self.arguments = (True, args[0][1:], args[1][1:-1])
+            else:
+                self.arguments = (False, args[0], args[1][1:-1])
+        elif name == 'string-length':
+            self.__filter = self.__string_length
+            args = map(string.strip, arguments.split(','))
+            if args[0][0] == '@':
+                self.arguments = (True, args[0][1:])
+            else:
+                self.arguments = (False, args[0])
+            self.value = int(value)
+        else:
+            raise NotImplementedError("This XPath function is not implemented")
+
+    def __call__(self, node):
+        if self.negate:
+            return not self.__filter(node)
+        else:
+            return self.__filter(node)
+
+    def __position(self, node):
+        if isinstance(node, BeautifulSoup.NavigableString):
+            actual_position = len(node.findPreviousSiblings(text=True)) + 1
+        else:
+            actual_position = len(node.findPreviousSiblings(node.name)) + 1
+        return actual_position == self.value
+
+    def __axis(self, node):
+        if self.node_test == 'text()':
+            return node.string == self.value
+        else:
+            children = node.findAll(self.node_test, recursive=False)
+            if len(children) > 0 and self.value is None:
+                return True
+            for child in children:
+                if child.string == self.value:
+                    return True
+            return False
+
+    def __starts_with(self, node):
+        if self.arguments[0]:
+            # this is an attribute
+            attribute_name = self.arguments[1]
+            if node.has_key(attribute_name):
+                first = node[attribute_name]
+                return first.startswith(self.arguments[2])
+        elif self.arguments[1] == 'text()':
+            first = node.contents[0]
+            if isinstance(first, BeautifulSoup.NavigableString):
+                return first.startswith(self.arguments[2])
+        return False
+
+    def __string_length(self, node):
+        if self.arguments[0]:
+            # this is an attribute
+            attribute_name = self.arguments[1]
+            if node.has_key(attribute_name):
+                value = node[attribute_name]
+            else:
+                value = None
+        elif self.arguments[1] == 'text()':
+            value = node.string
+        if value is not None:
+            return len(value) == self.value
+        return False
+
+
+_paths = {}
+_steps = {}
+
+def get_path(path):
+    """Utility for eliminating repeated parsings of the same paths and steps.
+    """
+    if not _paths.has_key(path):
+        p = Path(path, parse=False)
+        steps = tokenize_path(path)
+        for step in steps:
+            if not _steps.has_key(step):
+                _steps[step] = PathStep(step)
+            p.steps.append(_steps[step])
+        _paths[path] = p
+    return _paths[path]

libs/imdb/parser/http/bsouplxml/etree.py

@@ -0,0 +1,75 @@
+"""
+parser.http.bsouplxml.etree module (imdb.parser.http package).
+
+This module adapts the beautifulsoup interface to lxml.etree module.
+
+Copyright 2008 H. Turgut Uyar <uyar@tekir.org>
+          2008 Davide Alberani <da@erlug.linux.it>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import _bsoup as BeautifulSoup
+from _bsoup import Tag as Element
+
+import bsoupxpath
+
+# Not directly used by IMDbPY, but do not remove: it's used by IMDbPYKit,
+# for example.
+def fromstring(xml_string):
+    """Return a DOM representation of the string."""
+    # We try to not use BeautifulSoup.BeautifulStoneSoup.XML_ENTITIES,
+    # for convertEntities.
+    return BeautifulSoup.BeautifulStoneSoup(xml_string,
+                        convertEntities=None).findChild(True)
+
+
+def tostring(element, encoding=None, pretty_print=False):
+    """Return a string or unicode representation of an element."""
+    if encoding is unicode:
+        encoding = None
+    # For BeautifulSoup 3.1
+    #encArgs = {'prettyPrint': pretty_print}
+    #if encoding is not None:
+    #    encArgs['encoding'] = encoding
+    #return element.encode(**encArgs)
+    return element.__str__(encoding, pretty_print)
+
+def setattribute(tag, name, value):
+    tag[name] = value
+
+def xpath(node, expr):
+    """Apply an xpath expression to a node. Return a list of nodes."""
+    #path = bsoupxpath.Path(expr)
+    path = bsoupxpath.get_path(expr)
+    return path.apply(node)
+
+
+# XXX: monkey patching the beautifulsoup tag class
+class _EverythingIsNestable(dict):
+    """"Fake that every tag is nestable."""
+    def get(self, key, *args, **kwds):
+        return []
+
+BeautifulSoup.BeautifulStoneSoup.NESTABLE_TAGS = _EverythingIsNestable()
+BeautifulSoup.Tag.tag = property(fget=lambda self: self.name)
+BeautifulSoup.Tag.attrib = property(fget=lambda self: self)
+BeautifulSoup.Tag.text = property(fget=lambda self: self.string)
+BeautifulSoup.Tag.set = setattribute
+BeautifulSoup.Tag.getparent = lambda self: self.parent
+BeautifulSoup.Tag.drop_tree = BeautifulSoup.Tag.extract
+BeautifulSoup.Tag.xpath = xpath
+
+# TODO: setting the text attribute for tags

libs/imdb/parser/http/bsouplxml/html.py

@@ -0,0 +1,31 @@
+"""
+parser.http.bsouplxml.html module (imdb.parser.http package).
+
+This module adapts the beautifulsoup interface to lxml.html module.
+
+Copyright 2008 H. Turgut Uyar <uyar@tekir.org>
+          2008 Davide Alberani <da@erlug.linux.it>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import _bsoup as BeautifulSoup
+
+
+def fromstring(html_string):
+    """Return a DOM representation of the string."""
+    return BeautifulSoup.BeautifulSoup(html_string,
+        convertEntities=BeautifulSoup.BeautifulSoup.HTML_ENTITIES
+        ).findChild(True)

libs/imdb/parser/http/characterParser.py

@@ -0,0 +1,203 @@
+"""
+parser.http.characterParser module (imdb package).
+
+This module provides the classes (and the instances), used to parse
+the IMDb pages on the akas.imdb.com server about a character.
+E.g., for "Jesse James" the referred pages would be:
+    main details:   http://www.imdb.com/character/ch0000001/
+    biography:      http://www.imdb.com/character/ch0000001/bio
+    ...and so on...
+
+Copyright 2007-2009 Davide Alberani <da@erlug.linux.it>
+               2008 H. Turgut Uyar <uyar@tekir.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import re
+from utils import Attribute, Extractor, DOMParserBase, build_movie, \
+                    analyze_imdbid
+from personParser import DOMHTMLMaindetailsParser
+
+from imdb.Movie import Movie
+
+_personIDs = re.compile(r'/name/nm([0-9]{7})')
+class DOMHTMLCharacterMaindetailsParser(DOMHTMLMaindetailsParser):
+    """Parser for the "filmography" page of a given character.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        bparser = DOMHTMLCharacterMaindetailsParser()
+        result = bparser.parse(character_biography_html_string)
+    """
+    _containsObjects = True
+
+    _film_attrs = [Attribute(key=None,
+                      multi=True,
+                      path={
+                          'link': "./a[1]/@href",
+                          'title': ".//text()",
+                          'status': "./i/a//text()",
+                          'roleID': "./a/@href"
+                          },
+                      postprocess=lambda x:
+                          build_movie(x.get('title') or u'',
+                              movieID=analyze_imdbid(x.get('link') or u''),
+                              roleID=_personIDs.findall(x.get('roleID') or u''),
+                              status=x.get('status') or None,
+                              _parsingCharacter=True))]
+
+    extractors = [
+            Extractor(label='title',
+                        path="//title",
+                        attrs=Attribute(key='name',
+                            path="./text()",
+                            postprocess=lambda x: \
+                                    x.replace(' (Character)', '').replace(
+                                        '- Filmography by type', '').strip())),
+
+            Extractor(label='headshot',
+                        path="//a[@name='headshot']",
+                        attrs=Attribute(key='headshot',
+                            path="./img/@src")),
+
+            Extractor(label='akas',
+                        path="//div[h5='Alternate Names:']",
+                        attrs=Attribute(key='akas',
+                            path="./div//text()",
+                            postprocess=lambda x: x.strip().split(' / '))),
+
+            Extractor(label='filmography',
+                        path="//div[@class='filmo'][not(h5)]/ol/li",
+                        attrs=_film_attrs),
+
+            Extractor(label='filmography sections',
+                        group="//div[@class='filmo'][h5]",
+                        group_key="./h5/a/text()",
+                        group_key_normalize=lambda x: x.lower()[:-1],
+                        path="./ol/li",
+                        attrs=_film_attrs),
+            ]
+
+    preprocessors = [
+            # Check that this doesn't cut "status"...
+            (re.compile(r'<br>(\.\.\.|   ).+?</li>', re.I | re.M), '</li>')]
+
+
+class DOMHTMLCharacterBioParser(DOMParserBase):
+    """Parser for the "biography" page of a given character.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        bparser = DOMHTMLCharacterBioParser()
+        result = bparser.parse(character_biography_html_string)
+    """
+    _defGetRefs = True
+
+    extractors = [
+            Extractor(label='introduction',
+                        path="//div[@id='_intro']",
+                        attrs=Attribute(key='introduction',
+                            path=".//text()",
+                            postprocess=lambda x: x.strip())),
+
+            Extractor(label='biography',
+                        path="//span[@class='_biography']",
+                        attrs=Attribute(key='biography',
+                            multi=True,
+                            path={
+                                'info': "./preceding-sibling::h4[1]//text()",
+                                'text': ".//text()"
+                            },
+                            postprocess=lambda x: u'%s: %s' % (
+                                x.get('info').strip(),
+                                x.get('text').replace('\n',
+                                    ' ').replace('||', '\n\n').strip()))),
+    ]
+
+    preprocessors = [
+        (re.compile('(<div id="swiki.2.3.1">)', re.I), r'\1<div id="_intro">'),
+        (re.compile('(<a name="history">)\s*(<table .*?</table>)',
+                    re.I | re.DOTALL),
+         r'</div>\2\1</a>'),
+        (re.compile('(<a name="[^"]+">)(<h4>)', re.I), r'</span>\1</a>\2'),
+        (re.compile('(</h4>)</a>', re.I), r'\1<span class="_biography">'),
+        (re.compile('<br/><br/>', re.I), r'||'),
+        (re.compile('\|\|\n', re.I), r'</span>'),
+        ]
+
+
+class DOMHTMLCharacterQuotesParser(DOMParserBase):
+    """Parser for the "quotes" page of a given character.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        qparser = DOMHTMLCharacterQuotesParser()
+        result = qparser.parse(character_quotes_html_string)
+    """
+    _defGetRefs = True
+
+    extractors = [
+        Extractor(label='charquotes',
+                    group="//h5",
+                    group_key="./a/text()",
+                    path="./following-sibling::div[1]",
+                    attrs=Attribute(key=None,
+                        path={'txt': ".//text()",
+                              'movieID': ".//a[1]/@href"},
+                        postprocess=lambda x: (analyze_imdbid(x['movieID']),
+                                    x['txt'].strip().replace(':   ',
+                                    ': ').replace(':  ', ': ').split('||'))))
+    ]
+
+    preprocessors = [
+        (re.compile('(</h5>)', re.I), r'\1<div>'),
+        (re.compile('\s*<br/><br/>\s*', re.I), r'||'),
+        (re.compile('\|\|\s*(<hr/>)', re.I), r'</div>\1'),
+        (re.compile('\s*<br/>\s*', re.I), r'::')
+        ]
+
+    def postprocess_data(self, data):
+        if not data:
+            return {}
+        newData = {}
+        for title in data:
+            movieID, quotes = data[title]
+            if movieID is None:
+                movie = title
+            else:
+                movie = Movie(title=title, movieID=movieID,
+                              accessSystem=self._as, modFunct=self._modFunct)
+            newData[movie] = [quote.split('::') for quote in quotes]
+        return {'quotes': newData}
+
+
+from personParser import DOMHTMLSeriesParser
+
+_OBJECTS = {
+    'character_main_parser': ((DOMHTMLCharacterMaindetailsParser,),
+                                {'kind': 'character'}),
+    'character_series_parser': ((DOMHTMLSeriesParser,), None),
+    'character_bio_parser': ((DOMHTMLCharacterBioParser,), None),
+    'character_quotes_parser': ((DOMHTMLCharacterQuotesParser,), None)
+}
+
+

libs/imdb/parser/http/companyParser.py

@@ -0,0 +1,91 @@
+"""
+parser.http.companyParser module (imdb package).
+
+This module provides the classes (and the instances), used to parse
+the IMDb pages on the akas.imdb.com server about a company.
+E.g., for "Columbia Pictures [us]" the referred page would be:
+    main details:   http://akas.imdb.com/company/co0071509/
+
+Copyright 2008-2009 Davide Alberani <da@erlug.linux.it>
+          2008 H. Turgut Uyar <uyar@tekir.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import re
+from utils import build_movie, Attribute, Extractor, DOMParserBase, \
+                    analyze_imdbid
+
+from imdb.utils import analyze_company_name
+
+
+class DOMCompanyParser(DOMParserBase):
+    """Parser for the main page of a given company.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        cparser = DOMCompanyParser()
+        result = cparser.parse(company_html_string)
+    """
+    _containsObjects = True
+
+    extractors = [
+            Extractor(label='name',
+                        path="//title",
+                        attrs=Attribute(key='name',
+                            path="./text()",
+                        postprocess=lambda x: \
+                                analyze_company_name(x, stripNotes=True))),
+
+            Extractor(label='filmography',
+                        group="//b/a[@name]",
+                        group_key="./text()",
+                        group_key_normalize=lambda x: x.lower(),
+                        path="../following-sibling::ol[1]/li",
+                        attrs=Attribute(key=None,
+                            multi=True,
+                            path={
+                                'link': "./a[1]/@href",
+                                'title': "./a[1]/text()",
+                                'year': "./text()[1]"
+                                },
+                            postprocess=lambda x:
+                                build_movie(u'%s %s' % \
+                                (x.get('title'), x.get('year').strip()),
+                                movieID=analyze_imdbid(x.get('link') or u''),
+                                _parsingCompany=True))),
+            ]
+
+    preprocessors = [
+        (re.compile('(<b><a name=)', re.I), r'</p>\1')
+        ]
+
+    def postprocess_data(self, data):
+        for key in data.keys():
+            new_key = key.replace('company', 'companies')
+            new_key = new_key.replace('other', 'miscellaneous')
+            new_key = new_key.replace('distributor', 'distributors')
+            if new_key != key:
+                data[new_key] = data[key]
+                del data[key]
+        return data
+
+
+_OBJECTS = {
+    'company_main_parser': ((DOMCompanyParser,), None)
+}
+

libs/imdb/parser/http/personParser.py

@@ -0,0 +1,509 @@
+"""
+parser.http.personParser module (imdb package).
+
+This module provides the classes (and the instances), used to parse
+the IMDb pages on the akas.imdb.com server about a person.
+E.g., for "Mel Gibson" the referred pages would be:
+    categorized:    http://akas.imdb.com/name/nm0000154/maindetails
+    biography:      http://akas.imdb.com/name/nm0000154/bio
+    ...and so on...
+
+Copyright 2004-2010 Davide Alberani <da@erlug.linux.it>
+               2008 H. Turgut Uyar <uyar@tekir.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import re
+from imdb.Movie import Movie
+from imdb.utils import analyze_name, canonicalName, normalizeName, \
+                        analyze_title, date_and_notes
+from utils import build_movie, DOMParserBase, Attribute, Extractor, \
+                        analyze_imdbid
+
+
+from movieParser import _manageRoles
+_reRoles = re.compile(r'(<li>.*? \.\.\.\. )(.*?)(</li>|<br>)',
+                        re.I | re.M | re.S)
+
+def build_date(date):
+    day = date.get('day')
+    year = date.get('year')
+    if day and year:
+        return "%s %s" % (day, year)
+    if day:
+        return day
+    if year:
+        return year
+    return ""
+
+class DOMHTMLMaindetailsParser(DOMParserBase):
+    """Parser for the "categorized" (maindetails) page of a given person.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        cparser = DOMHTMLMaindetailsParser()
+        result = cparser.parse(categorized_html_string)
+    """
+    _containsObjects = True
+
+    _birth_attrs = [Attribute(key='birth date',
+                        path={
+                            'day': "./div/a[starts-with(@href, " \
+                                    "'/date/')]/text()",
+                            'year': "./div/a[starts-with(@href, " \
+                                    "'/search/name?birth_year=')]/text()"
+                            },
+                        postprocess=build_date),
+                    Attribute(key='birth notes',
+                        path="./div/a[starts-with(@href, " \
+                                "'/search/name?birth_place=')]/text()")]
+    _death_attrs = [Attribute(key='death date',
+                        path={
+                            'day': "./div/a[starts-with(@href, " \
+                                    "'/date/')]/text()",
+                            'year': "./div/a[starts-with(@href, " \
+                                    "'/search/name?death_date=')]/text()"
+                            },
+                        postprocess=build_date),
+                    Attribute(key='death notes',
+                        path="./div/text()",
+                        # TODO: check if this slicing is always correct
+                        postprocess=lambda x: x.strip()[2:])]
+    _film_attrs = [Attribute(key=None,
+                      multi=True,
+                      path={
+                          'link': "./a[1]/@href",
+                          'title': ".//text()",
+                          'status': "./i/a//text()",
+                          'roleID': "./div[@class='_imdbpyrole']/@roleid"
+                          },
+                      postprocess=lambda x:
+                          build_movie(x.get('title') or u'',
+                              movieID=analyze_imdbid(x.get('link') or u''),
+                              roleID=(x.get('roleID') or u'').split('/'),
+                              status=x.get('status') or None))]
+
+    extractors = [
+            Extractor(label='page title',
+                        path="//title",
+                        attrs=Attribute(key='name',
+                            path="./text()",
+                            postprocess=lambda x: analyze_name(x,
+                                                            canonical=1))),
+
+            Extractor(label='birth info',
+                        path="//div[h5='Date of Birth:']",
+                        attrs=_birth_attrs),
+
+            Extractor(label='death info',
+                        path="//div[h5='Date of Death:']",
+                        attrs=_death_attrs),
+
+            Extractor(label='headshot',
+                        path="//a[@name='headshot']",
+                        attrs=Attribute(key='headshot',
+                            path="./img/@src")),
+
+            Extractor(label='akas',
+                        path="//div[h5='Alternate Names:']",
+                        attrs=Attribute(key='akas',
+                            path="./div/text()",
+                            postprocess=lambda x: x.strip().split(' | '))),
+
+            Extractor(label='filmography',
+                        group="//div[@class='filmo'][h5]",
+                        group_key="./h5/a[@name]/text()",
+                        group_key_normalize=lambda x: x.lower()[:-1],
+                        path="./ol/li",
+                        attrs=_film_attrs)
+            ]
+    preprocessors = [
+            # XXX: check that this doesn't cut "status" or other info...
+            (re.compile(r'<br>(\.\.\.|    ?).+?</li>', re.I | re.M | re.S),
+                '</li>'),
+            (_reRoles, _manageRoles)]
+
+    def postprocess_data(self, data):
+        for what in 'birth date', 'death date':
+            if what in data and not data[what]:
+                del data[what]
+        return data
+
+
+class DOMHTMLBioParser(DOMParserBase):
+    """Parser for the "biography" page of a given person.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        bioparser = DOMHTMLBioParser()
+        result = bioparser.parse(biography_html_string)
+    """
+    _defGetRefs = True
+
+    _birth_attrs = [Attribute(key='birth date',
+                        path={
+                            'day': "./a[starts-with(@href, " \
+                                    "'/date/')]/text()",
+                            'year': "./a[starts-with(@href, " \
+                                    "'/search/name?birth_year=')]/text()"
+                            },
+                        postprocess=build_date),
+                    Attribute(key='birth notes',
+                        path="./a[starts-with(@href, " \
+                                "'/search/name?birth_place=')]/text()")]
+    _death_attrs = [Attribute(key='death date',
+                        path={
+                            'day': "./a[starts-with(@href, " \
+                                    "'/date/')]/text()",
+                            'year': "./a[starts-with(@href, " \
+                                    "'/search/name?death_date=')]/text()"
+                            },
+                        postprocess=build_date),
+                    Attribute(key='death notes',
+                        path="./text()",
+                        # TODO: check if this slicing is always correct
+                        postprocess=lambda x: u''.join(x).strip()[2:])]
+    extractors = [
+            Extractor(label='birth info',
+                        path="//div[h5='Date of Birth']",
+                        attrs=_birth_attrs),
+            Extractor(label='death info',
+                        path="//div[h5='Date of Death']",
+                        attrs=_death_attrs),
+            Extractor(label='nick names',
+                        path="//div[h5='Nickname']",
+                        attrs=Attribute(key='nick names',
+                            path="./text()",
+                            joiner='|',
+                            postprocess=lambda x: [n.strip().replace(' (',
+                                    '::(', 1) for n in x.split('|')
+                                    if n.strip()])),
+            Extractor(label='birth name',
+                        path="//div[h5='Birth Name']",
+                        attrs=Attribute(key='birth name',
+                            path="./text()",
+                            postprocess=lambda x: canonicalName(x.strip()))),
+            Extractor(label='height',
+                        path="//div[h5='Height']",
+                        attrs=Attribute(key='height',
+                            path="./text()",
+                            postprocess=lambda x: x.strip())),
+            Extractor(label='mini biography',
+                        path="//div[h5='Mini Biography']",
+                        attrs=Attribute(key='mini biography',
+                            multi=True,
+                            path={
+                                'bio': "./p//text()",
+                                'by': "./b/following-sibling::a/text()"
+                                },
+                            postprocess=lambda x: "%s::%s" % \
+                                (x.get('bio').strip(),
+                                (x.get('by') or u'').strip() or u'Anonymous'))),
+            Extractor(label='spouse',
+                        path="//div[h5='Spouse']/table/tr",
+                        attrs=Attribute(key='spouse',
+                            multi=True,
+                            path={
+                                'name': "./td[1]//text()",
+                                'info': "./td[2]//text()"
+                                },
+                            postprocess=lambda x: ("%s::%s" % \
+                                (x.get('name').strip(),
+                                (x.get('info') or u'').strip())).strip(':'))),
+            Extractor(label='trade mark',
+                        path="//div[h5='Trade Mark']/p",
+                        attrs=Attribute(key='trade mark',
+                            multi=True,
+                            path=".//text()",
+                            postprocess=lambda x: x.strip())),
+            Extractor(label='trivia',
+                        path="//div[h5='Trivia']/p",
+                        attrs=Attribute(key='trivia',
+                            multi=True,
+                            path=".//text()",
+                            postprocess=lambda x: x.strip())),
+            Extractor(label='quotes',
+                        path="//div[h5='Personal Quotes']/p",
+                        attrs=Attribute(key='quotes',
+                            multi=True,
+                            path=".//text()",
+                            postprocess=lambda x: x.strip())),
+            Extractor(label='salary',
+                        path="//div[h5='Salary']/table/tr",
+                        attrs=Attribute(key='salary history',
+                            multi=True,
+                            path={
+                                'title': "./td[1]//text()",
+                                'info': "./td[2]/text()",
+                                },
+                            postprocess=lambda x: "%s::%s" % \
+                                    (x.get('title').strip(),
+                                        x.get('info').strip()))),
+            Extractor(label='where now',
+                        path="//div[h5='Where Are They Now']/p",
+                        attrs=Attribute(key='where now',
+                            multi=True,
+                            path=".//text()",
+                            postprocess=lambda x: x.strip())),
+            ]
+
+    preprocessors = [
+        (re.compile('(<h5>)', re.I), r'</div><div class="_imdbpy">\1'),
+        (re.compile('(</table>\n</div>\s+)</div>', re.I + re.DOTALL), r'\1'),
+        (re.compile('(<div id="tn15bot">)'), r'</div>\1'),
+        (re.compile('\.<br><br>([^\s])', re.I), r'. \1')
+        ]
+
+    def postprocess_data(self, data):
+        for what in 'birth date', 'death date':
+            if what in data and not data[what]:
+                del data[what]
+        return data
+
+
+class DOMHTMLOtherWorksParser(DOMParserBase):
+    """Parser for the "other works" and "agent" pages of a given person.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        owparser = DOMHTMLOtherWorksParser()
+        result = owparser.parse(otherworks_html_string)
+    """
+    _defGetRefs = True
+    kind = 'other works'
+
+    # XXX: looks like the 'agent' page is no more public.
+    extractors = [
+            Extractor(label='other works',
+                        path="//h5[text()='Other works']/" \
+                                "following-sibling::div[1]",
+                        attrs=Attribute(key='self.kind',
+                            path=".//text()",
+                            postprocess=lambda x: x.strip().split('\n\n')))
+            ]
+
+    preprocessors = [
+        (re.compile('(<h5>[^<]+</h5>)', re.I),
+            r'</div>\1<div class="_imdbpy">'),
+        (re.compile('(</table>\n</div>\s+)</div>', re.I), r'\1'),
+        (re.compile('(<div id="tn15bot">)'), r'</div>\1'),
+        (re.compile('<br/><br/>', re.I), r'\n\n')
+        ]
+
+
+def _build_episode(link, title, minfo, role, roleA, roleAID):
+    """Build an Movie object for a given episode of a series."""
+    episode_id = analyze_imdbid(link)
+    notes = u''
+    minidx = minfo.find(' -')
+    # Sometimes, for some unknown reason, the role is left in minfo.
+    if minidx != -1:
+        slfRole = minfo[minidx+3:].lstrip()
+        minfo = minfo[:minidx].rstrip()
+        if slfRole.endswith(')'):
+            commidx = slfRole.rfind('(')
+            if commidx != -1:
+                notes = slfRole[commidx:]
+                slfRole = slfRole[:commidx]
+        if slfRole and role is None and roleA is None:
+            role = slfRole
+    eps_data = analyze_title(title)
+    eps_data['kind'] = u'episode'
+    # FIXME: it's wrong for multiple characters (very rare on tv series?).
+    if role is None:
+        role = roleA # At worse, it's None.
+    if role is None:
+        roleAID = None
+    if roleAID is not None:
+        roleAID = analyze_imdbid(roleAID)
+    e = Movie(movieID=episode_id, data=eps_data, currentRole=role,
+            roleID=roleAID, notes=notes)
+    # XXX: are we missing some notes?
+    # XXX: does it parse things as "Episode dated 12 May 2005 (12 May 2005)"?
+    if minfo.startswith('('):
+        pe = minfo.find(')')
+        if pe != -1:
+            date = minfo[1:pe]
+            if date != '????':
+                e['original air date'] = date
+                if eps_data.get('year', '????') == '????':
+                    syear = date.split()[-1]
+                    if syear.isdigit():
+                        e['year'] = int(syear)
+    return e
+
+
+class DOMHTMLSeriesParser(DOMParserBase):
+    """Parser for the "by TV series" page of a given person.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        sparser = DOMHTMLSeriesParser()
+        result = sparser.parse(filmoseries_html_string)
+    """
+    _containsObjects = True
+
+    extractors = [
+            Extractor(label='series',
+                        group="//div[@class='filmo']/span[1]",
+                        group_key="./a[1]",
+                        path="./following-sibling::ol[1]/li/a[1]",
+                        attrs=Attribute(key=None,
+                            multi=True,
+                            path={
+                                'link': "./@href",
+                                'title': "./text()",
+                                'info': "./following-sibling::text()",
+                                'role': "./following-sibling::i[1]/text()",
+                                'roleA': "./following-sibling::a[1]/text()",
+                                'roleAID': "./following-sibling::a[1]/@href"
+                                },
+                            postprocess=lambda x: _build_episode(x.get('link'),
+                                x.get('title'),
+                                (x.get('info') or u'').strip(),
+                                x.get('role'),
+                                x.get('roleA'),
+                                x.get('roleAID'))))
+            ]
+
+    def postprocess_data(self, data):
+        if len(data) == 0:
+            return {}
+        nd = {}
+        for key in data.keys():
+            dom = self.get_dom(key)
+            link = self.xpath(dom, "//a/@href")[0]
+            title = self.xpath(dom, "//a/text()")[0][1:-1]
+            series = Movie(movieID=analyze_imdbid(link),
+                           data=analyze_title(title),
+                           accessSystem=self._as, modFunct=self._modFunct)
+            nd[series] = []
+            for episode in data[key]:
+                # XXX: should we create a copy of 'series', to avoid
+                #      circular references?
+                episode['episode of'] = series
+                nd[series].append(episode)
+        return {'episodes': nd}
+
+
+class DOMHTMLPersonGenresParser(DOMParserBase):
+    """Parser for the "by genre" and "by keywords" pages of a given person.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        gparser = DOMHTMLPersonGenresParser()
+        result = gparser.parse(bygenre_html_string)
+    """
+    kind = 'genres'
+    _containsObjects = True
+
+    extractors = [
+            Extractor(label='genres',
+                        group="//b/a[@name]/following-sibling::a[1]",
+                        group_key="./text()",
+                        group_key_normalize=lambda x: x.lower(),
+                        path="../../following-sibling::ol[1]/li//a[1]",
+                        attrs=Attribute(key=None,
+                            multi=True,
+                            path={
+                                'link': "./@href",
+                                'title': "./text()",
+                                'info': "./following-sibling::text()"
+                                },
+                            postprocess=lambda x: \
+                                    build_movie(x.get('title') + \
+                                    x.get('info').split('[')[0],
+                                    analyze_imdbid(x.get('link')))))
+            ]
+
+    def postprocess_data(self, data):
+        if len(data) == 0:
+            return {}
+        return {self.kind: data}
+
+
+from movieParser import _parse_merchandising_link
+
+class DOMHTMLPersonSalesParser(DOMParserBase):
+    """Parser for the "merchandising links" page of a given person.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        sparser = DOMHTMLPersonSalesParser()
+        result = sparser.parse(sales_html_string)
+    """
+    extractors = [
+        Extractor(label='merchandising links',
+                    group="//span[@class='merch_title']",
+                    group_key=".//text()",
+                    path="./following-sibling::table[1]/" \
+                            "/td[@class='w_rowtable_colshop']//tr[1]",
+                    attrs=Attribute(key=None,
+                        multi=True,
+                        path={
+                            'link': "./td[2]/a[1]/@href",
+                            'text': "./td[1]/img[1]/@alt",
+                            'cover': "./ancestor::td[1]/../" \
+                                    "td[1]/a[1]/img[1]/@src",
+                            },
+                        postprocess=_parse_merchandising_link)),
+    ]
+
+    preprocessors = [
+        (re.compile('(<a name="[^"]+" )/>', re.I), r'\1></a>')
+        ]
+
+    def postprocess_data(self, data):
+        if len(data) == 0:
+            return {}
+        return {'merchandising links': data}
+
+
+from movieParser import DOMHTMLTechParser
+from movieParser import DOMHTMLOfficialsitesParser
+from movieParser import DOMHTMLAwardsParser
+from movieParser import DOMHTMLNewsParser
+
+
+_OBJECTS = {
+    'maindetails_parser': ((DOMHTMLMaindetailsParser,), None),
+    'bio_parser': ((DOMHTMLBioParser,), None),
+    'otherworks_parser': ((DOMHTMLOtherWorksParser,), None),
+    #'agent_parser': ((DOMHTMLOtherWorksParser,), {'kind': 'agent'}),
+    'person_officialsites_parser': ((DOMHTMLOfficialsitesParser,), None),
+    'person_awards_parser': ((DOMHTMLAwardsParser,), {'subject': 'name'}),
+    'publicity_parser': ((DOMHTMLTechParser,), {'kind': 'publicity'}),
+    'person_series_parser': ((DOMHTMLSeriesParser,), None),
+    'person_contacts_parser': ((DOMHTMLTechParser,), {'kind': 'contacts'}),
+    'person_genres_parser': ((DOMHTMLPersonGenresParser,), None),
+    'person_keywords_parser': ((DOMHTMLPersonGenresParser,),
+                                {'kind': 'keywords'}),
+    'news_parser': ((DOMHTMLNewsParser,), None),
+    'sales_parser': ((DOMHTMLPersonSalesParser,), None)
+}
+

libs/imdb/parser/http/searchCharacterParser.py

@@ -0,0 +1,69 @@
+"""
+parser.http.searchCharacterParser module (imdb package).
+
+This module provides the HTMLSearchCharacterParser class (and the
+search_character_parser instance), used to parse the results of a search
+for a given character.
+E.g., when searching for the name "Jesse James", the parsed page would be:
+    http://akas.imdb.com/find?s=Characters;mx=20;q=Jesse+James
+
+Copyright 2007-2009 Davide Alberani <da@erlug.linux.it>
+               2008 H. Turgut Uyar <uyar@tekir.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+from imdb.utils import analyze_name, build_name
+from utils import Extractor, Attribute, analyze_imdbid
+
+from searchMovieParser import DOMHTMLSearchMovieParser, DOMBasicMovieParser
+
+
+class DOMBasicCharacterParser(DOMBasicMovieParser):
+    """Simply get the name of a character and the imdbID.
+
+    It's used by the DOMHTMLSearchCharacterParser class to return a result
+    for a direct match (when a search on IMDb results in a single
+    character, the web server sends directly the movie page."""
+    _titleFunct = lambda self, x: analyze_name(x or u'', canonical=False)
+
+
+class DOMHTMLSearchCharacterParser(DOMHTMLSearchMovieParser):
+    _BaseParser = DOMBasicCharacterParser
+    _notDirectHitTitle = '<title>imdb search'
+    _titleBuilder = lambda self, x: build_name(x, canonical=False)
+    _linkPrefix = '/character/ch'
+
+    _attrs = [Attribute(key='data',
+                        multi=True,
+                        path={
+                            'link': "./a[1]/@href",
+                            'name': "./a[1]/text()"
+                            },
+                        postprocess=lambda x: (
+                            analyze_imdbid(x.get('link') or u''),
+                            {'name': x.get('name')}
+                        ))]
+    extractors = [Extractor(label='search',
+                            path="//td[3]/a[starts-with(@href, " \
+                                    "'/character/ch')]/..",
+                            attrs=_attrs)]
+
+
+_OBJECTS = {
+        'search_character_parser': ((DOMHTMLSearchCharacterParser,),
+                {'kind': 'character', '_basic_parser': DOMBasicCharacterParser})
+}
+

libs/imdb/parser/http/searchCompanyParser.py

@@ -0,0 +1,71 @@
+"""
+parser.http.searchCompanyParser module (imdb package).
+
+This module provides the HTMLSearchCompanyParser class (and the
+search_company_parser instance), used to parse the results of a search
+for a given company.
+E.g., when searching for the name "Columbia Pictures", the parsed page would be:
+    http://akas.imdb.com/find?s=co;mx=20;q=Columbia+Pictures
+
+Copyright 2008-2009 Davide Alberani <da@erlug.linux.it>
+          2008 H. Turgut Uyar <uyar@tekir.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+from imdb.utils import analyze_company_name, build_company_name
+from utils import Extractor, Attribute, analyze_imdbid
+
+from searchMovieParser import DOMHTMLSearchMovieParser, DOMBasicMovieParser
+
+class DOMBasicCompanyParser(DOMBasicMovieParser):
+    """Simply get the name of a company and the imdbID.
+
+    It's used by the DOMHTMLSearchCompanyParser class to return a result
+    for a direct match (when a search on IMDb results in a single
+    company, the web server sends directly the company page.
+    """
+    _titleFunct = lambda self, x: analyze_company_name(x or u'')
+
+
+class DOMHTMLSearchCompanyParser(DOMHTMLSearchMovieParser):
+    _BaseParser = DOMBasicCompanyParser
+    _notDirectHitTitle = '<title>imdb company'
+    _titleBuilder = lambda self, x: build_company_name(x)
+    _linkPrefix = '/company/co'
+
+    _attrs = [Attribute(key='data',
+                        multi=True,
+                        path={
+                            'link': "./a[1]/@href",
+                            'name': "./a[1]/text()",
+                            'notes': "./text()[1]"
+                            },
+                        postprocess=lambda x: (
+                            analyze_imdbid(x.get('link')),
+                            analyze_company_name(x.get('name')+(x.get('notes')
+                                                or u''), stripNotes=True)
+                        ))]
+    extractors = [Extractor(label='search',
+                            path="//td[3]/a[starts-with(@href, " \
+                                    "'/company/co')]/..",
+                            attrs=_attrs)]
+
+
+_OBJECTS = {
+        'search_company_parser': ((DOMHTMLSearchCompanyParser,),
+                {'kind': 'company', '_basic_parser': DOMBasicCompanyParser})
+}
+

libs/imdb/parser/http/searchKeywordParser.py

@@ -0,0 +1,111 @@
+"""
+parser.http.searchKeywordParser module (imdb package).
+
+This module provides the HTMLSearchKeywordParser class (and the
+search_company_parser instance), used to parse the results of a search
+for a given keyword.
+E.g., when searching for the keyword "alabama", the parsed page would be:
+    http://akas.imdb.com/find?s=kw;mx=20;q=alabama
+
+Copyright 2009 Davide Alberani <da@erlug.linux.it>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+from utils import Extractor, Attribute, analyze_imdbid
+from imdb.utils import analyze_title, analyze_company_name
+
+from searchMovieParser import DOMHTMLSearchMovieParser, DOMBasicMovieParser
+
+class DOMBasicKeywordParser(DOMBasicMovieParser):
+    """Simply get the name of a keyword.
+
+    It's used by the DOMHTMLSearchKeywordParser class to return a result
+    for a direct match (when a search on IMDb results in a single
+    keyword, the web server sends directly the keyword page.
+    """
+    # XXX: it's still to be tested!
+    # I'm not even sure there can be a direct hit, searching for keywords.
+    _titleFunct = lambda self, x: analyze_company_name(x or u'')
+
+
+class DOMHTMLSearchKeywordParser(DOMHTMLSearchMovieParser):
+    """Parse the html page that the IMDb web server shows when the
+    "new search system" is used, searching for keywords similar to
+    the one given."""
+
+    _BaseParser = DOMBasicKeywordParser
+    _notDirectHitTitle = '<title>imdb keyword'
+    _titleBuilder = lambda self, x: x
+    _linkPrefix = '/keyword/'
+
+    _attrs = [Attribute(key='data',
+                        multi=True,
+                        path="./a[1]/text()"
+                            )]
+    extractors = [Extractor(label='search',
+                            path="//td[3]/a[starts-with(@href, " \
+                                    "'/keyword/')]/..",
+                            attrs=_attrs)]
+
+
+def custom_analyze_title4kwd(title, yearNote, outline):
+    """Return a dictionary with the needed info."""
+    title = title.strip()
+    if not title:
+        return {}
+    if yearNote:
+        yearNote = '%s)' % yearNote.split(' ')[0]
+        title = title + ' ' + yearNote
+    retDict = analyze_title(title)
+    if outline:
+        retDict['plot outline'] = outline
+    return retDict
+
+
+class DOMHTMLSearchMovieKeywordParser(DOMHTMLSearchMovieParser):
+    """Parse the html page that the IMDb web server shows when the
+    "new search system" is used, searching for movies with the given
+    keyword."""
+
+    _notDirectHitTitle = '<title>best'
+
+    _attrs = [Attribute(key='data',
+                        multi=True,
+                        path={
+                            'link': "./a[1]/@href",
+                            'info': "./a[1]//text()",
+                            'ynote': "./span[@class='desc']/text()",
+                            'outline': "./span[@class='outline']//text()"
+                            },
+                        postprocess=lambda x: (
+                            analyze_imdbid(x.get('link') or u''),
+                            custom_analyze_title4kwd(x.get('info') or u'',
+                                                    x.get('ynote') or u'',
+                                                    x.get('outline') or u'')
+                        ))]
+
+    extractors = [Extractor(label='search',
+                            path="//td[3]/a[starts-with(@href, " \
+                                    "'/title/tt')]/..",
+                            attrs=_attrs)]
+
+
+_OBJECTS = {
+        'search_keyword_parser': ((DOMHTMLSearchKeywordParser,),
+                {'kind': 'keyword', '_basic_parser': DOMBasicKeywordParser}),
+        'search_moviekeyword_parser': ((DOMHTMLSearchMovieKeywordParser,), None)
+}
+

libs/imdb/parser/http/searchMovieParser.py

@@ -0,0 +1,178 @@
+"""
+parser.http.searchMovieParser module (imdb package).
+
+This module provides the HTMLSearchMovieParser class (and the
+search_movie_parser instance), used to parse the results of a search
+for a given title.
+E.g., for when searching for the title "the passion", the parsed
+page would be:
+    http://akas.imdb.com/find?q=the+passion&tt=on&mx=20
+
+Copyright 2004-2010 Davide Alberani <da@erlug.linux.it>
+               2008 H. Turgut Uyar <uyar@tekir.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import re
+from imdb.utils import analyze_title, build_title
+from utils import DOMParserBase, Attribute, Extractor, analyze_imdbid
+
+
+class DOMBasicMovieParser(DOMParserBase):
+    """Simply get the title of a movie and the imdbID.
+
+    It's used by the DOMHTMLSearchMovieParser class to return a result
+    for a direct match (when a search on IMDb results in a single
+    movie, the web server sends directly the movie page."""
+    # Stay generic enough to be used also for other DOMBasic*Parser classes.
+    _titleAttrPath = ".//text()"
+    _linkPath = "//link[@rel='canonical']"
+    _titleFunct = lambda self, x: analyze_title(x or u'')
+
+    def _init(self):
+        self.preprocessors += [('<span class="tv-extra">TV mini-series</span>',
+                                '<span class="tv-extra">(mini)</span>')]
+        self.extractors = [Extractor(label='title',
+                                path="//h1",
+                                attrs=Attribute(key='title',
+                                                path=self._titleAttrPath,
+                                                postprocess=self._titleFunct)),
+                            Extractor(label='link',
+                                path=self._linkPath,
+                                attrs=Attribute(key='link', path="./@href",
+                                postprocess=lambda x: \
+                                        analyze_imdbid((x or u'').replace(
+                                            'http://pro.imdb.com', ''))
+                                    ))]
+
+    # Remove 'More at IMDb Pro' links.
+    preprocessors = [(re.compile(r'<span class="pro-link".*?</span>'), ''),
+            (re.compile(r'<a href="http://ad.doubleclick.net.*?;id=(co[0-9]{7});'), r'<a href="http://pro.imdb.com/company/\1"></a>< a href="')]
+
+    def postprocess_data(self, data):
+        if not 'link' in data:
+            data = []
+        else:
+            link = data.pop('link')
+            if (link and data):
+                data = [(link, data)]
+            else:
+                data = []
+        return data
+
+
+def custom_analyze_title(title):
+    """Remove garbage notes after the (year), (year/imdbIndex) or (year) (TV)"""
+    # XXX: very crappy. :-(
+    nt = title.split('    ')[0]
+    if nt:
+        title = nt
+    if not title:
+        return {}
+    return analyze_title(title)
+
+# Manage AKAs.
+_reAKAStitles = re.compile(r'(?:aka) <em>"(.*?)(<br>|<\/td>)', re.I | re.M)
+
+class DOMHTMLSearchMovieParser(DOMParserBase):
+    """Parse the html page that the IMDb web server shows when the
+    "new search system" is used, for movies."""
+
+    _BaseParser = DOMBasicMovieParser
+    _notDirectHitTitle = '<title>imdb title'
+    _titleBuilder = lambda self, x: build_title(x)
+    _linkPrefix = '/title/tt'
+
+    _attrs = [Attribute(key='data',
+                        multi=True,
+                        path={
+                            'link': "./a[1]/@href",
+                            'info': ".//text()",
+                            #'akas': ".//div[@class='_imdbpyAKA']//text()"
+                            'akas': ".//p[@class='find-aka']//text()"
+                            },
+                        postprocess=lambda x: (
+                            analyze_imdbid(x.get('link') or u''),
+                            custom_analyze_title(x.get('info') or u''),
+                            x.get('akas')
+                        ))]
+    extractors = [Extractor(label='search',
+                        path="//td[3]/a[starts-with(@href, '/title/tt')]/..",
+                        attrs=_attrs)]
+    def _init(self):
+        self.url = u''
+
+    def _reset(self):
+        self.url = u''
+
+    def preprocess_string(self, html_string):
+        if self._notDirectHitTitle in html_string[:1024].lower():
+            if self._linkPrefix == '/title/tt':
+                # Only for movies.
+                html_string = html_string.replace('(TV mini-series)', '(mini)')
+                html_string = html_string.replace('<p class="find-aka">',
+                        '<p class="find-aka">::')
+                #html_string = _reAKAStitles.sub(
+                #        r'<div class="_imdbpyAKA">\1::</div>\2', html_string)
+            return html_string
+        # Direct hit!
+        dbme = self._BaseParser(useModule=self._useModule)
+        res = dbme.parse(html_string, url=self.url)
+        if not res: return u''
+        res = res['data']
+        if not (res and res[0]): return u''
+        link = '%s%s' % (self._linkPrefix, res[0][0])
+        #    # Tries to cope with companies for which links to pro.imdb.com
+        #    # are missing.
+        #    link = self.url.replace(imdbURL_base[:-1], '')
+        title = self._titleBuilder(res[0][1])
+        if not (link and title): return u''
+        link = link.replace('http://pro.imdb.com', '')
+        new_html = '<td></td><td></td><td><a href="%s">%s</a></td>' % (link,
+                                                                    title)
+        return new_html
+
+    def postprocess_data(self, data):
+        if not data.has_key('data'):
+            data['data'] = []
+        results = getattr(self, 'results', None)
+        if results is not None:
+            data['data'][:] = data['data'][:results]
+        # Horrible hack to support AKAs.
+        if data and data['data'] and len(data['data'][0]) == 3 and \
+                isinstance(data['data'][0], tuple):
+            for idx, datum in enumerate(data['data']):
+                if not isinstance(datum, tuple):
+                    continue
+                if datum[2] is not None:
+                    akas = filter(None, datum[2].split('::'))
+                    if self._linkPrefix == '/title/tt':
+                        akas = [a.replace('" - ', '::').rstrip() for a in akas]
+                        akas = [a.replace('aka "', '', 1).lstrip() for a in akas]
+                    datum[1]['akas'] = akas
+                    data['data'][idx] = (datum[0], datum[1])
+                else:
+                    data['data'][idx] = (datum[0], datum[1])
+        return data
+
+    def add_refs(self, data):
+        return data
+
+
+_OBJECTS = {
+        'search_movie_parser': ((DOMHTMLSearchMovieParser,), None)
+}
+

libs/imdb/parser/http/searchPersonParser.py

@@ -0,0 +1,92 @@
+"""
+parser.http.searchPersonParser module (imdb package).
+
+This module provides the HTMLSearchPersonParser class (and the
+search_person_parser instance), used to parse the results of a search
+for a given person.
+E.g., when searching for the name "Mel Gibson", the parsed page would be:
+    http://akas.imdb.com/find?q=Mel+Gibson&nm=on&mx=20
+
+Copyright 2004-2010 Davide Alberani <da@erlug.linux.it>
+               2008 H. Turgut Uyar <uyar@tekir.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import re
+from imdb.utils import analyze_name, build_name
+from utils import Extractor, Attribute, analyze_imdbid
+
+from searchMovieParser import DOMHTMLSearchMovieParser, DOMBasicMovieParser
+
+
+def _cleanName(n):
+    """Clean the name in a title tag."""
+    if not n:
+        return u''
+    n = n.replace('Filmography by type for', '') # FIXME: temporary.
+    return n
+
+class DOMBasicPersonParser(DOMBasicMovieParser):
+    """Simply get the name of a person and the imdbID.
+
+    It's used by the DOMHTMLSearchPersonParser class to return a result
+    for a direct match (when a search on IMDb results in a single
+    person, the web server sends directly the movie page."""
+    _titleFunct = lambda self, x: analyze_name(_cleanName(x), canonical=1)
+
+
+_reAKASp = re.compile(r'(?:aka|birth name) (<em>")(.*?)"(<br>|<\/em>|<\/td>)',
+                    re.I | re.M)
+
+class DOMHTMLSearchPersonParser(DOMHTMLSearchMovieParser):
+    """Parse the html page that the IMDb web server shows when the
+    "new search system" is used, for persons."""
+    _BaseParser = DOMBasicPersonParser
+    _notDirectHitTitle = '<title>imdb name'
+    _titleBuilder = lambda self, x: build_name(x, canonical=True)
+    _linkPrefix = '/name/nm'
+
+    _attrs = [Attribute(key='data',
+                        multi=True,
+                        path={
+                            'link': "./a[1]/@href",
+                            'name': "./a[1]/text()",
+                            'index': "./text()[1]",
+                            'akas': ".//div[@class='_imdbpyAKA']/text()"
+                            },
+                        postprocess=lambda x: (
+                            analyze_imdbid(x.get('link') or u''),
+                            analyze_name((x.get('name') or u'') + \
+                                        (x.get('index') or u''),
+                                         canonical=1), x.get('akas')
+                        ))]
+    extractors = [Extractor(label='search',
+                            path="//td[3]/a[starts-with(@href, '/name/nm')]/..",
+                            attrs=_attrs)]
+
+    def preprocess_string(self, html_string):
+        if self._notDirectHitTitle in html_string[:1024].lower():
+            html_string = _reAKASp.sub(
+                                    r'\1<div class="_imdbpyAKA">\2::</div>\3',
+                                    html_string)
+        return DOMHTMLSearchMovieParser.preprocess_string(self, html_string)
+
+
+_OBJECTS = {
+        'search_person_parser': ((DOMHTMLSearchPersonParser,),
+                    {'kind': 'person', '_basic_parser': DOMBasicPersonParser})
+}
+

libs/imdb/parser/http/topBottomParser.py

@@ -0,0 +1,106 @@
+"""
+parser.http.topBottomParser module (imdb package).
+
+This module provides the classes (and the instances), used to parse the
+lists of top 250 and bottom 100 movies.
+E.g.:
+    http://akas.imdb.com/chart/top
+    http://akas.imdb.com/chart/bottom
+
+Copyright 2009 Davide Alberani <da@erlug.linux.it>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+from imdb.utils import analyze_title
+from utils import DOMParserBase, Attribute, Extractor, analyze_imdbid
+
+
+class DOMHTMLTop250Parser(DOMParserBase):
+    """Parser for the "top 250" page.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        tparser = DOMHTMLTop250Parser()
+        result = tparser.parse(top250_html_string)
+    """
+    label = 'top 250'
+    ranktext = 'top 250 rank'
+
+    def _init(self):
+        self.extractors = [Extractor(label=self.label,
+                        path="//div[@id='main']//table//tr",
+                        attrs=Attribute(key=None,
+                                multi=True,
+                                path={self.ranktext: "./td[1]//text()",
+                                        'rating': "./td[2]//text()",
+                                        'title': "./td[3]//text()",
+                                        'movieID': "./td[3]//a/@href",
+                                        'votes': "./td[4]//text()"
+                                        }))]
+
+    def postprocess_data(self, data):
+        if not data or self.label not in data:
+            return []
+        mlist = []
+        data = data[self.label]
+        # Avoid duplicates.  A real fix, using XPath, is auspicabile.
+        # XXX: probably this is no more needed.
+        seenIDs = []
+        for d in data:
+            if 'movieID' not in d: continue
+            if self.ranktext not in d: continue
+            if 'title' not in d: continue
+            theID = analyze_imdbid(d['movieID'])
+            if theID is None:
+                continue
+            theID = str(theID)
+            if theID in seenIDs:
+                continue
+            seenIDs.append(theID)
+            minfo = analyze_title(d['title'])
+            try: minfo[self.ranktext] = int(d[self.ranktext].replace('.', ''))
+            except: pass
+            if 'votes' in d:
+                try: minfo['votes'] = int(d['votes'].replace(',', ''))
+                except: pass
+            if 'rating' in d:
+                try: minfo['rating'] = float(d['rating'])
+                except: pass
+            mlist.append((theID, minfo))
+        return mlist
+
+
+class DOMHTMLBottom100Parser(DOMHTMLTop250Parser):
+    """Parser for the "bottom 100" page.
+    The page should be provided as a string, as taken from
+    the akas.imdb.com server.  The final result will be a
+    dictionary, with a key for every relevant section.
+
+    Example:
+        tparser = DOMHTMLBottom100Parser()
+        result = tparser.parse(bottom100_html_string)
+    """
+    label = 'bottom 100'
+    ranktext = 'bottom 100 rank'
+
+
+_OBJECTS = {
+    'top250_parser':  ((DOMHTMLTop250Parser,), None),
+    'bottom100_parser':  ((DOMHTMLBottom100Parser,), None)
+}
+

libs/imdb/parser/http/utils.py

@@ -0,0 +1,817 @@
+"""
+parser.http.utils module (imdb package).
+
+This module provides miscellaneous utilities used by
+the imdb.parser.http classes.
+
+Copyright 2004-2010 Davide Alberani <da@erlug.linux.it>
+               2008 H. Turgut Uyar <uyar@tekir.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import re
+import logging
+
+from imdb._exceptions import IMDbError
+
+from imdb.utils import flatten, _Container
+from imdb.Movie import Movie
+from imdb.Person import Person
+from imdb.Character import Character
+
+
+# Year, imdbIndex and kind.
+re_yearKind_index = re.compile(r'(\([0-9\?]{4}(?:/[IVXLCDM]+)?\)(?: \(mini\)| \(TV\)| \(V\)| \(VG\))?)')
+
+# Match imdb ids in href tags
+re_imdbid = re.compile(r'(title/tt|name/nm|character/ch|company/co)([0-9]+)')
+
+def analyze_imdbid(href):
+    """Return an imdbID from an URL."""
+    if not href:
+        return None
+    match = re_imdbid.search(href)
+    if not match:
+        return None
+    return str(match.group(2))
+
+
+_modify_keys = list(Movie.keys_tomodify_list) + list(Person.keys_tomodify_list)
+def _putRefs(d, re_titles, re_names, re_characters, lastKey=None):
+    """Iterate over the strings inside list items or dictionary values,
+    substitutes movie titles and person names with the (qv) references."""
+    if isinstance(d, list):
+        for i in xrange(len(d)):
+            if isinstance(d[i], (unicode, str)):
+                if lastKey in _modify_keys:
+                    if re_names:
+                        d[i] = re_names.sub(ur"'\1' (qv)", d[i])
+                    if re_titles:
+                        d[i] = re_titles.sub(ur'_\1_ (qv)', d[i])
+                    if re_characters:
+                        d[i] = re_characters.sub(ur'#\1# (qv)', d[i])
+            elif isinstance(d[i], (list, dict)):
+                _putRefs(d[i], re_titles, re_names, re_characters,
+                        lastKey=lastKey)
+    elif isinstance(d, dict):
+        for k, v in d.items():
+            lastKey = k
+            if isinstance(v, (unicode, str)):
+                if lastKey in _modify_keys:
+                    if re_names:
+                        d[k] = re_names.sub(ur"'\1' (qv)", v)
+                    if re_titles:
+                        d[k] = re_titles.sub(ur'_\1_ (qv)', v)
+                    if re_characters:
+                        d[k] = re_characters.sub(ur'#\1# (qv)', v)
+            elif isinstance(v, (list, dict)):
+                _putRefs(d[k], re_titles, re_names, re_characters,
+                        lastKey=lastKey)
+
+
+# Handle HTML/XML/SGML entities.
+from htmlentitydefs import entitydefs
+entitydefs = entitydefs.copy()
+entitydefsget = entitydefs.get
+entitydefs['nbsp'] = ' '
+
+sgmlentity = {'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''}
+sgmlentityget = sgmlentity.get
+_sgmlentkeys = sgmlentity.keys()
+
+entcharrefs = {}
+entcharrefsget = entcharrefs.get
+for _k, _v in entitydefs.items():
+    if _k in _sgmlentkeys: continue
+    if _v[0:2] == '&#':
+        dec_code = _v[1:-1]
+        _v = unichr(int(_v[2:-1]))
+        entcharrefs[dec_code] = _v
+    else:
+        dec_code = '#' + str(ord(_v))
+        _v = unicode(_v, 'latin_1', 'replace')
+        entcharrefs[dec_code] = _v
+    entcharrefs[_k] = _v
+del _sgmlentkeys, _k, _v
+entcharrefs['#160'] = u' '
+entcharrefs['#xA0'] = u' '
+entcharrefs['#xa0'] = u' '
+entcharrefs['#XA0'] = u' '
+entcharrefs['#x22'] = u'"'
+entcharrefs['#X22'] = u'"'
+# convert &x26; to &amp;, to make BeautifulSoup happy; beware that this
+# leaves lone '&' in the html broken, but I assume this is better than
+# the contrary...
+entcharrefs['#38'] = u'&amp;'
+entcharrefs['#x26'] = u'&amp;'
+entcharrefs['#x26'] = u'&amp;'
+
+re_entcharrefs = re.compile('&(%s|\#160|\#\d{1,5}|\#x[0-9a-f]{1,4});' %
+                            '|'.join(map(re.escape, entcharrefs)), re.I)
+re_entcharrefssub = re_entcharrefs.sub
+
+sgmlentity.update(dict([('#34', u'"'), ('#38', u'&'),
+                        ('#60', u'<'), ('#62', u'>'), ('#39', u"'")]))
+re_sgmlref = re.compile('&(%s);' % '|'.join(map(re.escape, sgmlentity)))
+re_sgmlrefsub = re_sgmlref.sub
+
+# Matches XML-only single tags, like <br/> ; they are invalid in HTML,
+# but widely used by IMDb web site. :-/
+re_xmltags = re.compile('<([a-zA-Z]+)/>')
+
+
+def _replXMLRef(match):
+    """Replace the matched XML/HTML entities and references;
+    replace everything except sgml entities like &lt;, &gt;, ..."""
+    ref = match.group(1)
+    value = entcharrefsget(ref)
+    if value is None:
+        if ref[0] == '#':
+            ref_code = ref[1:]
+            if ref_code in ('34', '38', '60', '62', '39'):
+                return match.group(0)
+            elif ref_code[0].lower() == 'x':
+                #if ref[2:] == '26':
+                #    # Don't convert &x26; to &amp;, to make BeautifulSoup happy.
+                #    return '&amp;'
+                return unichr(int(ref[2:], 16))
+            else:
+                return unichr(int(ref[1:]))
+        else:
+            return ref
+    return value
+
+def subXMLRefs(s):
+    """Return the given html string with entity and char references
+    replaced."""
+    return re_entcharrefssub(_replXMLRef, s)
+
+# XXX: no more used here; move it to mobile (they are imported by helpers, too)?
+def _replSGMLRefs(match):
+    """Replace the matched SGML entity."""
+    ref = match.group(1)
+    return sgmlentityget(ref, ref)
+
+def subSGMLRefs(s):
+    """Return the given html string with sgml entity and char references
+    replaced."""
+    return re_sgmlrefsub(_replSGMLRefs, s)
+
+
+_b_p_logger = logging.getLogger('imdbpy.parser.http.build_person')
+def build_person(txt, personID=None, billingPos=None,
+                roleID=None, accessSystem='http', modFunct=None):
+    """Return a Person instance from the tipical <tr>...</tr> strings
+    found in the IMDb's web site."""
+    #if personID is None
+    #    _b_p_logger.debug('empty name or personID for "%s"', txt)
+    notes = u''
+    role = u''
+    # Search the (optional) separator between name and role/notes.
+    if txt.find('....') != -1:
+        sep = '....'
+    elif txt.find('...') != -1:
+        sep = '...'
+    else:
+        sep = '...'
+        # Replace the first parenthesis, assuming there are only
+        # notes, after.
+        # Rationale: no imdbIndex is (ever?) showed on the web site.
+        txt = txt.replace('(', '...(', 1)
+    txt_split = txt.split(sep, 1)
+    name = txt_split[0].strip()
+    if len(txt_split) == 2:
+        role_comment = txt_split[1].strip()
+        # Strip common endings.
+        if role_comment[-4:] == ' and':
+            role_comment = role_comment[:-4].rstrip()
+        elif role_comment[-2:] == ' &':
+            role_comment = role_comment[:-2].rstrip()
+        elif role_comment[-6:] == '& ....':
+            role_comment = role_comment[:-6].rstrip()
+        # Get the notes.
+        if roleID is not None:
+            if not isinstance(roleID, list):
+                cmt_idx = role_comment.find('(')
+                if cmt_idx != -1:
+                    role = role_comment[:cmt_idx].rstrip()
+                    notes = role_comment[cmt_idx:]
+                else:
+                    # Just a role, without notes.
+                    role = role_comment
+            else:
+                role = role_comment
+        else:
+            # We're managing something that doesn't have a 'role', so
+            # everything are notes.
+            notes = role_comment
+    if role == '....': role = u''
+    roleNotes = []
+    # Manages multiple roleIDs.
+    if isinstance(roleID, list):
+        rolesplit = role.split('/')
+        role = []
+        for r in rolesplit:
+            nidx = r.find('(')
+            if nidx != -1:
+                role.append(r[:nidx].rstrip())
+                roleNotes.append(r[nidx:])
+            else:
+                role.append(r)
+                roleNotes.append(None)
+        lr = len(role)
+        lrid = len(roleID)
+        if lr > lrid:
+            roleID += [None] * (lrid - lr)
+        elif lr < lrid:
+            roleID = roleID[:lr]
+        for i, rid in enumerate(roleID):
+            if rid is not None:
+                roleID[i] = str(rid)
+        if lr == 1:
+            role = role[0]
+            roleID = roleID[0]
+    elif roleID is not None:
+        roleID = str(roleID)
+    if personID is not None:
+        personID = str(personID)
+    if (not name) or (personID is None):
+        # Set to 'debug', since build_person is expected to receive some crap.
+        _b_p_logger.debug('empty name or personID for "%s"', txt)
+    # XXX: return None if something strange is detected?
+    person = Person(name=name, personID=personID, currentRole=role,
+                    roleID=roleID, notes=notes, billingPos=billingPos,
+                    modFunct=modFunct, accessSystem=accessSystem)
+    if roleNotes and len(roleNotes) == len(roleID):
+        for idx, role in enumerate(person.currentRole):
+            if roleNotes[idx]:
+                role.notes = roleNotes[idx]
+    return person
+
+
+_b_m_logger = logging.getLogger('imdbpy.parser.http.build_movie')
+# To shrink spaces.
+re_spaces = re.compile(r'\s+')
+def build_movie(txt, movieID=None, roleID=None, status=None,
+                accessSystem='http', modFunct=None, _parsingCharacter=False,
+                _parsingCompany=False):
+    """Given a string as normally seen on the "categorized" page of
+    a person on the IMDb's web site, returns a Movie instance."""
+    if _parsingCharacter:
+        _defSep = ' Played by '
+    elif _parsingCompany:
+        _defSep = ' ... '
+    else:
+        _defSep = ' .... '
+    title = re_spaces.sub(' ', txt).strip()
+    # Split the role/notes from the movie title.
+    tsplit = title.split(_defSep, 1)
+    role = u''
+    notes = u''
+    roleNotes = []
+    if len(tsplit) == 2:
+        title = tsplit[0].rstrip()
+        role = tsplit[1].lstrip()
+    if title[-9:] == 'TV Series':
+        title = title[:-9].rstrip()
+    elif title[-14:] == 'TV mini-series':
+        title = title[:-14] + ' (mini)'
+    # Try to understand where the movie title ends.
+    while True:
+        if title[-1:] != ')':
+            # Ignore the silly "TV Series" notice.
+            if title[-9:] == 'TV Series':
+                title = title[:-9].rstrip()
+                continue
+            else:
+                # Just a title: stop here.
+                break
+        # Try to match paired parentheses; yes: sometimes there are
+        # parentheses inside comments...
+        nidx = title.rfind('(')
+        while (nidx != -1 and \
+                    title[nidx:].count('(') != title[nidx:].count(')')):
+            nidx = title[:nidx].rfind('(')
+        # Unbalanced parentheses: stop here.
+        if nidx == -1: break
+        # The last item in parentheses seems to be a year: stop here.
+        first4 = title[nidx+1:nidx+5]
+        if (first4.isdigit() or first4 == '????') and \
+                title[nidx+5:nidx+6] in (')', '/'): break
+        # The last item in parentheses is a known kind: stop here.
+        if title[nidx+1:-1] in ('TV', 'V', 'mini', 'VG'): break
+        # Else, in parentheses there are some notes.
+        # XXX: should the notes in the role half be kept separated
+        #      from the notes in the movie title half?
+        if notes: notes = '%s %s' % (title[nidx:], notes)
+        else: notes = title[nidx:]
+        title = title[:nidx].rstrip()
+    if _parsingCharacter and roleID and not role:
+        roleID = None
+    if not roleID:
+        roleID = None
+    elif len(roleID) == 1:
+        roleID = roleID[0]
+    # Manages multiple roleIDs.
+    if isinstance(roleID, list):
+        tmprole = role.split('/')
+        role = []
+        for r in tmprole:
+            nidx = r.find('(')
+            if nidx != -1:
+                role.append(r[:nidx].rstrip())
+                roleNotes.append(r[nidx:])
+            else:
+                role.append(r)
+                roleNotes.append(None)
+        lr = len(role)
+        lrid = len(roleID)
+        if lr > lrid:
+            roleID += [None] * (lrid - lr)
+        elif lr < lrid:
+            roleID = roleID[:lr]
+        for i, rid in enumerate(roleID):
+            if rid is not None:
+                roleID[i] = str(rid)
+        if lr == 1:
+            role = role[0]
+            roleID = roleID[0]
+    elif roleID is not None:
+        roleID = str(roleID)
+    if movieID is not None:
+        movieID = str(movieID)
+    if (not title) or (movieID is None):
+        _b_m_logger.error('empty title or movieID for "%s"', txt)
+    m = Movie(title=title, movieID=movieID, notes=notes, currentRole=role,
+                roleID=roleID, roleIsPerson=_parsingCharacter,
+                modFunct=modFunct, accessSystem=accessSystem)
+    if roleNotes and len(roleNotes) == len(roleID):
+        for idx, role in enumerate(m.currentRole):
+            if roleNotes[idx]:
+                role.notes = roleNotes[idx]
+    # Status can't be checked here, and must be detected by the parser.
+    if status:
+        m['status'] = status
+    return m
+
+
+class DOMParserBase(object):
+    """Base parser to handle HTML data from the IMDb's web server."""
+    _defGetRefs = False
+    _containsObjects = False
+
+    preprocessors = []
+    extractors = []
+    usingModule = None
+
+    _logger = logging.getLogger('imdbpy.parser.http.domparser')
+
+    def __init__(self, useModule=None):
+        """Initialize the parser. useModule can be used to force it
+        to use 'BeautifulSoup' or 'lxml'; by default, it's auto-detected,
+        using 'lxml' if available and falling back to 'BeautifulSoup'
+        otherwise."""
+        # Module to use.
+        if useModule is None:
+            useModule = ('lxml', 'BeautifulSoup')
+        if not isinstance(useModule, (tuple, list)):
+            useModule = [useModule]
+        self._useModule = useModule
+        nrMods = len(useModule)
+        _gotError = False
+        for idx, mod in enumerate(useModule):
+            mod = mod.strip().lower()
+            try:
+                if mod == 'lxml':
+                    from lxml.html import fromstring
+                    from lxml.etree import tostring
+                    self._is_xml_unicode = False
+                    self.usingModule = 'lxml'
+                elif mod == 'beautifulsoup':
+                    from bsouplxml.html import fromstring
+                    from bsouplxml.etree import tostring
+                    self._is_xml_unicode = True
+                    self.usingModule = 'beautifulsoup'
+                else:
+                    self._logger.warn('unknown module "%s"' % mod)
+                    continue
+                self.fromstring = fromstring
+                self._tostring = tostring
+                if _gotError:
+                    self._logger.warn('falling back to "%s"' % mod)
+                break
+            except ImportError, e:
+                if idx+1 >= nrMods:
+                    # Raise the exception, if we don't have any more
+                    # options to try.
+                    raise IMDbError, 'unable to use any parser in %s: %s' % \
+                                    (str(useModule), str(e))
+                else:
+                    self._logger.warn('unable to use "%s": %s' % (mod, str(e)))
+                    _gotError = True
+                continue
+        else:
+            raise IMDbError, 'unable to use parsers in %s' % str(useModule)
+        # Fall-back defaults.
+        self._modFunct = None
+        self._as = 'http'
+        self._cname = self.__class__.__name__
+        self._init()
+        self.reset()
+
+    def reset(self):
+        """Reset the parser."""
+        # Names and titles references.
+        self._namesRefs = {}
+        self._titlesRefs = {}
+        self._charactersRefs = {}
+        self._reset()
+
+    def _init(self):
+        """Subclasses can override this method, if needed."""
+        pass
+
+    def _reset(self):
+        """Subclasses can override this method, if needed."""
+        pass
+
+    def parse(self, html_string, getRefs=None, **kwds):
+        """Return the dictionary generated from the given html string;
+        getRefs can be used to force the gathering of movies/persons/characters
+        references."""
+        self.reset()
+        if getRefs is not None:
+            self.getRefs = getRefs
+        else:
+            self.getRefs = self._defGetRefs
+        # Useful only for the testsuite.
+        if not isinstance(html_string, unicode):
+            html_string = unicode(html_string, 'latin_1', 'replace')
+        html_string = subXMLRefs(html_string)
+        # Temporary fix: self.parse_dom must work even for empty strings.
+        html_string = self.preprocess_string(html_string)
+        html_string = html_string.strip()
+        # tag attributes like title="&#x22;Family Guy&#x22;" will be
+        # converted to title=""Family Guy"" and this confuses BeautifulSoup.
+        if self.usingModule == 'beautifulsoup':
+            html_string = html_string.replace('""', '"')
+        if html_string:
+            dom = self.get_dom(html_string)
+            try:
+                dom = self.preprocess_dom(dom)
+            except Exception, e:
+                self._logger.error('%s: caught exception preprocessing DOM',
+                                    self._cname, exc_info=True)
+            if self.getRefs:
+                try:
+                    self.gather_refs(dom)
+                except Exception, e:
+                    self._logger.warn('%s: unable to gather refs: %s',
+                                    self._cname, exc_info=True)
+            data = self.parse_dom(dom)
+        else:
+            data = {}
+        try:
+            data = self.postprocess_data(data)
+        except Exception, e:
+            self._logger.error('%s: caught exception postprocessing data',
+                                self._cname, exc_info=True)
+        if self._containsObjects:
+            self.set_objects_params(data)
+        data = self.add_refs(data)
+        return data
+
+    def _build_empty_dom(self):
+        from bsouplxml import _bsoup
+        return _bsoup.BeautifulSoup('')
+
+    def get_dom(self, html_string):
+        """Return a dom object, from the given string."""
+        try:
+            dom = self.fromstring(html_string)
+            if dom is None:
+                dom = self._build_empty_dom()
+                self._logger.error('%s: using a fake empty DOM', self._cname)
+            return dom
+        except Exception, e:
+            self._logger.error('%s: caught exception parsing DOM',
+                                self._cname, exc_info=True)
+            return self._build_empty_dom()
+
+    def xpath(self, element, path):
+        """Return elements matching the given XPath."""
+        try:
+            xpath_result = element.xpath(path)
+            if self._is_xml_unicode:
+                return xpath_result
+            result = []
+            for item in xpath_result:
+                if isinstance(item, str):
+                    item = unicode(item)
+                result.append(item)
+            return result
+        except Exception, e:
+            self._logger.error('%s: caught exception extracting XPath "%s"',
+                                self._cname, path, exc_info=True)
+            return []
+
+    def tostring(self, element):
+        """Convert the element to a string."""
+        if isinstance(element, (unicode, str)):
+            return unicode(element)
+        else:
+            try:
+                return self._tostring(element, encoding=unicode)
+            except Exception, e:
+                self._logger.error('%s: unable to convert to string',
+                                    self._cname, exc_info=True)
+                return u''
+
+    def clone(self, element):
+        """Clone an element."""
+        return self.fromstring(self.tostring(element))
+
+    def preprocess_string(self, html_string):
+        """Here we can modify the text, before it's parsed."""
+        if not html_string:
+            return html_string
+        # Remove silly &nbsp;&raquo; chars.
+        html_string = html_string.replace(u' \xbb', u'')
+        try:
+            preprocessors = self.preprocessors
+        except AttributeError:
+            return html_string
+        for src, sub in preprocessors:
+            # re._pattern_type is present only since Python 2.5.
+            if callable(getattr(src, 'sub', None)):
+                html_string = src.sub(sub, html_string)
+            elif isinstance(src, str):
+                html_string = html_string.replace(src, sub)
+            elif callable(src):
+                try:
+                    html_string = src(html_string)
+                except Exception, e:
+                    _msg = '%s: caught exception preprocessing html'
+                    self._logger.error(_msg, self._cname, exc_info=True)
+                    continue
+        ##print html_string.encode('utf8')
+        return html_string
+
+    def gather_refs(self, dom):
+        """Collect references."""
+        grParser = GatherRefs(useModule=self._useModule)
+        grParser._as = self._as
+        grParser._modFunct = self._modFunct
+        refs = grParser.parse_dom(dom)
+        refs = grParser.postprocess_data(refs)
+        self._namesRefs = refs['names refs']
+        self._titlesRefs = refs['titles refs']
+        self._charactersRefs = refs['characters refs']
+
+    def preprocess_dom(self, dom):
+        """Last chance to modify the dom, before the rules in self.extractors
+        are applied by the parse_dom method."""
+        return dom
+
+    def parse_dom(self, dom):
+        """Parse the given dom according to the rules specified
+        in self.extractors."""
+        result = {}
+        for extractor in self.extractors:
+            ##print extractor.label
+            if extractor.group is None:
+                elements = [(extractor.label, element)
+                            for element in self.xpath(dom, extractor.path)]
+            else:
+                groups = self.xpath(dom, extractor.group)
+                elements = []
+                for group in groups:
+                    group_key = self.xpath(group, extractor.group_key)
+                    if not group_key: continue
+                    group_key = group_key[0]
+                    # XXX: always tries the conversion to unicode:
+                    #      BeautifulSoup.NavigableString is a subclass
+                    #      of unicode, and so it's never converted.
+                    group_key = self.tostring(group_key)
+                    normalizer = extractor.group_key_normalize
+                    if normalizer is not None:
+                        if callable(normalizer):
+                            try:
+                                group_key = normalizer(group_key)
+                            except Exception, e:
+                                _m = '%s: unable to apply group_key normalizer'
+                                self._logger.error(_m, self._cname,
+                                                    exc_info=True)
+                    group_elements = self.xpath(group, extractor.path)
+                    elements.extend([(group_key, element)
+                                     for element in group_elements])
+            for group_key, element in elements:
+                for attr in extractor.attrs:
+                    if isinstance(attr.path, dict):
+                        data = {}
+                        for field in attr.path.keys():
+                            path = attr.path[field]
+                            value = self.xpath(element, path)
+                            if not value:
+                                data[field] = None
+                            else:
+                                # XXX: use u'' , to join?
+                                data[field] = ''.join(value)
+                    else:
+                        data = self.xpath(element, attr.path)
+                        if not data:
+                            data = None
+                        else:
+                            data = attr.joiner.join(data)
+                    if not data:
+                        continue
+                    attr_postprocess = attr.postprocess
+                    if callable(attr_postprocess):
+                        try:
+                            data = attr_postprocess(data)
+                        except Exception, e:
+                            _m = '%s: unable to apply attr postprocess'
+                            self._logger.error(_m, self._cname, exc_info=True)
+                    key = attr.key
+                    if key is None:
+                        key = group_key
+                    elif key.startswith('.'):
+                        # assuming this is an xpath
+                        try:
+                            key = self.xpath(element, key)[0]
+                        except IndexError:
+                            self._logger.error('%s: XPath returned no items',
+                                                self._cname, exc_info=True)
+                    elif key.startswith('self.'):
+                        key = getattr(self, key[5:])
+                    if attr.multi:
+                        if key not in result:
+                            result[key] = []
+                        result[key].append(data)
+                    else:
+                        if isinstance(data, dict):
+                            result.update(data)
+                        else:
+                            result[key] = data
+        return result
+
+    def postprocess_data(self, data):
+        """Here we can modify the data."""
+        return data
+
+    def set_objects_params(self, data):
+        """Set parameters of Movie/Person/... instances, since they are
+        not always set in the parser's code."""
+        for obj in flatten(data, yieldDictKeys=True, scalar=_Container):
+            obj.accessSystem = self._as
+            obj.modFunct = self._modFunct
+
+    def add_refs(self, data):
+        """Modify data according to the expected output."""
+        if self.getRefs:
+            titl_re = ur'(%s)' % '|'.join([re.escape(x) for x
+                                            in self._titlesRefs.keys()])
+            if titl_re != ur'()': re_titles = re.compile(titl_re, re.U)
+            else: re_titles = None
+            nam_re = ur'(%s)' % '|'.join([re.escape(x) for x
+                                            in self._namesRefs.keys()])
+            if nam_re != ur'()': re_names = re.compile(nam_re, re.U)
+            else: re_names = None
+            chr_re = ur'(%s)' % '|'.join([re.escape(x) for x
+                                            in self._charactersRefs.keys()])
+            if chr_re != ur'()': re_characters = re.compile(chr_re, re.U)
+            else: re_characters = None
+            _putRefs(data, re_titles, re_names, re_characters)
+        return {'data': data, 'titlesRefs': self._titlesRefs,
+                'namesRefs': self._namesRefs,
+                'charactersRefs': self._charactersRefs}
+
+
+class Extractor(object):
+    """Instruct the DOM parser about how to parse a document."""
+    def __init__(self, label, path, attrs, group=None, group_key=None,
+                 group_key_normalize=None):
+        """Initialize an Extractor object, used to instruct the DOM parser
+        about how to parse a document."""
+        # rarely (never?) used, mostly for debugging purposes.
+        self.label = label
+        self.group = group
+        if group_key is None:
+            self.group_key = ".//text()"
+        else:
+            self.group_key = group_key
+        self.group_key_normalize = group_key_normalize
+        self.path = path
+        # A list of attributes to fetch.
+        if isinstance(attrs, Attribute):
+            attrs = [attrs]
+        self.attrs = attrs
+
+    def __repr__(self):
+        """String representation of an Extractor object."""
+        r = '<Extractor id:%s (label=%s, path=%s, attrs=%s, group=%s, ' \
+                'group_key=%s group_key_normalize=%s)>' % (id(self),
+                        self.label, self.path, repr(self.attrs), self.group,
+                        self.group_key, self.group_key_normalize)
+        return r
+
+
+class Attribute(object):
+    """The attribute to consider, for a given node."""
+    def __init__(self, key, multi=False, path=None, joiner=None,
+                 postprocess=None):
+        """Initialize an Attribute object, used to specify the
+        attribute to consider, for a given node."""
+        # The key under which information will be saved; can be a string or an
+        # XPath. If None, the label of the containing extractor will be used.
+        self.key = key
+        self.multi = multi
+        self.path = path
+        if joiner is None:
+            joiner = ''
+        self.joiner = joiner
+        # Post-process this set of information.
+        self.postprocess = postprocess
+
+    def __repr__(self):
+        """String representation of an Attribute object."""
+        r = '<Attribute id:%s (key=%s, multi=%s, path=%s, joiner=%s, ' \
+                'postprocess=%s)>' % (id(self), self.key,
+                        self.multi, repr(self.path),
+                        self.joiner, repr(self.postprocess))
+        return r
+
+
+def _parse_ref(text, link, info):
+    """Manage links to references."""
+    if link.find('/title/tt') != -1:
+        yearK = re_yearKind_index.match(info)
+        if yearK and yearK.start() == 0:
+            text += ' %s' % info[:yearK.end()]
+    return (text.replace('\n', ' '), link)
+
+
+class GatherRefs(DOMParserBase):
+    """Parser used to gather references to movies, persons and characters."""
+    _attrs = [Attribute(key=None, multi=True,
+                        path={
+                            'text': './text()',
+                            'link': './@href',
+                            'info': './following::text()[1]'
+                            },
+        postprocess=lambda x: _parse_ref(x.get('text'), x.get('link'),
+                                         (x.get('info') or u'').strip()))]
+    extractors = [
+        Extractor(label='names refs',
+            path="//a[starts-with(@href, '/name/nm')][string-length(@href)=16]",
+            attrs=_attrs),
+
+        Extractor(label='titles refs',
+            path="//a[starts-with(@href, '/title/tt')]" \
+                    "[string-length(@href)=17]",
+            attrs=_attrs),
+
+        Extractor(label='characters refs',
+            path="//a[starts-with(@href, '/character/ch')]" \
+                    "[string-length(@href)=21]",
+            attrs=_attrs),
+            ]
+
+    def postprocess_data(self, data):
+        result = {}
+        for item in ('names refs', 'titles refs', 'characters refs'):
+            result[item] = {}
+            for k, v in data.get(item, []):
+                if not v.endswith('/'): continue
+                imdbID = analyze_imdbid(v)
+                if item == 'names refs':
+                    obj = Person(personID=imdbID, name=k,
+                                accessSystem=self._as, modFunct=self._modFunct)
+                elif item == 'titles refs':
+                    obj = Movie(movieID=imdbID, title=k,
+                                accessSystem=self._as, modFunct=self._modFunct)
+                else:
+                    obj = Character(characterID=imdbID, name=k,
+                                accessSystem=self._as, modFunct=self._modFunct)
+                # XXX: companies aren't handled: are they ever found in text,
+                #      as links to their page?
+                result[item][k] = obj
+        return result
+
+    def add_refs(self, data):
+        return data
+
+

libs/imdb/parser/mobile/__init__.py

@@ -0,0 +1,811 @@
+"""
+parser.mobile package (imdb package).
+
+This package provides the IMDbMobileAccessSystem class used to access
+IMDb's data for mobile systems.
+the imdb.IMDb function will return an instance of this class when
+called with the 'accessSystem' argument set to "mobile".
+
+Copyright 2005-2010 Davide Alberani <da@erlug.linux.it>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import re
+import logging
+from urllib import unquote
+
+from imdb import imdbURL_movie_main, imdbURL_person_main, imdbURL_character_main
+from imdb.Movie import Movie
+from imdb.utils import analyze_title, analyze_name, canonicalName, \
+                        date_and_notes
+from imdb._exceptions import IMDbDataAccessError
+from imdb.parser.http import IMDbHTTPAccessSystem
+from imdb.parser.http.utils import subXMLRefs, subSGMLRefs, build_person, \
+                                    build_movie, re_spaces
+
+# XXX NOTE: the first version of this module was heavily based on
+#           regular expressions.  This new version replace regexps with
+#           find() strings' method calls; despite being less flexible, it
+#           seems to be at least as fast and, hopefully, much more
+#           lightweight.  Yes: the regexp-based version was too heavyweight
+#           for systems with very limited CPU power and memory footprint.
+re_spacessub = re_spaces.sub
+# Strip html.
+re_unhtml = re.compile(r'<.+?>')
+re_unhtmlsub = re_unhtml.sub
+# imdb person or movie ids.
+re_imdbID = re.compile(r'(?<=nm|tt|ch)([0-9]{7})\b')
+
+# movie AKAs.
+re_makas = re.compile('(<p class="find-aka">.*?</p>)')
+
+
+def _unHtml(s):
+    """Return a string without tags and no multiple spaces."""
+    return subSGMLRefs(re_spacessub(' ', re_unhtmlsub('', s)).strip())
+
+
+_inttype = type(0)
+
+def _getTagsWith(s, cont, toClosure=False, maxRes=None):
+    """Return the html tags in the 's' string containing the 'cont'
+    string; if toClosure is True, everything between the opening
+    tag and the closing tag is returned."""
+    lres = []
+    bi = s.find(cont)
+    if bi != -1:
+        btag = s[:bi].rfind('<')
+        if btag != -1:
+            if not toClosure:
+                etag = s[bi+1:].find('>')
+                if etag != -1:
+                    endidx = bi+2+etag
+                    lres.append(s[btag:endidx])
+                    if maxRes is not None and len(lres) >= maxRes: return lres
+                    lres += _getTagsWith(s[endidx:], cont,
+                                        toClosure=toClosure)
+            else:
+                spaceidx = s[btag:].find(' ')
+                if spaceidx != -1:
+                    ctag = '</%s>' % s[btag+1:btag+spaceidx]
+                    closeidx = s[bi:].find(ctag)
+                    if closeidx != -1:
+                        endidx = bi+closeidx+len(ctag)
+                        lres.append(s[btag:endidx])
+                        if maxRes is not None and len(lres) >= maxRes:
+                            return lres
+                        lres += _getTagsWith(s[endidx:], cont,
+                                            toClosure=toClosure)
+    return lres
+
+
+def _findBetween(s, begins, ends, beginindx=0, maxRes=None, lres=None):
+    """Return the list of strings from the 's' string which are included
+    between the 'begins' and 'ends' strings."""
+    if lres is None:
+        lres = []
+    bi = s.find(begins, beginindx)
+    if bi != -1:
+        lbegins = len(begins)
+        if isinstance(ends, (list, tuple)):
+            eset = [s.find(end, bi+lbegins) for end in ends]
+            eset[:] = [x for x in eset if x != -1]
+            if not eset: ei = -1
+            else: ei = min(eset)
+        else:
+            ei = s.find(ends, bi+lbegins)
+        if ei != -1:
+            match = s[bi+lbegins:ei]
+            lres.append(match)
+            if maxRes is not None and len(lres) >= maxRes: return lres
+            _findBetween(s, begins, ends, beginindx=ei, maxRes=maxRes,
+                        lres=lres)
+    return lres
+
+
+class IMDbMobileAccessSystem(IMDbHTTPAccessSystem):
+    """The class used to access IMDb's data through the web for
+    mobile terminals."""
+
+    accessSystem = 'mobile'
+    _mobile_logger = logging.getLogger('imdbpy.parser.mobile')
+
+    def __init__(self, isThin=1, *arguments, **keywords):
+        self.accessSystem = 'mobile'
+        IMDbHTTPAccessSystem.__init__(self, isThin, *arguments, **keywords)
+
+    def _clean_html(self, html):
+        """Normalize the retrieve html."""
+        html = re_spaces.sub(' ', html)
+        # Remove silly &nbsp;&raquo; chars.
+        html = html.replace('&nbsp;&raquo;', '')
+        return subXMLRefs(html)
+
+    def _mretrieve(self, url, size=-1):
+        """Retrieve an html page and normalize it."""
+        cont = self._retrieve(url, size=size)
+        return self._clean_html(cont)
+
+    def _getPersons(self, s, sep='<br/>'):
+        """Return a list of Person objects, from the string s; items
+        are assumed to be separated by the sep string."""
+        names = s.split(sep)
+        pl = []
+        plappend = pl.append
+        counter = 1
+        for name in names:
+            pid = re_imdbID.findall(name)
+            if not pid: continue
+            characters = _getTagsWith(name, 'class="char"',
+                                        toClosure=True, maxRes=1)
+            chpids = []
+            if characters:
+                for ch in characters[0].split(' / '):
+                    chid = re_imdbID.findall(ch)
+                    if not chid:
+                        chpids.append(None)
+                    else:
+                        chpids.append(chid[-1])
+            if not chpids:
+                chpids = None
+            elif len(chpids) == 1:
+                chpids = chpids[0]
+            name = _unHtml(name)
+            # Catch unclosed tags.
+            gt_indx = name.find('>')
+            if gt_indx != -1:
+                name = name[gt_indx+1:].lstrip()
+            if not name: continue
+            if name.endswith('...'):
+                name = name[:-3]
+            p = build_person(name, personID=str(pid[0]), billingPos=counter,
+                            modFunct=self._defModFunct, roleID=chpids,
+                            accessSystem=self.accessSystem)
+            plappend(p)
+            counter += 1
+        return pl
+
+    def _search_movie(self, title, results):
+        ##params = urllib.urlencode({'tt': 'on','mx': str(results),'q': title})
+        ##params = 'q=%s&tt=on&mx=%s' % (urllib.quote_plus(title), str(results))
+        ##cont = self._mretrieve(imdbURL_search % params)
+        cont = subXMLRefs(self._get_search_content('tt', title, results))
+        title = _findBetween(cont, '<title>', '</title>', maxRes=1)
+        res = []
+        if not title:
+            self._mobile_logger.error('no title tag searching for movie %s',
+                                    title)
+            return res
+        tl = title[0].lower()
+        if not tl.startswith('imdb title'):
+            # a direct hit!
+            title = _unHtml(title[0])
+            mid = None
+            midtag = _getTagsWith(cont, 'rel="canonical"', maxRes=1)
+            if midtag:
+                mid = _findBetween(midtag[0], '/title/tt', '/', maxRes=1)
+            if not (mid and title):
+                self._mobile_logger.error('no direct hit title/movieID for' \
+                                            ' title %s', title)
+                return res
+            if cont.find('<span class="tv-extra">TV mini-series</span>') != -1:
+                title += ' (mini)'
+            res[:] = [(str(mid[0]), analyze_title(title))]
+        else:
+            # XXX: this results*3 prevents some recursion errors, but...
+            #      it's not exactly understandable (i.e.: why 'results' is
+            #      not enough to get all the results?)
+            lis = _findBetween(cont, 'td valign="top">', '</td>',
+                                maxRes=results*3)
+            for li in lis:
+                akas = re_makas.findall(li)
+                for idx, aka in enumerate(akas):
+                    aka = aka.replace('" - ', '::', 1)
+                    aka = _unHtml(aka)
+                    if aka.startswith('aka "'):
+                        aka = aka[5:].strip()
+                    if aka[-1] == '"':
+                        aka = aka[:-1]
+                    akas[idx] = aka
+                imdbid = re_imdbID.findall(li)
+                li = re_makas.sub('', li)
+                mtitle = _unHtml(li)
+                if not (imdbid and mtitle):
+                    self._mobile_logger.debug('no title/movieID parsing' \
+                                            ' %s searching for title %s', li,
+                                            title)
+                    continue
+                mtitle = mtitle.replace('(TV mini-series)', '(mini)')
+                resd = analyze_title(mtitle)
+                if akas:
+                    resd['akas'] = akas
+                res.append((str(imdbid[0]), resd))
+        return res
+
+    def get_movie_main(self, movieID):
+        cont = self._mretrieve(imdbURL_movie_main % movieID + 'maindetails')
+        title = _findBetween(cont, '<title>', '</title>', maxRes=1)
+        if not title:
+            raise IMDbDataAccessError, 'unable to get movieID "%s"' % movieID
+        title = _unHtml(title[0])
+        if cont.find('<span class="tv-extra">TV mini-series</span>') != -1:
+            title += ' (mini)'
+        d = analyze_title(title)
+        kind = d.get('kind')
+        tv_series = _findBetween(cont, 'TV Series:</h5>', '</a>', maxRes=1)
+        if tv_series: mid = re_imdbID.findall(tv_series[0])
+        else: mid = None
+        if tv_series and mid:
+            s_title = _unHtml(tv_series[0])
+            s_data = analyze_title(s_title)
+            m = Movie(movieID=str(mid[0]), data=s_data,
+                        accessSystem=self.accessSystem,
+                        modFunct=self._defModFunct)
+            d['kind'] = kind = u'episode'
+            d['episode of'] = m
+        if kind in ('tv series', 'tv mini series'):
+            years = _findBetween(cont, '<h1>', '</h1>', maxRes=1)
+            if years:
+                years[:] = _findBetween(years[0], 'TV series', '</span>',
+                                        maxRes=1)
+                if years:
+                    d['series years'] = years[0].strip()
+        air_date = _findBetween(cont, 'Original Air Date:</h5>', '</div>',
+                                maxRes=1)
+        if air_date:
+            air_date = air_date[0]
+            vi = air_date.find('(')
+            if vi != -1:
+                date = _unHtml(air_date[:vi]).strip()
+                if date != '????':
+                    d['original air date'] = date
+                air_date = air_date[vi:]
+                season = _findBetween(air_date, 'Season', ',', maxRes=1)
+                if season:
+                    season = season[0].strip()
+                    try: season = int(season)
+                    except: pass
+                    if season or type(season) is _inttype:
+                        d['season'] = season
+                episode = _findBetween(air_date, 'Episode', ')', maxRes=1)
+                if episode:
+                    episode = episode[0].strip()
+                    try: episode = int(episode)
+                    except: pass
+                    if episode or type(season) is _inttype:
+                        d['episode'] = episode
+        direct = _findBetween(cont, '<h5>Director', ('</div>', '<br/> <br/>'),
+                                maxRes=1)
+        if direct:
+            direct = direct[0]
+            h5idx = direct.find('/h5>')
+            if h5idx != -1:
+                direct = direct[h5idx+4:]
+            direct = self._getPersons(direct)
+            if direct: d['director'] = direct
+        if kind in ('tv series', 'tv mini series', 'episode'):
+            if kind != 'episode':
+                seasons = _findBetween(cont, 'Seasons:</h5>', '</div>',
+                                        maxRes=1)
+                if seasons:
+                    d['number of seasons'] = seasons[0].count('|') + 1
+            creator = _findBetween(cont, 'Created by</h5>', ('class="tn15more"',
+                                                            '</div>',
+                                                            '<br/> <br/>'),
+                                                            maxRes=1)
+            if not creator:
+                # They change 'Created by' to 'Creator' and viceversa
+                # from time to time...
+                # XXX: is 'Creators' also used?
+                creator = _findBetween(cont, 'Creator:</h5>',
+                                        ('class="tn15more"', '</div>',
+                                        '<br/> <br/>'), maxRes=1)
+            if creator:
+                creator = creator[0]
+                if creator.find('tn15more'): creator = '%s>' % creator
+                creator = self._getPersons(creator)
+                if creator: d['creator'] = creator
+        writers = _findBetween(cont, '<h5>Writer', ('</div>', '<br/> <br/>'),
+                                maxRes=1)
+        if writers:
+            writers = writers[0]
+            h5idx = writers.find('/h5>')
+            if h5idx != -1:
+                writers = writers[h5idx+4:]
+            writers = self._getPersons(writers)
+            if writers: d['writer'] = writers
+        cvurl = _getTagsWith(cont, 'name="poster"', toClosure=True, maxRes=1)
+        if cvurl:
+            cvurl = _findBetween(cvurl[0], 'src="', '"', maxRes=1)
+            if cvurl: d['cover url'] = cvurl[0]
+        genres = _findBetween(cont, 'href="/Sections/Genres/', '/')
+        if genres:
+            d['genres'] = list(set(genres))
+        ur = _findBetween(cont, '<div class="starbar-meta">', '</div>',
+                            maxRes=1)
+        if ur:
+            rat = _findBetween(ur[0], '<b>', '</b>', maxRes=1)
+            if rat:
+                teni = rat[0].find('/10')
+                if teni != -1:
+                    rat = rat[0][:teni]
+                    try:
+                        rat = float(rat.strip())
+                        d['rating'] = rat
+                    except ValueError:
+                        self._mobile_logger.warn('wrong rating: %s', rat)
+            vi = ur[0].rfind('tn15more">')
+            if vi != -1 and ur[0][vi+10:].find('await') == -1:
+                try:
+                    votes = _unHtml(ur[0][vi+10:]).replace('votes', '').strip()
+                    votes = int(votes.replace(',', ''))
+                    d['votes'] = votes
+                except ValueError:
+                    self._mobile_logger.warn('wrong votes: %s', ur)
+        top250 = _findBetween(cont, 'href="/chart/top?', '</a>', maxRes=1)
+        if top250:
+            fn = top250[0].rfind('#')
+            if fn != -1:
+                try:
+                    td = int(top250[0][fn+1:])
+                    d['top 250 rank'] = td
+                except ValueError:
+                    self._mobile_logger.warn('wrong top250: %s', top250)
+        castdata = _findBetween(cont, 'Cast overview', '</table>', maxRes=1)
+        if not castdata:
+            castdata = _findBetween(cont, 'Credited cast', '</table>', maxRes=1)
+        if not castdata:
+            castdata = _findBetween(cont, 'Complete credited cast', '</table>',
+                                    maxRes=1)
+        if not castdata:
+            castdata = _findBetween(cont, 'Series Cast Summary', '</table>',
+                                    maxRes=1)
+        if not castdata:
+            castdata = _findBetween(cont, 'Episode Credited cast', '</table>',
+                                    maxRes=1)
+        if castdata:
+            castdata = castdata[0]
+            # Reintegrate the fist tag.
+            fl = castdata.find('href=')
+            if fl != -1: castdata = '<a ' + castdata[fl:]
+            # Exclude the 'rest of cast listed alphabetically' row.
+            smib = castdata.find('<tr><td align="center" colspan="4"><small>')
+            if smib != -1:
+                smie = castdata.rfind('</small></td></tr>')
+                if smie != -1:
+                    castdata = castdata[:smib].strip() + \
+                                castdata[smie+18:].strip()
+            castdata = castdata.replace('/tr> <tr', '/tr><tr')
+            cast = self._getPersons(castdata, sep='</tr><tr')
+            if cast: d['cast'] = cast
+        akas = _findBetween(cont, 'Also Known As:</h5>', '</div>', maxRes=1)
+        if akas:
+            # For some reason, here <br> is still used in place of <br/>.
+            akas[:] = [x for x in akas[0].split('<br>') if x.strip()]
+            akas = [_unHtml(x).replace('" - ','::', 1).lstrip('"').strip()
+                    for x in akas]
+            if 'See more' in akas: akas.remove('See more')
+            akas[:] = [x for x in akas if x]
+            if akas:
+                d['akas'] = akas
+        mpaa = _findBetween(cont, 'MPAA</a>:', '</div>', maxRes=1)
+        if mpaa: d['mpaa'] = _unHtml(mpaa[0])
+        runtimes = _findBetween(cont, 'Runtime:</h5>', '</div>', maxRes=1)
+        if runtimes:
+            runtimes = runtimes[0]
+            runtimes = [x.strip().replace(' min', '').replace(' (', '::(', 1)
+                    for x in runtimes.split('|')]
+            d['runtimes'] = [_unHtml(x).strip() for x in runtimes]
+        if kind == 'episode':
+            # number of episodes.
+            epsn = _findBetween(cont, 'title="Full Episode List">', '</a>',
+                                maxRes=1)
+            if epsn:
+                epsn = epsn[0].replace(' Episodes', '').strip()
+                if epsn:
+                    try:
+                        epsn = int(epsn)
+                    except:
+                        self._mobile_logger.warn('wrong episodes #: %s', epsn)
+                    d['number of episodes'] = epsn
+        country = _findBetween(cont, 'Country:</h5>', '</div>', maxRes=1)
+        if country:
+            country[:] = country[0].split(' | ')
+            country[:] = ['<a %s' % x for x in country if x]
+            country[:] = [_unHtml(x.replace(' <i>', '::')) for x in country]
+            if country: d['countries'] = country
+        lang = _findBetween(cont, 'Language:</h5>', '</div>', maxRes=1)
+        if lang:
+            lang[:] = lang[0].split(' | ')
+            lang[:] = ['<a %s' % x for x in lang if x]
+            lang[:] = [_unHtml(x.replace(' <i>', '::')) for x in lang]
+            if lang: d['languages'] = lang
+        col = _findBetween(cont, '"/search/title?colors=', '</div>')
+        if col:
+            col[:] = col[0].split(' | ')
+            col[:] = ['<a %s' % x for x in col if x]
+            col[:] = [_unHtml(x.replace(' <i>', '::')) for x in col]
+            if col: d['color info'] = col
+        sm = _findBetween(cont, '/search/title?sound_mixes=', '</div>',
+                            maxRes=1)
+        if sm:
+            sm[:] = sm[0].split(' | ')
+            sm[:] = ['<a %s' % x for x in sm if x]
+            sm[:] = [_unHtml(x.replace(' <i>', '::')) for x in sm]
+            if sm: d['sound mix'] = sm
+        cert = _findBetween(cont, 'Certification:</h5>', '</div>', maxRes=1)
+        if cert:
+            cert[:] = cert[0].split(' | ')
+            cert[:] = [_unHtml(x.replace(' <i>', '::')) for x in cert]
+            if cert: d['certificates'] = cert
+        plotoutline = _findBetween(cont, 'Plot:</h5>', ['<a ', '</div>'],
+                                    maxRes=1)
+        if plotoutline:
+            plotoutline = plotoutline[0].strip()
+            plotoutline = plotoutline.rstrip('|').rstrip()
+            if plotoutline: d['plot outline'] = _unHtml(plotoutline)
+        aratio = _findBetween(cont, 'Aspect Ratio:</h5>', ['<a ', '</div>'],
+                            maxRes=1)
+        if aratio:
+            aratio = aratio[0].strip().replace(' (', '::(', 1)
+            if aratio:
+                d['aspect ratio'] = _unHtml(aratio)
+        return {'data': d}
+
+    def get_movie_plot(self, movieID):
+        cont = self._mretrieve(imdbURL_movie_main % movieID + 'plotsummary')
+        plot = _findBetween(cont, '<p class="plotpar">', '</p>')
+        plot[:] = [_unHtml(x) for x in plot]
+        for i in xrange(len(plot)):
+            p = plot[i]
+            wbyidx = p.rfind(' Written by ')
+            if wbyidx != -1:
+                plot[i] = '%s::%s' % \
+                        (p[:wbyidx].rstrip(),
+                    p[wbyidx+12:].rstrip().replace('{','<').replace('}','>'))
+        if plot: return {'data': {'plot': plot}}
+        return {'data': {}}
+
+    def _search_person(self, name, results):
+        ##params = urllib.urlencode({'nm': 'on', 'mx': str(results), 'q': name})
+        ##params = 'q=%s&nm=on&mx=%s' % (urllib.quote_plus(name), str(results))
+        ##cont = self._mretrieve(imdbURL_search % params)
+        cont = subXMLRefs(self._get_search_content('nm', name, results))
+        name = _findBetween(cont, '<title>', '</title>', maxRes=1)
+        res = []
+        if not name:
+            self._mobile_logger.warn('no title tag searching for name %s', name)
+            return res
+        nl = name[0].lower()
+        if not nl.startswith('imdb name'):
+            # a direct hit!
+            name = _unHtml(name[0])
+            name = name.replace('- Filmography by type' , '').strip()
+            pid = None
+            pidtag = _getTagsWith(cont, 'rel="canonical"', maxRes=1)
+            if pidtag:
+                pid = _findBetween(pidtag[0], '/name/nm', '/', maxRes=1)
+            if not (pid and name):
+                self._mobile_logger.error('no direct hit name/personID for' \
+                                            ' name %s', name)
+                return res
+            res[:] = [(str(pid[0]), analyze_name(name, canonical=1))]
+        else:
+            lis = _findBetween(cont, 'td valign="top">', '</td>',
+                                maxRes=results*3)
+            for li in lis:
+                akas = _findBetween(li, '<em>"', '"</em>')
+                for sep in ['<small', '<br> aka', '<br> birth name']:
+                    sepIdx = li.find(sep)
+                    if sepIdx != -1:
+                        li = li[:sepIdx]
+                pid = re_imdbID.findall(li)
+                pname = _unHtml(li)
+                if not (pid and pname):
+                    self._mobile_logger.debug('no name/personID parsing' \
+                                            ' %s searching for name %s', li,
+                                            name)
+                    continue
+                resd = analyze_name(pname, canonical=1)
+                if akas:
+                    resd['akas'] = akas
+                res.append((str(pid[0]), resd))
+        return res
+
+    def get_person_main(self, personID, _parseChr=False):
+        if not _parseChr:
+            url = imdbURL_person_main % personID + 'maindetails'
+        else:
+            url = imdbURL_character_main % personID
+        s = self._mretrieve(url)
+        r = {}
+        name = _findBetween(s, '<title>', '</title>', maxRes=1)
+        if not name:
+            if _parseChr: w = 'characterID'
+            else: w = 'personID'
+            raise IMDbDataAccessError, 'unable to get %s "%s"' % (w, personID)
+        name = _unHtml(name[0])
+        if _parseChr:
+            name = name.replace('(Character)', '').strip()
+            name = name.replace('- Filmography by type', '').strip()
+        else:
+            name = name.replace('- Filmography by', '').strip()
+        r = analyze_name(name, canonical=not _parseChr)
+        for dKind in ('birth', 'death'):
+            date = _findBetween(s, '<h5>Date of %s:</h5>' % dKind.capitalize(),
+                                ('<a class', '</div>', '<br/><br/>'), maxRes=1)
+            if date:
+                date = _unHtml(date[0])
+                if date:
+                    date, notes = date_and_notes(date)
+                    if date:
+                        r['%s date' % dKind] = date
+                    if notes:
+                        r['%s notes' % dKind] = notes
+        akas = _findBetween(s, 'Alternate Names:</h5>', ('</div>',
+                            '<br/><br/>'), maxRes=1)
+        if akas:
+            akas = akas[0]
+            if akas.find(' | ') != -1:
+                akas = _unHtml(akas).split(' | ')
+            else:
+                akas = _unHtml(akas).split(' / ')
+            if akas: r['akas'] = akas
+        hs = _findBetween(s, 'name="headshot"', '</a>', maxRes=1)
+        if hs:
+            hs[:] = _findBetween(hs[0], 'src="', '"', maxRes=1)
+            if hs: r['headshot'] = hs[0]
+        # Build a list of tuples such [('hrefLink', 'section name')]
+        workkind = _findBetween(s, '<div class="strip jump">', '</div>',
+                                maxRes=1)
+        if workkind:
+            workkind[:] = _findBetween(workkind[0], 'href="#', '</a>')
+        else:
+            # Assume there's only one section and/or there are no
+            # section links, for some reason.
+            workkind[:] = _findBetween(s, '<h5><a name=', '</a></h5>')
+            workkind[:] = [x.lstrip('"').rstrip(':').lower() for x in workkind]
+        ws = []
+        for work in workkind:
+            wsplit = work.split('">', 1)
+            if len(wsplit) == 2:
+                sect = wsplit[0]
+                if '"' in sect:
+                    sect = sect[:sect.find('"')]
+                ws.append((sect, wsplit[1].lower()))
+        # XXX: I think "guest appearances" are gone.
+        if s.find('<a href="#guest-appearances"') != -1:
+            ws.append(('guest-appearances', 'notable tv guest appearances'))
+        #if _parseChr:
+        #    ws.append(('filmography', 'filmography'))
+        for sect, sectName in ws:
+            raws = u''
+            # Everything between the current section link and the end
+            # of the <ol> tag.
+            if _parseChr and sect == 'filmography':
+                inisect = s.find('<div class="filmo">')
+            else:
+                inisect = s.find('<a name="%s' % sect)
+            if inisect != -1:
+                endsect = s[inisect:].find('</ol>')
+                if endsect != -1: raws = s[inisect:inisect+endsect]
+            if not raws: continue
+            mlist = _findBetween(raws, '<li>', ('</li>', '<br>', '<br/>'))
+            for m in mlist:
+                # For every movie in the current section.
+                movieID = re_imdbID.findall(m)
+                if not movieID:
+                    self._mobile_logger.debug('no movieID in %s', m)
+                    continue
+                if not _parseChr:
+                    chrIndx = m.find(' .... ')
+                else:
+                    chrIndx = m.find(' Played by ')
+                chids = []
+                if chrIndx != -1:
+                    chrtxt = m[chrIndx+6:]
+                    if _parseChr:
+                        chrtxt = chrtxt[5:]
+                    for ch in chrtxt.split(' / '):
+                        chid = re_imdbID.findall(ch)
+                        if not chid:
+                            chids.append(None)
+                        else:
+                            chids.append(chid[-1])
+                if not chids:
+                    chids = None
+                elif len(chids) == 1:
+                    chids = chids[0]
+                movieID = str(movieID[0])
+                # Search the status.
+                stidx = m.find('<i>')
+                status = u''
+                if stidx != -1:
+                    stendidx = m.rfind('</i>')
+                    if stendidx != -1:
+                        status = _unHtml(m[stidx+3:stendidx])
+                        m = m.replace(m[stidx+3:stendidx], '')
+                m = _unHtml(m)
+                if not m:
+                    self._mobile_logger.warn('no title fo rmovieID %s', movieID)
+                    continue
+                movie = build_movie(m, movieID=movieID, status=status,
+                                    roleID=chids, modFunct=self._defModFunct,
+                                    accessSystem=self.accessSystem,
+                                    _parsingCharacter=_parseChr)
+                r.setdefault(sectName, []).append(movie)
+        # If available, take the always correct name from a form.
+        itag = _getTagsWith(s, 'NAME="primary"', maxRes=1)
+        if not itag:
+            itag = _getTagsWith(s, 'name="primary"', maxRes=1)
+        if itag:
+            vtag = _findBetween(itag[0], 'VALUE="', ('"', '>'), maxRes=1)
+            if not vtag:
+                vtag = _findBetween(itag[0], 'value="', ('"', '>'), maxRes=1)
+            if vtag:
+                try:
+                    vtag = unquote(str(vtag[0]))
+                    vtag = unicode(vtag, 'latin_1')
+                    r.update(analyze_name(vtag))
+                except UnicodeEncodeError:
+                    pass
+        return {'data': r, 'info sets': ('main', 'filmography')}
+
+    def get_person_biography(self, personID):
+        cont = self._mretrieve(imdbURL_person_main % personID + 'bio')
+        d = {}
+        spouses = _findBetween(cont, 'Spouse</h5>', ('</table>', '</dd>'),
+                                maxRes=1)
+        if spouses:
+            sl = []
+            for spouse in spouses[0].split('</tr>'):
+                if spouse.count('</td>') > 1:
+                    spouse = spouse.replace('</td>', '::</td>', 1)
+                spouse = _unHtml(spouse)
+                spouse = spouse.replace(':: ', '::').strip()
+                if spouse: sl.append(spouse)
+            if sl: d['spouse'] = sl
+        nnames = _findBetween(cont, '<h5>Nickname</h5>', ('<br/> <br/>','<h5>'),
+                                maxRes=1)
+        if nnames:
+            nnames = nnames[0]
+            if nnames:
+                nnames = [x.strip().replace(' (', '::(', 1)
+                            for x in nnames.split('<br/>')]
+                if nnames:
+                    d['nick names'] = nnames
+        misc_sects = _findBetween(cont, '<h5>', '<br/>')
+        misc_sects[:] = [x.split('</h5>') for x in misc_sects]
+        misc_sects[:] = [x for x in misc_sects if len(x) == 2]
+        for sect, data in misc_sects:
+            sect = sect.lower().replace(':', '').strip()
+            if d.has_key(sect) and sect != 'mini biography': continue
+            elif sect in ('spouse', 'nickname'): continue
+            if sect == 'salary': sect = 'salary history'
+            elif sect == 'where are they now': sect = 'where now'
+            elif sect == 'personal quotes': sect = 'quotes'
+            data = data.replace('</p><p>', '::')
+            data = data.replace('<br><br>', ' ') # for multi-paragraphs 'bio'
+            data = data.replace('</td> <td valign="top">', '@@@@')
+            data = data.replace('</td> </tr>', '::')
+            data = _unHtml(data)
+            data = [x.strip() for x in data.split('::')]
+            data[:] = [x.replace('@@@@', '::') for x in data if x]
+            if sect == 'height' and data: data = data[0]
+            elif sect == 'birth name': data = canonicalName(data[0])
+            elif sect == 'date of birth':
+                date, notes = date_and_notes(data[0])
+                if date:
+                    d['birth date'] = date
+                if notes:
+                    d['birth notes'] = notes
+                continue
+            elif sect == 'date of death':
+                date, notes = date_and_notes(data[0])
+                if date:
+                    d['death date'] = date
+                if notes:
+                    d['death notes'] = notes
+                continue
+            elif sect == 'mini biography':
+                ndata = []
+                for bio in data:
+                    byidx = bio.rfind('IMDb Mini Biography By')
+                    if byidx != -1:
+                        bioAuth = bio[:byidx].rstrip()
+                    else:
+                        bioAuth = 'Anonymous'
+                    bio = u'%s::%s' % (bioAuth, bio[byidx+23:].lstrip())
+                    ndata.append(bio)
+                data[:] = ndata
+                if 'mini biography' in d:
+                    d['mini biography'].append(ndata[0])
+                    continue
+            d[sect] = data
+        return {'data': d}
+
+    def _search_character(self, name, results):
+        cont = subXMLRefs(self._get_search_content('char', name, results))
+        name = _findBetween(cont, '<title>', '</title>', maxRes=1)
+        res = []
+        if not name:
+            self._mobile_logger.error('no title tag searching character %s',
+                                    name)
+            return res
+        nl = name[0].lower()
+        if not (nl.startswith('imdb search') or nl.startswith('imdb  search') \
+                or nl.startswith('imdb character')):
+            # a direct hit!
+            name = _unHtml(name[0]).replace('(Character)', '').strip()
+            pid = None
+            pidtag = _getTagsWith(cont, 'rel="canonical"', maxRes=1)
+            if pidtag:
+                pid = _findBetween(pidtag[0], '/character/ch', '/', maxRes=1)
+            if not (pid and name):
+                self._mobile_logger.error('no direct hit name/characterID for' \
+                                            ' character %s', name)
+                return res
+            res[:] = [(str(pid[0]), analyze_name(name))]
+        else:
+            sects = _findBetween(cont, '<b>Popular Characters</b>', '</table>',
+                                maxRes=results*3)
+            sects += _findBetween(cont, '<b>Characters', '</table>',
+                                maxRes=results*3)
+            for sect in sects:
+                lis = _findBetween(sect, '<a href="/character/',
+                                    ['<small', '</td>', '<br'])
+                for li in lis:
+                    li = '<%s' % li
+                    pid = re_imdbID.findall(li)
+                    pname = _unHtml(li)
+                    if not (pid and pname):
+                        self._mobile_logger.debug('no name/characterID' \
+                                                ' parsing %s searching for' \
+                                                ' character %s', li, name)
+                        continue
+                    res.append((str(pid[0]), analyze_name(pname)))
+        return res
+
+    def get_character_main(self, characterID):
+        return self.get_person_main(characterID, _parseChr=True)
+
+    def get_character_biography(self, characterID):
+        cont = self._mretrieve(imdbURL_character_main % characterID + 'bio')
+        d = {}
+        intro = _findBetween(cont, '<div class="display">',
+                            ('<span>', '<h4>'), maxRes=1)
+        if intro:
+            intro = _unHtml(intro[0]).strip()
+            if intro:
+                d['introduction'] = intro
+        bios = _findBetween(cont, '<div class="display">',
+                            '<div class="history">')
+        if bios:
+            bios = _findBetween(bios[0], '<h4>', ('<h4>', '</div>'))
+        if bios:
+            for bio in bios:
+                bio = bio.replace('</h4>', '::')
+                bio = bio.replace('\n', ' ')
+                bio = bio.replace('<br>', '\n')
+                bio = bio.replace('<br/>', '\n')
+                bio = subSGMLRefs(re_unhtmlsub('', bio).strip())
+                bio = bio.replace(' ::', '::').replace(':: ', '::')
+                bio = bio.replace('::', ': ', 1)
+                if bio:
+                    d.setdefault('biography', []).append(bio)
+        return {'data': d}
+
+