imdbPy update

libs/imdb/__init__.py

@@ -25,7 +25,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 __all__ = ['IMDb', 'IMDbError', 'Movie', 'Person', 'Character', 'Company',
             'available_access_systems']
-__version__ = VERSION = '4.6'
+__version__ = VERSION = '4.8dev20110303'
 
 # Import compatibility module (importing it is enough).
 import _compat
@@ -156,6 +156,7 @@ def IMDb(accessSystem=None, *arguments, **keywords):
             kwds.update(keywords)
             keywords = kwds
         except Exception, e:
+            import logging
             logging.getLogger('imdbpy').warn('Unable to read configuration' \
                                             ' file; complete error: %s' % e)
             # It just LOOKS LIKE a bad habit: we tried to read config
@@ -179,6 +180,10 @@ def IMDb(accessSystem=None, *arguments, **keywords):
         from parser.http import IMDbHTTPAccessSystem
         return IMDbHTTPAccessSystem(*arguments, **keywords)
     elif accessSystem in ('httpThin', 'webThin', 'htmlThin'):
+        import logging
+        logging.warn('httpThin is badly broken and' \
+                                        ' will not be fixed; please switch' \
+                                        ' to "http" or "mobile"')
         from parser.http import IMDbHTTPAccessSystem
         return IMDbHTTPAccessSystem(isThin=1, *arguments, **keywords)
     elif accessSystem in ('mobile',):

libs/imdb/_exceptions.py

@@ -23,7 +23,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 import logging
 
 
-class IMDbError(Exception):
+class IMDbError(Exception, object):
     """Base class for every exception raised by the imdb package."""
     _logger = logging.getLogger('imdbpy')
 

libs/imdb/parser/http/__init__.py

@@ -109,6 +109,10 @@ _old_cookie_uu = '3M3AXsquTU5Gur/Svik+ewflPm5Rk2ieY3BIPlLjyK3C0Dp9F8UoPgbTyKiGtZ
 _cookie_id = 'rH1jNAkjTlNXvHolvBVBsgaPICNZbNdjVjzFwzas9JRmusdjVoqBs/Hs12NR+1WFxEoR9bGKEDUg6sNlADqXwkas12N131Rwdb+UQNGKN8PWrNdjcdqBQVLq8mbGDHP3hqzxhbD692NQi9D0JjpBtRaPIbP1zNdjUOqENQYv1ADWrNcT9vyXU1'
 _cookie_uu = 'su4/m8cho4c6HP+W1qgq6wchOmhnF0w+lIWvHjRUPJ6nRA9sccEafjGADJ6hQGrMd4GKqLcz2X4z5+w+M4OIKnRn7FpENH7dxDQu3bQEHyx0ZEyeRFTPHfQEX03XF+yeN1dsPpcXaqjUZAw+lGRfXRQEfz3RIX9IgVEffdBAHw2wQXyf9xdMPrQELw0QNB8dsffsqcdQemjPB0w+moLcPh0JrKrHJ9hjBzdMPpcXTH7XRwwOk='
 
+# imdbpy2010 account.
+#_cookie_id = 'QrCdxVi+L+WgqOLrQJJgBgRRXGInphxiBPU/YXSFDyExMFzCp6YcYgSVXyEUhS/xMID8wqemHGID4DlntwZ49vemP5UXsAxiJ4D6goSmHGIgNT9hMXBaRSF2vMS3phxB0bVfQiQlP1RxdrzhB6YcRHFASyIhQVowwXCKtDSlD2YhgRvxBsCKtGemHBKH9mxSI='
+#_cookie_uu = 'oiEo2yoJFCA2Zbn/o7Z1LAPIwotAu6QdALv3foDb1x5F/tdrFY63XkSfty4kntS8Y8jkHSDLt3406+d+JThEilPI0mtTaOQdA/t2/iErp22jaLdeVU5ya4PIREpj7HFdpzhEHadcIAngSER50IoHDpD6Bz4Qy3b+UIhE/hBbhz5Q63ceA2hEvhPo5B0FnrL9Q8jkWjDIbA0Au3d+AOtnXoCIRL4Q28c+UOtnXpP4RL4T6OQdA+6ijUCI5B0AW2d+UOtnXpPYRL4T6OQdA8jkTUOYlC0A=='
+
 
 class _FakeURLOpener(object):
     """Fake URLOpener object, used to return empty strings instead of

libs/imdb/parser/http/movieParser.py

@@ -225,8 +225,8 @@ class DOMHTMLMovieParser(DOMParserBase):
                                 postprocess=lambda x: x.strip()),
                             Attribute(key="countries",
                                 path="./h5[starts-with(text(), " \
-                                        "'Countr')]/..//a/text()",
-                                    postprocess=makeSplitter(sep='\n')),
+                            "'Countr')]/../div[@class='info-content']//text()",
+                            postprocess=makeSplitter('|')),
                             Attribute(key="language",
                                 path="./h5[starts-with(text(), " \
                                         "'Language')]/..//text()",
@@ -541,11 +541,13 @@ class DOMHTMLPlotParser(DOMParserBase):
 
 def _process_award(x):
     award = {}
+    award['award'] = x.get('award').strip()
+    if not award['award']:
+        return {}
     award['year'] = x.get('year').strip()
     if award['year'] and award['year'].isdigit():
         award['year'] = int(award['year'])
     award['result'] = x.get('result').strip()
-    award['award'] = x.get('award').strip()
     category = x.get('category').strip()
     if category:
         award['category'] = category
@@ -649,6 +651,8 @@ class DOMHTMLAwardsParser(DOMParserBase):
             assigner = self.xpath(dom, "//a/text()")[0]
             for entry in data[key]:
                 if not entry.has_key('name'):
+                    if not entry:
+                        continue
                     # this is an award, not a recipient
                     entry['assigner'] = assigner.strip()
                     # find the recipients
@@ -996,8 +1000,10 @@ class DOMHTMLRatingsParser(DOMParserBase):
         if votes:
             nd['number of votes'] = {}
             for i in xrange(1, 11):
-                nd['number of votes'][int(votes[i]['ordinal'])] = \
-                        int(votes[i]['votes'].replace(',', ''))
+                _ordinal = int(votes[i]['ordinal'])
+                _strvts = votes[i]['votes'] or '0'
+                nd['number of votes'][_ordinal] = \
+                        int(_strvts.replace(',', ''))
         mean = data.get('mean and median', '')
         if mean:
             means = self.re_means.findall(mean)
@@ -1699,10 +1705,14 @@ class DOMHTMLEpisodesParser(DOMParserBase):
                 try: season_key = int(season_key)
                 except: pass
                 nd[season_key] = {}
+                ep_counter = 1
                 for episode in data[key]:
                     if not episode: continue
                     episode_key = episode.get('episode')
                     if episode_key is None: continue
+                    if not isinstance(episode_key, int):
+                        episode_key = ep_counter
+                        ep_counter += 1
                     cast_key = 'Season %s, Episode %s:' % (season_key,
                                                             episode_key)
                     if data.has_key(cast_key):

libs/imdb/parser/http/personParser.py

@@ -63,85 +63,131 @@ class DOMHTMLMaindetailsParser(DOMParserBase):
 
     _birth_attrs = [Attribute(key='birth date',
                         path={
-                            'day': "./div/a[starts-with(@href, " \
+                            'day': ".//a[starts-with(@href, " \
                                     "'/date/')]/text()",
-                            'year': "./div/a[starts-with(@href, " \
+                            'year': ".//a[starts-with(@href, " \
                                     "'/search/name?birth_year=')]/text()"
                             },
                         postprocess=build_date),
-                    Attribute(key='birth notes',
-                        path="./div/a[starts-with(@href, " \
+                    Attribute(key='birth place',
+                        path=".//a[starts-with(@href, " \
                                 "'/search/name?birth_place=')]/text()")]
     _death_attrs = [Attribute(key='death date',
                         path={
-                            'day': "./div/a[starts-with(@href, " \
+                            'day': ".//a[starts-with(@href, " \
                                     "'/date/')]/text()",
-                            'year': "./div/a[starts-with(@href, " \
-                                    "'/search/name?death_date=')]/text()"
+                            'year': ".//a[starts-with(@href, " \
+                                    "'/search/name?death_year=')]/text()"
                             },
                         postprocess=build_date),
-                    Attribute(key='death notes',
-                        path="./div/text()",
-                        # TODO: check if this slicing is always correct
-                        postprocess=lambda x: x.strip()[2:])]
+                    Attribute(key='death place',
+                        path=".//a[starts-with(@href, " \
+                                "'/search/name?death_place=')]/text()")]
     _film_attrs = [Attribute(key=None,
                       multi=True,
                       path={
-                          'link': "./a[1]/@href",
-                          'title': ".//text()",
-                          'status': "./i/a//text()",
-                          'roleID': "./div[@class='_imdbpyrole']/@roleid"
+                          'link': "./b/a[1]/@href",
+                          'title': "./b/a[1]/text()",
+                          'notes': "./b/following-sibling::text()",
+                          'year': "./span[@class='year_column']/text()",
+                          'status': "./a[@class='in_production']/text()",
+                          'rolesNoChar': './/br/following-sibling::text()',
+                          'chrRoles': "./a[@imdbpyname]/@imdbpyname",
+                          'roleID': "./a[starts-with(@href, '/character/')]/@href"
                           },
                       postprocess=lambda x:
                           build_movie(x.get('title') or u'',
+                              year=x.get('year'),
                               movieID=analyze_imdbid(x.get('link') or u''),
-                              roleID=(x.get('roleID') or u'').split('/'),
+                              rolesNoChar=(x.get('rolesNoChar') or u'').strip(),
+                              chrRoles=(x.get('chrRoles') or u'').strip(),
+                              additionalNotes=x.get('notes'),
+                              roleID=(x.get('roleID') or u''),
                               status=x.get('status') or None))]
 
     extractors = [
-            Extractor(label='page title',
-                        path="//title",
+            Extractor(label='name',
+                        path="//h1[@class='header']",
                         attrs=Attribute(key='name',
-                            path="./text()",
+                            path=".//text()",
                             postprocess=lambda x: analyze_name(x,
-                                                            canonical=1))),
+                                                               canonical=1))),
 
             Extractor(label='birth info',
-                        path="//div[h5='Date of Birth:']",
+                        path="//div[h4='Born:']",
                         attrs=_birth_attrs),
 
             Extractor(label='death info',
-                        path="//div[h5='Date of Death:']",
+                        path="//div[h4='Died:']",
                         attrs=_death_attrs),
 
             Extractor(label='headshot',
-                        path="//a[@name='headshot']",
+                        path="//td[@id='img_primary']/a",
                         attrs=Attribute(key='headshot',
                             path="./img/@src")),
 
             Extractor(label='akas',
-                        path="//div[h5='Alternate Names:']",
+                        path="//div[h4='Alternate Names:']",
                         attrs=Attribute(key='akas',
-                            path="./div/text()",
-                            postprocess=lambda x: x.strip().split(' | '))),
+                            path="./text()",
+                            postprocess=lambda x: x.strip().split('  '))),
 
             Extractor(label='filmography',
-                        group="//div[@class='filmo'][h5]",
-                        group_key="./h5/a[@name]/text()",
-                        group_key_normalize=lambda x: x.lower()[:-1],
-                        path="./ol/li",
-                        attrs=_film_attrs)
+                        group="//div[starts-with(@id, 'filmo-head-')]",
+                        group_key="./a[@name]/text()",
+                        group_key_normalize=lambda x: x.lower().replace(': ', ' '),
+                        path="./following-sibling::div[1]" \
+                                "/div[starts-with(@class, 'filmo-row')]",
+                        attrs=_film_attrs),
+
+            Extractor(label='indevelopment',
+                        path="//div[starts-with(@class,'devitem')]",
+                        attrs=Attribute(key='in development',
+                            multi=True,
+                            path={
+                                'link': './a/@href',
+                                'title': './a/text()'
+                                },
+                                postprocess=lambda x:
+                                    build_movie(x.get('title') or u'',
+                                        movieID=analyze_imdbid(x.get('link') or u''),
+                                        roleID=(x.get('roleID') or u'').split('/'),
+                                        status=x.get('status') or None)))
             ]
-    preprocessors = [
-            # XXX: check that this doesn't cut "status" or other info...
-            (re.compile(r'<br>(\.\.\.|    ?).+?</li>', re.I | re.M | re.S),
-                '</li>'),
-            (_reRoles, _manageRoles)]
+
+    preprocessors = [('<div class="clear"/> </div>', ''),
+            ('<br/>', '<br />'),
+            (re.compile(r'(<a href="/character/ch[0-9]{7}")>(.*?)</a>'),
+                r'\1 imdbpyname="\2@@">\2</a>')]
 
     def postprocess_data(self, data):
         for what in 'birth date', 'death date':
             if what in data and not data[what]:
                 del data[what]
+        # XXX: the code below is for backwards compatibility
+        # probably could be removed
+        for key in data.keys():
+            if key.startswith('actor '):
+                if not data.has_key('actor'):
+                    data['actor'] = []
+                data['actor'].extend(data[key])
+                del data[key]
+            if key.startswith('actress '):
+                if not data.has_key('actress'):
+                    data['actress'] = []
+                data['actress'].extend(data[key])
+                del data[key]
+            if key.startswith('self '):
+                if not data.has_key('self'):
+                    data['self'] = []
+                data['self'].extend(data[key])
+                del data[key]
+            if key == 'birth place':
+                data['birth notes'] = data[key]
+                del data[key]
+            if key == 'death place':
+                data['death notes'] = data[key]
+                del data[key]
         return data
 
 
@@ -181,6 +227,10 @@ class DOMHTMLBioParser(DOMParserBase):
                         # TODO: check if this slicing is always correct
                         postprocess=lambda x: u''.join(x).strip()[2:])]
     extractors = [
+            Extractor(label='headshot',
+                        path="//a[@name='headshot']",
+                        attrs=Attribute(key='headshot',
+                            path="./img/@src")),
             Extractor(label='birth info',
                         path="//div[h5='Date of Birth']",
                         attrs=_birth_attrs),

libs/imdb/parser/http/utils.py

@@ -262,14 +262,20 @@ def build_person(txt, personID=None, billingPos=None,
     return person
 
 
+_re_chrIDs = re.compile('[0-9]{7}')
+
 _b_m_logger = logging.getLogger('imdbpy.parser.http.build_movie')
 # To shrink spaces.
 re_spaces = re.compile(r'\s+')
 def build_movie(txt, movieID=None, roleID=None, status=None,
                 accessSystem='http', modFunct=None, _parsingCharacter=False,
-                _parsingCompany=False):
+                _parsingCompany=False, year=None, chrRoles=None,
+                rolesNoChar=None, additionalNotes=None):
     """Given a string as normally seen on the "categorized" page of
     a person on the IMDb's web site, returns a Movie instance."""
+    # FIXME: Oook, lets face it: build_movie and build_person are now
+    # two horrible sets of patches to support the new IMDb design.  They
+    # must be rewritten from scratch.
     if _parsingCharacter:
         _defSep = ' Played by '
     elif _parsingCompany:
@@ -291,6 +297,8 @@ def build_movie(txt, movieID=None, roleID=None, status=None,
         title = title[:-14] + ' (mini)'
     # Try to understand where the movie title ends.
     while True:
+        if year:
+            break
         if title[-1:] != ')':
             # Ignore the silly "TV Series" notice.
             if title[-9:] == 'TV Series':
@@ -319,12 +327,24 @@ def build_movie(txt, movieID=None, roleID=None, status=None,
         if notes: notes = '%s %s' % (title[nidx:], notes)
         else: notes = title[nidx:]
         title = title[:nidx].rstrip()
+    if year:
+        year = year.strip()
+        if title[-1] == ')':
+            fpIdx = title.rfind('(')
+            if fpIdx != -1:
+                if notes: notes = '%s %s' % (title[fpIdx:], notes)
+                else: notes = title[fpIdx:]
+                title = title[:fpIdx].rstrip()
+        title = u'%s (%s)' % (title, year)
     if _parsingCharacter and roleID and not role:
         roleID = None
     if not roleID:
         roleID = None
     elif len(roleID) == 1:
         roleID = roleID[0]
+    if not role and chrRoles and isinstance(roleID, (str, unicode)):
+        roleID = _re_chrIDs.findall(roleID)
+        role = ' / '.join(filter(None, chrRoles.split('@@')))
     # Manages multiple roleIDs.
     if isinstance(roleID, list):
         tmprole = role.split('/')
@@ -355,13 +375,29 @@ def build_movie(txt, movieID=None, roleID=None, status=None,
         movieID = str(movieID)
     if (not title) or (movieID is None):
         _b_m_logger.error('empty title or movieID for "%s"', txt)
+    if rolesNoChar:
+        rolesNoChar = filter(None, [x.strip() for x in rolesNoChar.split('/')])
+        if not role:
+            role = []
+        elif not isinstance(role, list):
+            role = [role]
+        role += rolesNoChar
+    notes = notes.strip()
+    if additionalNotes:
+        additionalNotes = re_spaces.sub(' ', additionalNotes).strip()
+        if notes:
+            notes += u' '
+        notes += additionalNotes
     m = Movie(title=title, movieID=movieID, notes=notes, currentRole=role,
                 roleID=roleID, roleIsPerson=_parsingCharacter,
                 modFunct=modFunct, accessSystem=accessSystem)
     if roleNotes and len(roleNotes) == len(roleID):
         for idx, role in enumerate(m.currentRole):
-            if roleNotes[idx]:
-                role.notes = roleNotes[idx]
+            try:
+                if roleNotes[idx]:
+                    role.notes = roleNotes[idx]
+            except IndexError:
+                break
     # Status can't be checked here, and must be detected by the parser.
     if status:
         m['status'] = status
@@ -468,8 +504,10 @@ class DOMParserBase(object):
         # converted to title=""Family Guy"" and this confuses BeautifulSoup.
         if self.usingModule == 'beautifulsoup':
             html_string = html_string.replace('""', '"')
+        #print html_string.encode('utf8')
         if html_string:
             dom = self.get_dom(html_string)
+            #print self.tostring(dom).encode('utf8')
             try:
                 dom = self.preprocess_dom(dom)
             except Exception, e:

libs/imdb/parser/mobile/__init__.py

@@ -52,6 +52,10 @@ re_imdbID = re.compile(r'(?<=nm|tt|ch)([0-9]{7})\b')
 # movie AKAs.
 re_makas = re.compile('(<p class="find-aka">.*?</p>)')
 
+# Remove episode numbers.
+re_filmo_episodes = re.compile('<div class="filmo-episodes">.*?</div>',
+        re.M | re.I)
+
 
 def _unHtml(s):
     """Return a string without tags and no multiple spaces."""
@@ -537,24 +541,33 @@ class IMDbMobileAccessSystem(IMDbHTTPAccessSystem):
             if _parseChr: w = 'characterID'
             else: w = 'personID'
             raise IMDbDataAccessError, 'unable to get %s "%s"' % (w, personID)
-        name = _unHtml(name[0])
+        name = _unHtml(name[0].replace(' - IMDb', ''))
         if _parseChr:
             name = name.replace('(Character)', '').strip()
             name = name.replace('- Filmography by type', '').strip()
         else:
             name = name.replace('- Filmography by', '').strip()
         r = analyze_name(name, canonical=not _parseChr)
-        for dKind in ('birth', 'death'):
-            date = _findBetween(s, '<h5>Date of %s:</h5>' % dKind.capitalize(),
-                                ('<a class', '</div>', '<br/><br/>'), maxRes=1)
+        for dKind in ('Born', 'Died'):
+            date = _findBetween(s, '%s:</h4>' % dKind.capitalize(),
+                                ('<div class', '</div>', '<br/><br/>'), maxRes=1)
             if date:
                 date = _unHtml(date[0])
                 if date:
-                    date, notes = date_and_notes(date)
+                    #date, notes = date_and_notes(date)
+                    # TODO: fix to handle real names.
+                    date_notes = date.split(' in ', 1)
+                    notes = u''
+                    date = date_notes[0]
+                    if len(date_notes) == 2:
+                        notes = date_notes[1]
+                    dtitle = 'birth'
+                    if dKind == 'Died':
+                        dtitle = 'death'
                     if date:
-                        r['%s date' % dKind] = date
+                        r['%s date' % dtitle] = date
                     if notes:
-                        r['%s notes' % dKind] = notes
+                        r['%s notes' % dtitle] = notes
         akas = _findBetween(s, 'Alternate Names:</h5>', ('</div>',
                             '<br/><br/>'), maxRes=1)
         if akas:
@@ -569,18 +582,13 @@ class IMDbMobileAccessSystem(IMDbHTTPAccessSystem):
             hs[:] = _findBetween(hs[0], 'src="', '"', maxRes=1)
             if hs: r['headshot'] = hs[0]
         # Build a list of tuples such [('hrefLink', 'section name')]
-        workkind = _findBetween(s, '<div class="strip jump">', '</div>',
-                                maxRes=1)
-        if workkind:
-            workkind[:] = _findBetween(workkind[0], 'href="#', '</a>')
-        else:
-            # Assume there's only one section and/or there are no
-            # section links, for some reason.
-            workkind[:] = _findBetween(s, '<h5><a name=', '</a></h5>')
-            workkind[:] = [x.lstrip('"').rstrip(':').lower() for x in workkind]
+        workkind = _findBetween(s, 'id="jumpto_', '</a>')
         ws = []
         for work in workkind:
-            wsplit = work.split('">', 1)
+            sep = '" >'
+            if '">' in work:
+                sep = '">'
+            wsplit = work.split(sep, 1)
             if len(wsplit) == 2:
                 sect = wsplit[0]
                 if '"' in sect:
@@ -600,16 +608,22 @@ class IMDbMobileAccessSystem(IMDbHTTPAccessSystem):
             else:
                 inisect = s.find('<a name="%s' % sect)
             if inisect != -1:
-                endsect = s[inisect:].find('</ol>')
+                endsect = s[inisect:].find('<div id="filmo-head-')
                 if endsect != -1: raws = s[inisect:inisect+endsect]
             if not raws: continue
-            mlist = _findBetween(raws, '<li>', ('</li>', '<br>', '<br/>'))
+            mlist = _findBetween(raws, '<div class="filmo-row',
+                    ('<div class="clear"/>',))
             for m in mlist:
+                fCB = m.find('>')
+                if fCB != -1:
+                    m = m[fCB+1:].lstrip()
+                m = re_filmo_episodes.sub('', m)
                 # For every movie in the current section.
                 movieID = re_imdbID.findall(m)
                 if not movieID:
                     self._mobile_logger.debug('no movieID in %s', m)
                     continue
+                m = m.replace('<br/>', ' .... ', 1)
                 if not _parseChr:
                     chrIndx = m.find(' .... ')
                 else:
@@ -638,14 +652,22 @@ class IMDbMobileAccessSystem(IMDbHTTPAccessSystem):
                     if stendidx != -1:
                         status = _unHtml(m[stidx+3:stendidx])
                         m = m.replace(m[stidx+3:stendidx], '')
+                year = _findBetween(m, 'year_column">', '</span>', maxRes=1)
+                if year:
+                    year = year[0]
+                    m = m.replace('<span class="year_column">%s</span>' % year,
+                            '')
+                else:
+                    year = None
                 m = _unHtml(m)
                 if not m:
-                    self._mobile_logger.warn('no title fo rmovieID %s', movieID)
+                    self._mobile_logger.warn('no title for movieID %s', movieID)
                     continue
                 movie = build_movie(m, movieID=movieID, status=status,
                                     roleID=chids, modFunct=self._defModFunct,
                                     accessSystem=self.accessSystem,
-                                    _parsingCharacter=_parseChr)
+                                    _parsingCharacter=_parseChr, year=year)
+                sectName = sectName.split(':')[0]
                 r.setdefault(sectName, []).append(movie)
         # If available, take the always correct name from a form.
         itag = _getTagsWith(s, 'NAME="primary"', maxRes=1)

libs/imdb/parser/sql/alchemyadapter.py

@@ -0,0 +1,508 @@
+"""
+parser.sql.alchemyadapter module (imdb.parser.sql package).
+
+This module adapts the SQLAlchemy ORM to the internal mechanism.
+
+Copyright 2008-2010 Davide Alberani <da@erlug.linux.it>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import re
+import sys
+import logging
+from sqlalchemy import *
+from sqlalchemy import schema
+try: from sqlalchemy import exc # 0.5
+except ImportError: from sqlalchemy import exceptions as exc # 0.4
+
+_alchemy_logger = logging.getLogger('imdbpy.parser.sql.alchemy')
+
+try:
+    import migrate.changeset
+    HAS_MC = True
+except ImportError:
+    HAS_MC = False
+    _alchemy_logger.warn('Unable to import migrate.changeset: Foreign ' \
+                         'Keys will not be created.')
+
+from imdb._exceptions import IMDbDataAccessError
+from dbschema import *
+
+# Used to convert table and column names.
+re_upper = re.compile(r'([A-Z])')
+
+# XXX: I'm not sure at all that this is the best method to connect
+#      to the database and bind that connection to every table.
+metadata = MetaData()
+
+# Maps our placeholders to SQLAlchemy's column types.
+MAP_COLS = {
+    INTCOL: Integer,
+    UNICODECOL: UnicodeText,
+    STRINGCOL: String
+}
+
+
+class NotFoundError(IMDbDataAccessError):
+    """Exception raised when Table.get(id) returns no value."""
+    pass
+
+
+def _renameTable(tname):
+    """Build the name of a table, as done by SQLObject."""
+    tname = re_upper.sub(r'_\1', tname)
+    if tname.startswith('_'):
+        tname = tname[1:]
+    return tname.lower()
+
+def _renameColumn(cname):
+    """Build the name of a column, as done by SQLObject."""
+    cname = cname.replace('ID', 'Id')
+    return _renameTable(cname)
+
+
+class DNNameObj(object):
+    """Used to access table.sqlmeta.columns[column].dbName (a string)."""
+    def __init__(self, dbName):
+        self.dbName = dbName
+
+    def __repr__(self):
+        return '<DNNameObj(dbName=%s) [id=%s]>' % (self.dbName, id(self))
+
+
+class DNNameDict(object):
+    """Used to access table.sqlmeta.columns (a dictionary)."""
+    def __init__(self, colMap):
+        self.colMap = colMap
+
+    def __getitem__(self, key):
+        return DNNameObj(self.colMap[key])
+
+    def __repr__(self):
+        return '<DNNameDict(colMap=%s) [id=%s]>' % (self.colMap, id(self))
+
+
+class SQLMetaAdapter(object):
+    """Used to access table.sqlmeta (an object with .table, .columns and
+    .idName attributes)."""
+    def __init__(self, table, colMap=None):
+        self.table = table
+        if colMap is None:
+            colMap = {}
+        self.colMap = colMap
+
+    def __getattr__(self, name):
+        if name == 'table':
+            return getattr(self.table, name)
+        if name == 'columns':
+            return DNNameDict(self.colMap)
+        if name == 'idName':
+            return self.colMap.get('id', 'id')
+        return None
+
+    def __repr__(self):
+        return '<SQLMetaAdapter(table=%s, colMap=%s) [id=%s]>' % \
+                (repr(self.table), repr(self.colMap), id(self))
+
+
+class QAdapter(object):
+    """Used to access table.q attribute (remapped to SQLAlchemy table.c)."""
+    def __init__(self, table, colMap=None):
+        self.table = table
+        if colMap is None:
+            colMap = {}
+        self.colMap = colMap
+
+    def __getattr__(self, name):
+        try: return getattr(self.table.c, self.colMap[name])
+        except KeyError, e: raise AttributeError, "unable to get '%s'" % name
+
+    def __repr__(self):
+        return '<QAdapter(table=%s, colMap=%s) [id=%s]>' % \
+                (repr(self.table), repr(self.colMap), id(self))
+
+
+class RowAdapter(object):
+    """Adapter for a SQLAlchemy RowProxy object."""
+    def __init__(self, row, table, colMap=None):
+        self.row = row
+        # FIXME: it's OBSCENE that 'table' should be passed from
+        #        TableAdapter through ResultAdapter only to land here,
+        #        where it's used to directly update a row item.
+        self.table = table
+        if colMap is None:
+            colMap = {}
+        self.colMap = colMap
+        self.colMapKeys = colMap.keys()
+
+    def __getattr__(self, name):
+        try: return getattr(self.row, self.colMap[name])
+        except KeyError, e: raise AttributeError, "unable to get '%s'" % name
+
+    def __setattr__(self, name, value):
+        # FIXME: I can't even think about how much performances suffer,
+        #        for this horrible hack (and it's used so rarely...)
+        #        For sure something like a "property" to map column names
+        #        to getter/setter functions would be much better, but it's
+        #        not possible (or at least not easy) to build them for a
+        #        single instance.
+        if name in self.__dict__.get('colMapKeys', ()):
+            # Trying to update a value in the database.
+            row = self.__dict__['row']
+            table = self.__dict__['table']
+            colMap = self.__dict__['colMap']
+            params = {colMap[name]: value}
+            table.update(table.c.id==row.id).execute(**params)
+            # XXX: minor bug: after a value is assigned with the
+            #      'rowAdapterInstance.colName = value' syntax, for some
+            #      reason rowAdapterInstance.colName still returns the
+            #      previous value (even if the database is updated).
+            #      Fix it?  I'm not even sure it's ever used.
+            return
+        # For every other attribute.
+        object.__setattr__(self, name, value)
+
+    def __repr__(self):
+        return '<RowAdapter(row=%s, table=%s, colMap=%s) [id=%s]>' % \
+                (repr(self.row), repr(self.table), repr(self.colMap), id(self))
+
+
+class ResultAdapter(object):
+    """Adapter for a SQLAlchemy ResultProxy object."""
+    def __init__(self, result, table, colMap=None):
+        self.result = result
+        self.table = table
+        if colMap is None:
+            colMap = {}
+        self.colMap = colMap
+
+    def count(self):
+        return len(self)
+
+    def __len__(self):
+        # FIXME: why sqlite returns -1? (that's wrooong!)
+        if self.result.rowcount == -1:
+            return 0
+        return self.result.rowcount
+
+    def __getitem__(self, key):
+        res = list(self.result)[key]
+        if not isinstance(key, slice):
+            # A single item.
+            return RowAdapter(res, self.table, colMap=self.colMap)
+        else:
+            # A (possible empty) list of items.
+            return [RowAdapter(x, self.table, colMap=self.colMap)
+                    for x in res]
+
+    def __iter__(self):
+        for item in self.result:
+            yield RowAdapter(item, self.table, colMap=self.colMap)
+
+    def __repr__(self):
+        return '<ResultAdapter(result=%s, table=%s, colMap=%s) [id=%s]>' % \
+                (repr(self.result), repr(self.table),
+                    repr(self.colMap), id(self))
+
+
+class TableAdapter(object):
+    """Adapter for a SQLAlchemy Table object, to mimic a SQLObject class."""
+    def __init__(self, table, uri=None):
+        """Initialize a TableAdapter object."""
+        self._imdbpySchema = table
+        self._imdbpyName = table.name
+        self.connectionURI = uri
+        self.colMap = {}
+        columns = []
+        for col in table.cols:
+            # Column's paramters.
+            params = {'nullable': True}
+            params.update(col.params)
+            if col.name == 'id':
+                params['primary_key'] = True
+            if 'notNone' in params:
+                params['nullable'] = not params['notNone']
+                del params['notNone']
+            cname = _renameColumn(col.name)
+            self.colMap[col.name] = cname
+            colClass = MAP_COLS[col.kind]
+            colKindParams = {}
+            if 'length' in params:
+                colKindParams['length'] = params['length']
+                del params['length']
+            elif colClass is UnicodeText and col.index:
+                # XXX: limit length for UNICODECOLs that will have an index.
+                #      this can result in name.name and title.title truncations!
+                colClass = Unicode
+                # Should work for most of the database servers.
+                length = 511
+                if self.connectionURI:
+                    if self.connectionURI.startswith('mysql'):
+                        # To stay compatible with MySQL 4.x.
+                        length = 255
+                colKindParams['length'] = length
+            elif self._imdbpyName == 'PersonInfo' and col.name == 'info':
+                if self.connectionURI:
+                    if self.connectionURI.startswith('ibm'):
+                        # There are some entries longer than 32KB.
+                        colClass = CLOB
+                        # I really do hope that this space isn't wasted
+                        # for each other shorter entry... <g>
+                        colKindParams['length'] = 68*1024
+            colKind = colClass(**colKindParams)
+            if 'alternateID' in params:
+                # There's no need to handle them here.
+                del params['alternateID']
+            # Create a column.
+            colObj = Column(cname, colKind, **params)
+            columns.append(colObj)
+        self.tableName = _renameTable(table.name)
+        # Create the table.
+        self.table = Table(self.tableName, metadata, *columns)
+        self._ta_insert = self.table.insert()
+        self._ta_select = self.table.select
+        # Adapters for special attributes.
+        self.q = QAdapter(self.table, colMap=self.colMap)
+        self.sqlmeta = SQLMetaAdapter(self.table, colMap=self.colMap)
+
+    def select(self, conditions=None):
+        """Return a list of results."""
+        result = self._ta_select(conditions).execute()
+        return ResultAdapter(result, self.table, colMap=self.colMap)
+
+    def get(self, theID):
+        """Get an object given its ID."""
+        result = self.select(self.table.c.id == theID)
+        #if not result:
+        #    raise NotFoundError, 'no data for ID %s' % theID
+        # FIXME: isn't this a bit risky?  We can't check len(result),
+        #        because sqlite returns -1...
+        #        What about converting it to a list and getting the first item?
+        try:
+            return result[0]
+        except KeyError:
+            raise NotFoundError, 'no data for ID %s' % theID
+
+    def dropTable(self, checkfirst=True):
+        """Drop the table."""
+        dropParams = {'checkfirst': checkfirst}
+        # Guess what?  Another work-around for a ibm_db bug.
+        if self.table.bind.engine.url.drivername.startswith('ibm_db'):
+            del dropParams['checkfirst']
+        try:
+            self.table.drop(**dropParams)
+        except exc.ProgrammingError:
+            # As above: re-raise the exception, but only if it's not ibm_db.
+            if not self.table.bind.engine.url.drivername.startswith('ibm_db'):
+                raise
+
+    def createTable(self, checkfirst=True):
+        """Create the table."""
+        self.table.create(checkfirst=checkfirst)
+        # Create indexes for alternateID columns (other indexes will be
+        # created later, at explicit request for performances reasons).
+        for col in self._imdbpySchema.cols:
+            if col.name == 'id':
+                continue
+            if col.params.get('alternateID', False):
+                self._createIndex(col, checkfirst=checkfirst)
+
+    def _createIndex(self, col, checkfirst=True):
+        """Create an index for a given (schema) column."""
+        # XXX: indexLen is ignored in SQLAlchemy, and that means that
+        #      indexes will be over the whole 255 chars strings...
+        # NOTE: don't use a dot as a separator, or DB2 will do
+        #       nasty things.
+        idx_name = '%s_%s' % (self.table.name, col.index or col.name)
+        if checkfirst:
+            for index in self.table.indexes:
+                if index.name == idx_name:
+                    return
+        idx = Index(idx_name, getattr(self.table.c, self.colMap[col.name]))
+        # XXX: beware that exc.OperationalError can be raised, is some
+        #      strange circumstances; that's why the index name doesn't
+        #      follow the SQLObject convention, but includes the table name:
+        #      sqlite, for example, expects index names to be unique at
+        #      db-level.
+        try:
+            idx.create()
+        except exc.OperationalError, e:
+            _alchemy_logger.warn('Skipping creation of the %s.%s index: %s' %
+                                (self.sqlmeta.table, col.name, e))
+
+    def addIndexes(self, ifNotExists=True):
+        """Create all required indexes."""
+        for col in self._imdbpySchema.cols:
+            if col.index:
+                self._createIndex(col, checkfirst=ifNotExists)
+
+    def addForeignKeys(self, mapTables, ifNotExists=True):
+        """Create all required foreign keys."""
+        if not HAS_MC:
+            return
+        # It seems that there's no reason to prevent the creation of
+        # indexes for columns with FK constrains: if there's already
+        # an index, the FK index is not created.
+        countCols = 0
+        for col in self._imdbpySchema.cols:
+            countCols += 1
+            if not col.foreignKey:
+                continue
+            fks = col.foreignKey.split('.', 1)
+            foreignTableName = fks[0]
+            if len(fks) == 2:
+                foreignColName = fks[1]
+            else:
+                foreignColName = 'id'
+            foreignColName = mapTables[foreignTableName].colMap.get(
+                                                foreignColName, foreignColName)
+            thisColName = self.colMap.get(col.name, col.name)
+            thisCol = self.table.columns[thisColName]
+            foreignTable = mapTables[foreignTableName].table
+            foreignCol = getattr(foreignTable.c, foreignColName)
+            # Need to explicitly set an unique name, otherwise it will
+            # explode, if two cols points to the same table.
+            fkName = 'fk_%s_%s_%d' % (foreignTable.name, foreignColName,
+                                        countCols)
+            constrain = migrate.changeset.ForeignKeyConstraint([thisCol],
+                                                        [foreignCol],
+                                                        name=fkName)
+            try:
+                constrain.create()
+            except exc.OperationalError:
+                continue
+
+    def __call__(self, *args, **kwds):
+        """To insert a new row with the syntax: TableClass(key=value, ...)"""
+        taArgs = {}
+        for key, value in kwds.items():
+            taArgs[self.colMap.get(key, key)] = value
+        self._ta_insert.execute(*args, **taArgs)
+
+    def __repr__(self):
+        return '<TableAdapter(table=%s) [id=%s]>' % (repr(self.table), id(self))
+
+
+# Module-level "cache" for SQLObject classes, to prevent
+# "Table 'tableName' is already defined for this MetaData instance" errors,
+# when two or more connections to the database are made.
+# XXX: is this the best way to act?
+TABLES_REPOSITORY = {}
+
+def getDBTables(uri=None):
+    """Return a list of TableAdapter objects to be used to access the
+    database through the SQLAlchemy ORM.  The connection uri is optional, and
+    can be used to tailor the db schema to specific needs."""
+    DB_TABLES = []
+    for table in DB_SCHEMA:
+        if table.name in TABLES_REPOSITORY:
+            DB_TABLES.append(TABLES_REPOSITORY[table.name])
+            continue
+        tableAdapter = TableAdapter(table, uri)
+        DB_TABLES.append(tableAdapter)
+        TABLES_REPOSITORY[table.name] = tableAdapter
+    return DB_TABLES
+
+
+# Functions used to emulate SQLObject's logical operators.
+def AND(*params):
+    """Emulate SQLObject's AND."""
+    return and_(*params)
+
+def OR(*params):
+    """Emulate SQLObject's OR."""
+    return or_(*params)
+
+def IN(item, inList):
+    """Emulate SQLObject's IN."""
+    if not isinstance(item, schema.Column):
+        return OR(*[x == item for x in inList])
+    else:
+        return item.in_(inList)
+
+def ISNULL(x):
+    """Emulate SQLObject's ISNULL."""
+    # XXX: Should we use null()?  Can null() be a global instance?
+    # XXX: Is it safe to test None with the == operator, in this case?
+    return x == None
+
+def ISNOTNULL(x):
+    """Emulate SQLObject's ISNOTNULL."""
+    return x != None
+
+def CONTAINSSTRING(expr, pattern):
+    """Emulate SQLObject's CONTAINSSTRING."""
+    return expr.like('%%%s%%' % pattern)
+
+
+def toUTF8(s):
+    """For some strange reason, sometimes SQLObject wants utf8 strings
+    instead of unicode; with SQLAlchemy we just return the unicode text."""
+    return s
+
+
+class _AlchemyConnection(object):
+    """A proxy for the connection object, required since _ConnectionFairy
+    uses __slots__."""
+    def __init__(self, conn):
+        self.conn = conn
+
+    def __getattr__(self, name):
+        return getattr(self.conn, name)
+
+
+def setConnection(uri, tables, encoding='utf8', debug=False):
+    """Set connection for every table."""
+    # FIXME: why on earth MySQL requires an additional parameter,
+    #        is well beyond my understanding...
+    if uri.startswith('mysql'):
+        if '?' in uri:
+            uri += '&'
+        else:
+            uri += '?'
+        uri += 'charset=%s' % encoding
+    params = {'encoding': encoding}
+    if debug:
+        params['echo'] = True
+    if uri.startswith('ibm_db'):
+        # Try to work-around a possible bug of the ibm_db DB2 driver.
+        params['convert_unicode'] = True
+    # XXX: is this the best way to connect?
+    engine = create_engine(uri, **params)
+    metadata.bind = engine
+    eng_conn = engine.connect()
+    if uri.startswith('sqlite'):
+        major = sys.version_info[0]
+        minor = sys.version_info[1]
+        if major > 2 or (major == 2 and minor > 5):
+            eng_conn.connection.connection.text_factory = str
+    # XXX: OH MY, THAT'S A MESS!
+    #      We need to return a "connection" object, with the .dbName
+    #      attribute set to the db engine name (e.g. "mysql"), .paramstyle
+    #      set to the style of the paramters for query() calls, and the
+    #      .module attribute set to a module (?) with .OperationalError and
+    #      .IntegrityError attributes.
+    #      Another attribute of "connection" is the getConnection() function,
+    #      used to return an object with a .cursor() method.
+    connection = _AlchemyConnection(eng_conn.connection)
+    paramstyle = eng_conn.dialect.paramstyle
+    connection.module = eng_conn.dialect.dbapi
+    connection.paramstyle = paramstyle
+    connection.getConnection = lambda: connection.connection
+    connection.dbName = engine.url.drivername
+    return connection
+
+

libs/imdb/parser/sql/cutils.c

@@ -0,0 +1,269 @@
+/*
+ * cutils.c module.
+ *
+ * Miscellaneous functions to speed up the IMDbPY package.
+ *
+ * Contents:
+ * - pyratcliff():
+ *   Function that implements the Ratcliff-Obershelp comparison
+ *   amongst Python strings.
+ *
+ * - pysoundex():
+ *   Return a soundex code string, for the given string.
+ *
+ * Copyright 2004-2009 Davide Alberani <da@erlug.linux.it>
+ * Released under the GPL license.
+ *
+ * NOTE: The Ratcliff-Obershelp part was heavily based on code from the
+ * "simil" Python module.
+ * The "simil" module is copyright of Luca Montecchiani <cbm64 _at_ inwind.it>
+ * and can be found here: http://spazioinwind.libero.it/montecchiani/
+ * It was released under the GPL license; original comments are leaved
+ * below.
+ *
+ */
+
+
+/*========== Ratcliff-Obershelp ==========*/
+/*****************************************************************************
+ *
+ * Stolen code from :
+ *
+ * [Python-Dev] Why is soundex marked obsolete?
+ * by Eric S. Raymond [4]esr@thyrsus.com
+ * on Sun, 14 Jan 2001 14:09:01 -0500
+ *
+ *****************************************************************************/
+
+/*****************************************************************************
+ *
+ * Ratcliff-Obershelp common-subpattern similarity.
+ *
+ * This code first appeared in a letter to the editor in Doctor
+ * Dobbs's Journal, 11/1988.  The original article on the algorithm,
+ * "Pattern Matching by Gestalt" by John Ratcliff, had appeared in the
+ * July 1988 issue (#181) but the algorithm was presented in assembly.
+ * The main drawback of the Ratcliff-Obershelp algorithm is the cost
+ * of the pairwise comparisons.  It is significantly more expensive
+ * than stemming, Hamming distance, soundex, and the like.
+ *
+ * Running time quadratic in the data size, memory usage constant.
+ *
+ *****************************************************************************/
+
+#include <Python.h>
+
+#define DONTCOMPARE_NULL    0.0
+#define DONTCOMPARE_SAME    1.0
+#define COMPARE             2.0
+#define STRING_MAXLENDIFFER 0.7
+
+/* As of 05 Mar 2008, the longest title is ~600 chars. */
+#define MXLINELEN   1023
+
+#define MAX(a,b) ((a) > (b) ? (a) : (b))
+
+
+//*****************************************
+// preliminary check....
+//*****************************************
+static float
+strings_check(char const *s, char const *t)
+{
+    float threshold;    // lenght difference
+    int s_len = strlen(s);    // length of s
+    int t_len = strlen(t);    // length of t
+
+    // NULL strings ?
+    if ((t_len * s_len) == 0)
+        return (DONTCOMPARE_NULL);
+
+    // the same ?
+    if (strcmp(s, t) == 0)
+        return (DONTCOMPARE_SAME);
+
+    // string lenght difference threshold
+    // we don't want to compare too different lenght strings ;)
+    if (s_len < t_len)
+        threshold = (float) s_len / (float) t_len;
+    else
+        threshold = (float) t_len / (float) s_len;
+    if (threshold < STRING_MAXLENDIFFER)
+        return (DONTCOMPARE_NULL);
+
+    // proceed
+    return (COMPARE);
+}
+
+
+static int
+RatcliffObershelp(char *st1, char *end1, char *st2, char *end2)
+{
+    register char *a1, *a2;
+    char *b1, *b2;
+    char *s1 = st1, *s2 = st2;    /* initializations are just to pacify GCC */
+    short max, i;
+
+    if (end1 <= st1 || end2 <= st2)
+        return (0);
+    if (end1 == st1 + 1 && end2 == st2 + 1)
+        return (0);
+
+    max = 0;
+    b1 = end1;
+    b2 = end2;
+
+    for (a1 = st1; a1 < b1; a1++) {
+        for (a2 = st2; a2 < b2; a2++) {
+            if (*a1 == *a2) {
+                /* determine length of common substring */
+                for (i = 1; a1[i] && (a1[i] == a2[i]); i++)
+                    continue;
+                if (i > max) {
+                    max = i;
+                    s1 = a1;
+                    s2 = a2;
+                    b1 = end1 - max;
+                    b2 = end2 - max;
+                }
+            }
+        }
+    }
+    if (!max)
+        return (0);
+    max += RatcliffObershelp(s1 + max, end1, s2 + max, end2);    /* rhs */
+    max += RatcliffObershelp(st1, s1, st2, s2);    /* lhs */
+    return max;
+}
+
+
+static float
+ratcliff(char *s1, char *s2)
+/* compute Ratcliff-Obershelp similarity of two strings */
+{
+    int l1, l2;
+    float res;
+
+    // preliminary tests
+    res = strings_check(s1, s2);
+    if (res != COMPARE)
+        return(res);
+
+    l1 = strlen(s1);
+    l2 = strlen(s2);
+
+    return 2.0 * RatcliffObershelp(s1, s1 + l1, s2, s2 + l2) / (l1 + l2);
+}
+
+
+/* Change a string to lowercase. */
+static void
+strtolower(char *s1)
+{
+    int i;
+    for (i=0; i < strlen(s1); i++) s1[i] = tolower(s1[i]);
+}
+
+
+/* Ratcliff-Obershelp for two python strings; returns a python float. */
+static PyObject*
+pyratcliff(PyObject *self, PyObject *pArgs)
+{
+    char *s1 = NULL;
+    char *s2 = NULL;
+    PyObject *discard = NULL;
+    char s1copy[MXLINELEN+1];
+    char s2copy[MXLINELEN+1];
+
+    /* The optional PyObject parameter is here to be compatible
+     * with the pure python implementation, which uses a
+     * difflib.SequenceMatcher object. */
+    if (!PyArg_ParseTuple(pArgs, "ss|O", &s1, &s2, &discard))
+        return NULL;
+
+    strncpy(s1copy, s1, MXLINELEN);
+    strncpy(s2copy, s2, MXLINELEN);
+    /* Work on copies. */
+    strtolower(s1copy);
+    strtolower(s2copy);
+
+    return Py_BuildValue("f", ratcliff(s1copy, s2copy));
+}
+
+
+/*========== soundex ==========*/
+/* Max length of the soundex code to output (an uppercase char and
+ * _at most_ 4 digits). */
+#define SOUNDEX_LEN 5
+
+/* Group Number Lookup Table  */
+static char soundTable[26] =
+{ 0 /* A */, '1' /* B */, '2' /* C */, '3' /* D */, 0 /* E */, '1' /* F */,
+ '2' /* G */, 0 /* H */, 0 /* I */, '2' /* J */, '2' /* K */, '4' /* L */,
+ '5' /* M */, '5' /* N */, 0 /* O */, '1' /* P */, '2' /* Q */, '6' /* R */,
+ '2' /* S */, '3' /* T */, 0 /* U */, '1' /* V */, 0 /* W */, '2' /* X */,
+  0 /* Y */, '2' /* Z */};
+
+static PyObject*
+pysoundex(PyObject *self, PyObject *pArgs)
+{
+    int i, j, n;
+    char *s = NULL;
+    char word[MXLINELEN+1];
+    char soundCode[SOUNDEX_LEN+1];
+    char c;
+
+    if (!PyArg_ParseTuple(pArgs, "s", &s))
+        return NULL;
+
+    j = 0;
+    n = strlen(s);
+
+    /* Convert to uppercase and exclude non-ascii chars. */
+    for (i = 0; i < n; i++) {
+        c = toupper(s[i]);
+        if (c < 91 && c > 64) {
+            word[j] = c;
+            j++;
+        }
+    }
+    word[j] = '\0';
+
+    n = strlen(word);
+    if (n == 0) {
+        /* If the string is empty, returns None. */
+        return Py_BuildValue("");
+    }
+    soundCode[0] = word[0];
+
+    /* Build the soundCode string. */
+    j = 1;
+    for (i = 1; j < SOUNDEX_LEN && i < n; i++) {
+        c = soundTable[(word[i]-65)];
+        /* Compact zeroes and equal consecutive digits ("12234112"->"123412") */
+        if (c != 0 && c != soundCode[j-1]) {
+                soundCode[j++] = c;
+        }
+    }
+    soundCode[j] = '\0';
+
+    return Py_BuildValue("s", soundCode);
+}
+
+
+static PyMethodDef cutils_methods[] = {
+    {"ratcliff", pyratcliff,
+        METH_VARARGS, "Ratcliff-Obershelp similarity."},
+    {"soundex", pysoundex,
+        METH_VARARGS, "Soundex code for strings."},
+    {NULL}
+};
+
+
+void
+initcutils(void)
+{
+    Py_InitModule("cutils", cutils_methods);
+}
+
+

libs/imdb/parser/sql/dbschema.py

@@ -0,0 +1,461 @@
+#-*- encoding: utf-8 -*-
+"""
+parser.sql.dbschema module (imdb.parser.sql package).
+
+This module provides the schema used to describe the layout of the
+database used by the imdb.parser.sql package; functions to create/drop
+tables and indexes are also provided.
+
+Copyright 2005-2010 Davide Alberani <da@erlug.linux.it>
+               2006 Giuseppe "Cowo" Corbelli <cowo --> lugbs.linux.it>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import logging
+
+_dbschema_logger = logging.getLogger('imdbpy.parser.sql.dbschema')
+
+
+# Placeholders for column types.
+INTCOL = 1
+UNICODECOL = 2
+STRINGCOL = 3
+_strMap = {1: 'INTCOL', 2: 'UNICODECOL', 3: 'STRINGCOL'}
+
+class DBCol(object):
+    """Define column objects."""
+    def __init__(self, name, kind, **params):
+        self.name = name
+        self.kind = kind
+        self.index = None
+        self.indexLen = None
+        # If not None, two notations are accepted: 'TableName'
+        # and 'TableName.ColName'; in the first case, 'id' is assumed
+        # as the name of the pointed column.
+        self.foreignKey = None
+        if 'index' in params:
+            self.index = params['index']
+            del params['index']
+        if 'indexLen' in params:
+            self.indexLen = params['indexLen']
+            del params['indexLen']
+        if 'foreignKey' in params:
+            self.foreignKey = params['foreignKey']
+            del params['foreignKey']
+        self.params = params
+
+    def __str__(self):
+        """Class representation."""
+        s = '<DBCol %s %s' % (self.name, _strMap[self.kind])
+        if self.index:
+            s += ' INDEX'
+            if self.indexLen:
+                s += '[:%d]' % self.indexLen
+        if self.foreignKey:
+            s += ' FOREIGN'
+        if 'default' in self.params:
+            val = self.params['default']
+            if val is not None:
+                val = '"%s"' % val
+            s += ' DEFAULT=%s' % val
+        for param in self.params:
+            if param == 'default': continue
+            s += ' %s' % param.upper()
+        s += '>'
+        return s
+
+    def __repr__(self):
+        """Class representation."""
+        s = '<DBCol(name="%s", %s' % (self.name, _strMap[self.kind])
+        if self.index:
+            s += ', index="%s"' % self.index
+        if self.indexLen:
+             s += ', indexLen=%d' % self.indexLen
+        if self.foreignKey:
+            s += ', foreignKey="%s"' % self.foreignKey
+        for param in self.params:
+            val = self.params[param]
+            if isinstance(val, (unicode, str)):
+                val = u'"%s"' % val
+            s += ', %s=%s' % (param, val)
+        s += ')>'
+        return s
+
+
+class DBTable(object):
+    """Define table objects."""
+    def __init__(self, name, *cols, **kwds):
+        self.name = name
+        self.cols = cols
+        # Default values.
+        self.values = kwds.get('values', {})
+
+    def __str__(self):
+        """Class representation."""
+        return '<DBTable %s (%d cols, %d values)>' % (self.name,
+                len(self.cols), sum([len(v) for v in self.values.values()]))
+
+    def __repr__(self):
+        """Class representation."""
+        s = '<DBTable(name="%s"' % self.name
+        col_s = ', '.join([repr(col).rstrip('>').lstrip('<')
+                            for col in self.cols])
+        if col_s:
+            s += ', %s' % col_s
+        if self.values:
+            s += ', values=%s' % self.values
+        s += ')>'
+        return s
+
+
+# Default values to insert in some tables: {'column': (list, of, values, ...)}
+kindTypeDefs = {'kind': ('movie', 'tv series', 'tv movie', 'video movie',
+                        'tv mini series', 'video game', 'episode')}
+companyTypeDefs = {'kind': ('distributors', 'production companies',
+                        'special effects companies', 'miscellaneous companies')}
+infoTypeDefs = {'info': ('runtimes', 'color info', 'genres', 'languages',
+    'certificates', 'sound mix', 'tech info', 'countries', 'taglines',
+    'keywords', 'alternate versions', 'crazy credits', 'goofs',
+    'soundtrack', 'quotes', 'release dates', 'trivia', 'locations',
+    'mini biography', 'birth notes', 'birth date', 'height',
+    'death date', 'spouse', 'other works', 'birth name',
+    'salary history', 'nick names', 'books', 'agent address',
+    'biographical movies', 'portrayed in', 'where now', 'trade mark',
+    'interviews', 'article', 'magazine cover photo', 'pictorial',
+    'death notes', 'LD disc format', 'LD year', 'LD digital sound',
+    'LD official retail price', 'LD frequency response', 'LD pressing plant',
+    'LD length', 'LD language', 'LD review', 'LD spaciality', 'LD release date',
+    'LD production country', 'LD contrast', 'LD color rendition',
+    'LD picture format', 'LD video noise', 'LD video artifacts',
+    'LD release country', 'LD sharpness', 'LD dynamic range',
+    'LD audio noise', 'LD color information', 'LD group genre',
+    'LD quality program', 'LD close captions-teletext-ld-g',
+    'LD category', 'LD analog left', 'LD certification',
+    'LD audio quality', 'LD video quality', 'LD aspect ratio',
+    'LD analog right', 'LD additional information',
+    'LD number of chapter stops', 'LD dialogue intellegibility',
+    'LD disc size', 'LD master format', 'LD subtitles',
+    'LD status of availablility', 'LD quality of source',
+    'LD number of sides', 'LD video standard', 'LD supplement',
+    'LD original title', 'LD sound encoding', 'LD number', 'LD label',
+    'LD catalog number', 'LD laserdisc title', 'screenplay-teleplay',
+    'novel', 'adaption', 'book', 'production process protocol',
+    'printed media reviews', 'essays', 'other literature', 'mpaa',
+    'plot', 'votes distribution', 'votes', 'rating',
+    'production dates', 'copyright holder', 'filming dates', 'budget',
+    'weekend gross', 'gross', 'opening weekend', 'rentals',
+    'admissions', 'studios', 'top 250 rank', 'bottom 10 rank')}
+compCastTypeDefs = {'kind': ('cast', 'crew', 'complete', 'complete+verified')}
+linkTypeDefs = {'link': ('follows', 'followed by', 'remake of', 'remade as',
+                        'references', 'referenced in', 'spoofs', 'spoofed in',
+                        'features', 'featured in', 'spin off from', 'spin off',
+                        'version of', 'similar to', 'edited into',
+                        'edited from', 'alternate language version of',
+                        'unknown link')}
+roleTypeDefs = {'role': ('actor', 'actress', 'producer', 'writer',
+                        'cinematographer', 'composer', 'costume designer',
+                        'director', 'editor', 'miscellaneous crew',
+                        'production designer', 'guest')}
+
+# Schema of tables in our database.
+# XXX: Foreign keys can be used to create constrains between tables,
+#      but they create indexes in the database, and this
+#      means poor performances at insert-time.
+DB_SCHEMA = [
+    DBTable('Name',
+        # namePcodeCf is the soundex of the name in the canonical format.
+        # namePcodeNf is the soundex of the name in the normal format, if
+        # different from namePcodeCf.
+        # surnamePcode is the soundex of the surname, if different from the
+        # other two values.
+
+        # The 'id' column is simply skipped by SQLObject (it's a default);
+        # the alternateID attribute here will be ignored by SQLAlchemy.
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('name', UNICODECOL, notNone=True, index='idx_name', indexLen=6),
+        DBCol('imdbIndex', UNICODECOL, length=12, default=None),
+        DBCol('imdbID', INTCOL, default=None),
+        DBCol('namePcodeCf', STRINGCOL, length=5, default=None,
+                index='idx_pcodecf'),
+        DBCol('namePcodeNf', STRINGCOL, length=5, default=None,
+                index='idx_pcodenf'),
+        DBCol('surnamePcode', STRINGCOL, length=5, default=None,
+                index='idx_pcode'),
+        DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
+    ),
+
+    DBTable('CharName',
+        # namePcodeNf is the soundex of the name in the normal format.
+        # surnamePcode is the soundex of the surname, if different
+        # from namePcodeNf.
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('name', UNICODECOL, notNone=True, index='idx_name', indexLen=6),
+        DBCol('imdbIndex', UNICODECOL, length=12, default=None),
+        DBCol('imdbID', INTCOL, default=None),
+        DBCol('namePcodeNf', STRINGCOL, length=5, default=None,
+                index='idx_pcodenf'),
+        DBCol('surnamePcode', STRINGCOL, length=5, default=None,
+                index='idx_pcode'),
+        DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
+    ),
+
+    DBTable('CompanyName',
+        # namePcodeNf is the soundex of the name in the normal format.
+        # namePcodeSf is the soundex of the name plus the country code.
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('name', UNICODECOL, notNone=True, index='idx_name', indexLen=6),
+        DBCol('countryCode', UNICODECOL, length=255, default=None),
+        DBCol('imdbID', INTCOL, default=None),
+        DBCol('namePcodeNf', STRINGCOL, length=5, default=None,
+                index='idx_pcodenf'),
+        DBCol('namePcodeSf', STRINGCOL, length=5, default=None,
+                index='idx_pcodesf'),
+        DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
+    ),
+
+    DBTable('KindType',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('kind', STRINGCOL, length=15, default=None, alternateID=True),
+        values=kindTypeDefs
+    ),
+
+    DBTable('Title',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('title', UNICODECOL, notNone=True,
+                index='idx_title', indexLen=10),
+        DBCol('imdbIndex', UNICODECOL, length=12, default=None),
+        DBCol('kindID', INTCOL, notNone=True, foreignKey='KindType'),
+        DBCol('productionYear', INTCOL, default=None),
+        DBCol('imdbID', INTCOL, default=None),
+        DBCol('phoneticCode', STRINGCOL, length=5, default=None,
+                index='idx_pcode'),
+        DBCol('episodeOfID', INTCOL, default=None, index='idx_epof',
+                foreignKey='Title'),
+        DBCol('seasonNr', INTCOL, default=None),
+        DBCol('episodeNr', INTCOL, default=None),
+        # Maximum observed length is 44; 49 can store 5 comma-separated
+        # year-year pairs.
+        DBCol('seriesYears', STRINGCOL, length=49, default=None),
+        DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
+    ),
+
+    DBTable('CompanyType',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('kind', STRINGCOL, length=32, default=None, alternateID=True),
+        values=companyTypeDefs
+    ),
+
+    DBTable('AkaName',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('personID', INTCOL, notNone=True, index='idx_person',
+                foreignKey='Name'),
+        DBCol('name', UNICODECOL, notNone=True),
+        DBCol('imdbIndex', UNICODECOL, length=12, default=None),
+        DBCol('namePcodeCf',  STRINGCOL, length=5, default=None,
+                index='idx_pcodecf'),
+        DBCol('namePcodeNf',  STRINGCOL, length=5, default=None,
+                index='idx_pcodenf'),
+        DBCol('surnamePcode',  STRINGCOL, length=5, default=None,
+                index='idx_pcode'),
+        DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
+    ),
+
+    DBTable('AkaTitle',
+        # XXX: It's safer to set notNone to False, here.
+        #      alias for akas are stored completely in the AkaTitle table;
+        #      this means that episodes will set also a "tv series" alias name.
+        #      Reading the aka-title.list file it looks like there are
+        #      episode titles with aliases to different titles for both
+        #      the episode and the series title, while for just the series
+        #      there are no aliases.
+        #      E.g.:
+        #      aka title                                 original title
+        # "Series, The" (2005) {The Episode}  "Other Title" (2005) {Other Title}
+        # But there is no:
+        # "Series, The" (2005)                "Other Title" (2005)
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('movieID', INTCOL, notNone=True, index='idx_movieid',
+                foreignKey='Title'),
+        DBCol('title', UNICODECOL, notNone=True),
+        DBCol('imdbIndex', UNICODECOL, length=12, default=None),
+        DBCol('kindID', INTCOL, notNone=True, foreignKey='KindType'),
+        DBCol('productionYear', INTCOL, default=None),
+        DBCol('phoneticCode',  STRINGCOL, length=5, default=None,
+                index='idx_pcode'),
+        DBCol('episodeOfID', INTCOL, default=None, index='idx_epof',
+                foreignKey='AkaTitle'),
+        DBCol('seasonNr', INTCOL, default=None),
+        DBCol('episodeNr', INTCOL, default=None),
+        DBCol('note', UNICODECOL, default=None),
+        DBCol('md5sum', STRINGCOL, length=32, default=None, index='idx_md5')
+    ),
+
+    DBTable('RoleType',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('role', STRINGCOL, length=32, notNone=True, alternateID=True),
+        values=roleTypeDefs
+    ),
+
+    DBTable('CastInfo',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('personID', INTCOL, notNone=True, index='idx_pid',
+                foreignKey='Name'),
+        DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
+                foreignKey='Title'),
+        DBCol('personRoleID', INTCOL, default=None, index='idx_cid',
+                foreignKey='CharName'),
+        DBCol('note', UNICODECOL, default=None),
+        DBCol('nrOrder', INTCOL, default=None),
+        DBCol('roleID', INTCOL, notNone=True, foreignKey='RoleType')
+    ),
+
+    DBTable('CompCastType',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('kind', STRINGCOL, length=32, notNone=True, alternateID=True),
+        values=compCastTypeDefs
+    ),
+
+    DBTable('CompleteCast',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('movieID', INTCOL, index='idx_mid', foreignKey='Title'),
+        DBCol('subjectID', INTCOL, notNone=True, foreignKey='CompCastType'),
+        DBCol('statusID', INTCOL, notNone=True, foreignKey='CompCastType')
+    ),
+
+    DBTable('InfoType',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('info', STRINGCOL, length=32, notNone=True, alternateID=True),
+        values=infoTypeDefs
+    ),
+
+    DBTable('LinkType',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('link', STRINGCOL, length=32, notNone=True, alternateID=True),
+        values=linkTypeDefs
+    ),
+
+    DBTable('Keyword',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        # XXX: can't use alternateID=True, because it would create
+        #      a UNIQUE index; unfortunately (at least with a common
+        #      collation like utf8_unicode_ci) MySQL will consider
+        #      some different keywords identical - like
+        #      "fiancée" and "fiancee".
+        DBCol('keyword', UNICODECOL, length=255, notNone=True,
+                index='idx_keyword', indexLen=5),
+        DBCol('phoneticCode', STRINGCOL, length=5, default=None,
+                index='idx_pcode')
+    ),
+
+    DBTable('MovieKeyword',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
+                foreignKey='Title'),
+        DBCol('keywordID', INTCOL, notNone=True, index='idx_keywordid',
+                foreignKey='Keyword')
+    ),
+
+    DBTable('MovieLink',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
+                foreignKey='Title'),
+        DBCol('linkedMovieID', INTCOL, notNone=True, foreignKey='Title'),
+        DBCol('linkTypeID', INTCOL, notNone=True, foreignKey='LinkType')
+    ),
+
+    DBTable('MovieInfo',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
+                foreignKey='Title'),
+        DBCol('infoTypeID', INTCOL, notNone=True, foreignKey='InfoType'),
+        DBCol('info', UNICODECOL, notNone=True),
+        DBCol('note', UNICODECOL, default=None)
+    ),
+
+    # This table is identical to MovieInfo, except that both 'infoTypeID'
+    # and 'info' are indexed.
+    DBTable('MovieInfoIdx',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
+                foreignKey='Title'),
+        DBCol('infoTypeID', INTCOL, notNone=True, index='idx_infotypeid',
+                foreignKey='InfoType'),
+        DBCol('info', UNICODECOL, notNone=True, index='idx_info', indexLen=10),
+        DBCol('note', UNICODECOL, default=None)
+    ),
+
+    DBTable('MovieCompanies',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('movieID', INTCOL, notNone=True, index='idx_mid',
+                foreignKey='Title'),
+        DBCol('companyID', INTCOL, notNone=True, index='idx_cid',
+                foreignKey='CompanyName'),
+        DBCol('companyTypeID', INTCOL, notNone=True, foreignKey='CompanyType'),
+        DBCol('note', UNICODECOL, default=None)
+    ),
+
+    DBTable('PersonInfo',
+        DBCol('id', INTCOL, notNone=True, alternateID=True),
+        DBCol('personID', INTCOL, notNone=True, index='idx_pid',
+                foreignKey='Name'),
+        DBCol('infoTypeID', INTCOL, notNone=True, foreignKey='InfoType'),
+        DBCol('info', UNICODECOL, notNone=True),
+        DBCol('note', UNICODECOL, default=None)
+    )
+]
+
+
+# Functions to manage tables.
+def dropTables(tables, ifExists=True):
+    """Drop the tables."""
+    # In reverse order (useful to avoid errors about foreign keys).
+    DB_TABLES_DROP = list(tables)
+    DB_TABLES_DROP.reverse()
+    for table in DB_TABLES_DROP:
+        _dbschema_logger.info('dropping table %s', table._imdbpyName)
+        table.dropTable(ifExists)
+
+def createTables(tables, ifNotExists=True):
+    """Create the tables and insert default values."""
+    for table in tables:
+        # Create the table.
+        _dbschema_logger.info('creating table %s', table._imdbpyName)
+        table.createTable(ifNotExists)
+        # Insert default values, if any.
+        if table._imdbpySchema.values:
+            _dbschema_logger.info('inserting values into table %s',
+                                    table._imdbpyName)
+            for key in table._imdbpySchema.values:
+                for value in table._imdbpySchema.values[key]:
+                    table(**{key: unicode(value)})
+
+def createIndexes(tables, ifNotExists=True):
+    """Create the indexes in the database."""
+    for table in tables:
+        _dbschema_logger.info('creating indexes for table %s',
+                                table._imdbpyName)
+        table.addIndexes(ifNotExists)
+
+def createForeignKeys(tables, ifNotExists=True):
+    """Create Foreign Keys."""
+    mapTables = {}
+    for table in tables:
+        mapTables[table._imdbpyName] = table
+    for table in tables:
+        _dbschema_logger.info('creating foreign keys for table %s',
+                                table._imdbpyName)
+        table.addForeignKeys(mapTables, ifNotExists)
+

libs/imdb/parser/sql/objectadapter.py

@@ -0,0 +1,203 @@
+"""
+parser.sql.objectadapter module (imdb.parser.sql package).
+
+This module adapts the SQLObject ORM to the internal mechanism.
+
+Copyright 2008-2010 Davide Alberani <da@erlug.linux.it>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+import sys
+import logging
+
+from sqlobject import *
+from sqlobject.sqlbuilder import ISNULL, ISNOTNULL, AND, OR, IN, CONTAINSSTRING
+
+from dbschema import *
+
+_object_logger = logging.getLogger('imdbpy.parser.sql.object')
+
+
+# Maps our placeholders to SQLAlchemy's column types.
+MAP_COLS = {
+        INTCOL: IntCol,
+        UNICODECOL: UnicodeCol,
+        STRINGCOL: StringCol
+}
+
+
+# Exception raised when Table.get(id) returns no value.
+NotFoundError = SQLObjectNotFound
+
+
+# class method to be added to the SQLObject class.
+def addIndexes(cls, ifNotExists=True):
+    """Create all required indexes."""
+    for col in cls._imdbpySchema.cols:
+        if col.index:
+            idxName = col.index
+            colToIdx = col.name
+            if col.indexLen:
+                colToIdx = {'column': col.name, 'length': col.indexLen}
+            if idxName in [i.name for i in cls.sqlmeta.indexes]:
+                # Check if the index is already present.
+                continue
+            idx = DatabaseIndex(colToIdx, name=idxName)
+            cls.sqlmeta.addIndex(idx)
+    try:
+        cls.createIndexes(ifNotExists)
+    except dberrors.OperationalError, e:
+        _object_logger.warn('Skipping creation of the %s.%s index: %s' %
+                            (cls.sqlmeta.table, col.name, e))
+addIndexes = classmethod(addIndexes)
+
+
+# Global repository for "fake" tables with Foreign Keys - need to
+# prevent troubles if addForeignKeys is called more than one time.
+FAKE_TABLES_REPOSITORY = {}
+
+def _buildFakeFKTable(cls, fakeTableName):
+    """Return a "fake" table, with foreign keys where needed."""
+    countCols = 0
+    attrs = {}
+    for col in cls._imdbpySchema.cols:
+        countCols += 1
+        if col.name == 'id':
+            continue
+        if not col.foreignKey:
+            # A non-foreign key column - add it as usual.
+            attrs[col.name] = MAP_COLS[col.kind](**col.params)
+            continue
+        # XXX: Foreign Keys pointing to TableName.ColName not yet supported.
+        thisColName = col.name
+        if thisColName.endswith('ID'):
+            thisColName = thisColName[:-2]
+
+        fks = col.foreignKey.split('.', 1)
+        foreignTableName = fks[0]
+        if len(fks) == 2:
+            foreignColName = fks[1]
+        else:
+            foreignColName = 'id'
+        # Unused...
+        #fkName = 'fk_%s_%s_%d' % (foreignTableName, foreignColName,
+        #                        countCols)
+        # Create a Foreign Key column, with the correct references.
+        fk = ForeignKey(foreignTableName, name=thisColName, default=None)
+        attrs[thisColName] = fk
+    # Build a _NEW_ SQLObject subclass, with foreign keys, if needed.
+    newcls = type(fakeTableName, (SQLObject,), attrs)
+    return newcls
+
+def addForeignKeys(cls, mapTables, ifNotExists=True):
+    """Create all required foreign keys."""
+    # Do not even try, if there are no FK, in this table.
+    if not filter(None, [col.foreignKey for col in cls._imdbpySchema.cols]):
+        return
+    fakeTableName = 'myfaketable%s' % cls.sqlmeta.table
+    if fakeTableName in FAKE_TABLES_REPOSITORY:
+        newcls = FAKE_TABLES_REPOSITORY[fakeTableName]
+    else:
+        newcls = _buildFakeFKTable(cls, fakeTableName)
+        FAKE_TABLES_REPOSITORY[fakeTableName] = newcls
+    # Connect the class with foreign keys.
+    newcls.setConnection(cls._connection)
+    for col in cls._imdbpySchema.cols:
+        if col.name == 'id':
+            continue
+        if not col.foreignKey:
+            continue
+        # Get the SQL that _WOULD BE_ run, if we had to create
+        # this "fake" table.
+        fkQuery = newcls._connection.createReferenceConstraint(newcls,
+                                newcls.sqlmeta.columns[col.name])
+        if not fkQuery:
+            # Probably the db doesn't support foreign keys (SQLite).
+            continue
+        # Remove "myfaketable" to get references to _real_ tables.
+        fkQuery = fkQuery.replace('myfaketable', '')
+        # Execute the query.
+        newcls._connection.query(fkQuery)
+    # Disconnect it.
+    newcls._connection.close()
+addForeignKeys = classmethod(addForeignKeys)
+
+
+# Module-level "cache" for SQLObject classes, to prevent
+# "class TheClass is already in the registry" errors, when
+# two or more connections to the database are made.
+# XXX: is this the best way to act?
+TABLES_REPOSITORY = {}
+
+def getDBTables(uri=None):
+    """Return a list of classes to be used to access the database
+    through the SQLObject ORM.  The connection uri is optional, and
+    can be used to tailor the db schema to specific needs."""
+    DB_TABLES = []
+    for table in DB_SCHEMA:
+        if table.name in TABLES_REPOSITORY:
+            DB_TABLES.append(TABLES_REPOSITORY[table.name])
+            continue
+        attrs = {'_imdbpyName': table.name, '_imdbpySchema': table,
+                'addIndexes': addIndexes, 'addForeignKeys': addForeignKeys}
+        for col in table.cols:
+            if col.name == 'id':
+                continue
+            attrs[col.name] = MAP_COLS[col.kind](**col.params)
+        # Create a subclass of SQLObject.
+        # XXX: use a metaclass?  I can't see any advantage.
+        cls = type(table.name, (SQLObject,), attrs)
+        DB_TABLES.append(cls)
+        TABLES_REPOSITORY[table.name] = cls
+    return DB_TABLES
+
+
+def toUTF8(s):
+    """For some strange reason, sometimes SQLObject wants utf8 strings
+    instead of unicode."""
+    return s.encode('utf_8')
+
+
+def setConnection(uri, tables, encoding='utf8', debug=False):
+    """Set connection for every table."""
+    kw = {}
+    # FIXME: it's absolutely unclear what we should do to correctly
+    #        support unicode in MySQL; with some versions of SQLObject,
+    #        it seems that setting use_unicode=1 is the _wrong_ thing to do.
+    _uriLower = uri.lower()
+    if _uriLower.startswith('mysql'):
+        kw['use_unicode'] = 1
+        #kw['sqlobject_encoding'] = encoding
+        kw['charset'] = encoding
+    conn = connectionForURI(uri, **kw)
+    conn.debug = debug
+    if uri.startswith('sqlite'):
+        major = sys.version_info[0]
+        minor = sys.version_info[1]
+        if major > 2 or (major == 2 and minor > 5):
+            conn.connection.connection.text_factory = str
+    for table in tables:
+        table.setConnection(conn)
+        #table.sqlmeta.cacheValues = False
+        # FIXME: is it safe to set table._cacheValue to False?  Looks like
+        #        we can't retrieve correct values after an update (I think
+        #        it's never needed, but...)  Anyway, these are set to False
+        #        for performance reason at insert time (see imdbpy2sql.py).
+        table._cacheValue = False
+    # Required by imdbpy2sql.py.
+    conn.paramstyle = conn.module.paramstyle
+    return conn
+

libs/imdb/utils.py

@@ -141,6 +141,9 @@ def analyze_name(name, canonical=None):
         if cpi > opi and re_index.match(name[opi:cpi+1]):
             imdbIndex = name[opi+1:cpi]
             name = name[:opi].rstrip()
+        else:
+            # XXX: for the birth and death dates case like " (1926-2004)"
+            name = name[:opi-1]
     if not name:
         raise IMDbParserError, 'invalid name: "%s"' % original_n
     if canonical is not None:
@@ -377,6 +380,9 @@ def analyze_title(title, canonical=None, canonicalSeries=None,
     elif title.endswith('(V)'):
         kind = u'video movie'
         title = title[:-3].rstrip()
+    elif title.endswith('(video)'):
+        kind = u'video movie'
+        title = title[:-7].rstrip()
     elif title.endswith('(mini)'):
         kind = u'tv mini series'
         title = title[:-6].rstrip()
@@ -400,6 +406,9 @@ def analyze_title(title, canonical=None, canonicalSeries=None,
         if not kind:
             kind = u'tv series'
         title = title[1:-1].strip()
+    elif title.endswith('(TV series)'):
+        kind = u'tv series'
+        title = title[:-11].rstrip()
     if not title:
         raise IMDbParserError, 'invalid title: "%s"' % original_t
     if canonical is not None: