Merge pull request #817 from niphlod/fix/773

fixes issue #773

gluon/tests/test_is_url.py

@@ -498,7 +498,7 @@ class TestIsHttpUrl(unittest.TestCase):
             'http://1234567890.com.',
             'http://1234567890.com./path',
             'http://google.com./path',
-            'http://domain.xn--0zwm56d',
+            'http://domain.xn--d1acj3b',
             'http://127.123.0.256',
             'http://127.123.0.256/document/drawer',
             '127.123.0.256/document/',
@@ -669,5 +669,31 @@ class TestUnicode(unittest.TestCase):
 
 # ##############################################################################
 
+
+class TestSimple(unittest.TestCase):
+
+    def test_IS_URL(self):
+        rtn = IS_URL()('abc.com')
+        self.assertEqual(rtn, ('http://abc.com', None))
+        rtn = IS_URL(mode='generic')('abc.com')
+        self.assertEqual(rtn, ('abc.com', None))
+        rtn = IS_URL(allowed_schemes=['https'], prepend_scheme='https')('https://abc.com')
+        self.assertEqual(rtn, ('https://abc.com', None))
+        rtn = IS_URL(prepend_scheme='https')('abc.com')
+        self.assertEqual(rtn, ('https://abc.com', None))
+        rtn = IS_URL(mode='generic', allowed_schemes=['ftps', 'https'], prepend_scheme='https')('https://abc.com')
+        self.assertEqual(rtn, ('https://abc.com', None))
+        rtn = IS_URL(mode='generic', allowed_schemes=['ftps', 'https', None], prepend_scheme='https')('abc.com')
+        self.assertEqual(rtn, ('abc.com', None))
+        # regression test for issue 773
+        rtn = IS_URL()('domain.ninja')
+        self.assertEqual(rtn, ('http://domain.ninja', None))
+        # addition of allowed_tlds
+        rtn = IS_URL(allowed_tlds=['com', 'net', 'org'])('domain.ninja')
+        self.assertEqual(rtn, ('domain.ninja', 'Enter a valid URL'))
+        # mode = 'generic' doesn't consider allowed_tlds
+        rtn = IS_URL(mode='generic', allowed_tlds=['com', 'net', 'org'])('domain.ninja')
+        self.assertEqual(rtn, ('domain.ninja', None))
+
 if __name__ == '__main__':
     unittest.main()

gluon/tests/test_validators.py

@@ -583,8 +583,6 @@ class TestValidators(unittest.TestCase):
         rtn = IS_LENGTH(6)(a)
         self.assertEqual(rtn, (a, None))
 
-
-
     def test_IS_LOWER(self):
         rtn = IS_LOWER()('ABC')
         self.assertEqual(rtn, ('abc', None))
@@ -769,19 +767,6 @@ class TestValidators(unittest.TestCase):
         rtn = IS_UPPER()('ñ')
         self.assertEqual(rtn, ('\xc3\x91', None))
 
-    def test_IS_URL(self):
-        rtn = IS_URL()('abc.com')
-        self.assertEqual(rtn, ('http://abc.com', None))
-        rtn = IS_URL(mode='generic')('abc.com')
-        self.assertEqual(rtn, ('abc.com', None))
-        rtn = IS_URL(allowed_schemes=['https'], prepend_scheme='https')('https://abc.com')
-        self.assertEqual(rtn, ('https://abc.com', None))
-        rtn = IS_URL(prepend_scheme='https')('abc.com')
-        self.assertEqual(rtn, ('https://abc.com', None))
-        rtn = IS_URL(mode='generic', allowed_schemes=['ftps', 'https'], prepend_scheme='https')('https://abc.com')
-        self.assertEqual(rtn, ('https://abc.com', None))
-        rtn = IS_URL(mode='generic', allowed_schemes=['ftps', 'https', None], prepend_scheme='https')('abc.com')
-        self.assertEqual(rtn, ('abc.com', None))
 
     def test_IS_JSON(self):
         rtn = IS_JSON()('{"a": 100}')

gluon/validators.py

@@ -1579,300 +1579,173 @@ class IS_GENERIC_URL(Validator):
         # else the URL is not valid
         return (value, translate(self.error_message))
 
-# Sources (obtained 2008-Nov-11):
-#    http://en.wikipedia.org/wiki/Top-level_domain
-#    http://www.iana.org/domains/root/db/
+# Sources (obtained 2015-Feb-24):
+#    http://data.iana.org/TLD/tlds-alpha-by-domain.txt
+# see scripts/parse_top_level_domains.py for an easy update
 
 official_top_level_domains = [
-    'ac',
-    'ad',
-    'ae',
-    'aero',
-    'af',
-    'ag',
-    'ai',
-    'al',
-    'am',
-    'an',
-    'ao',
-    'aq',
-    'ar',
-    'arpa',
-    'as',
-    'asia',
-    'at',
-    'au',
-    'aw',
-    'ax',
-    'az',
-    'ba',
-    'bb',
-    'bd',
-    'be',
-    'bf',
-    'bg',
-    'bh',
-    'bi',
-    'biz',
-    'bj',
-    'bl',
-    'bm',
-    'bn',
-    'bo',
-    'br',
-    'bs',
-    'bt',
-    'bv',
-    'bw',
-    'by',
-    'bz',
-    'ca',
-    'cat',
-    'cc',
-    'cd',
-    'cf',
-    'cg',
-    'ch',
-    'ci',
-    'ck',
-    'cl',
-    'cm',
-    'cn',
-    'co',
-    'com',
-    'coop',
-    'cr',
-    'cu',
-    'cv',
-    'cx',
-    'cy',
+    # a
+    'abogado', 'ac', 'academy', 'accountants', 'active', 'actor',
+    'ad', 'adult', 'ae', 'aero', 'af', 'ag', 'agency', 'ai',
+    'airforce', 'al', 'allfinanz', 'alsace', 'am', 'amsterdam', 'an',
+    'android', 'ao', 'apartments', 'aq', 'aquarelle', 'ar', 'archi',
+    'army', 'arpa', 'as', 'asia', 'associates', 'at', 'attorney',
+    'au', 'auction', 'audio', 'autos', 'aw', 'ax', 'axa', 'az',
+    # b
+    'ba', 'band', 'bank', 'bar', 'barclaycard', 'barclays',
+    'bargains', 'bayern', 'bb', 'bd', 'be', 'beer', 'berlin', 'best',
+    'bf', 'bg', 'bh', 'bi', 'bid', 'bike', 'bingo', 'bio', 'biz',
+    'bj', 'black', 'blackfriday', 'bloomberg', 'blue', 'bm', 'bmw',
+    'bn', 'bnpparibas', 'bo', 'boo', 'boutique', 'br', 'brussels',
+    'bs', 'bt', 'budapest', 'build', 'builders', 'business', 'buzz',
+    'bv', 'bw', 'by', 'bz', 'bzh',
+    # c
+    'ca', 'cab', 'cal', 'camera', 'camp', 'cancerresearch', 'canon',
+    'capetown', 'capital', 'caravan', 'cards', 'care', 'career',
+    'careers', 'cartier', 'casa', 'cash', 'casino', 'cat',
+    'catering', 'cbn', 'cc', 'cd', 'center', 'ceo', 'cern', 'cf',
+    'cg', 'ch', 'channel', 'chat', 'cheap', 'christmas', 'chrome',
+    'church', 'ci', 'citic', 'city', 'ck', 'cl', 'claims',
+    'cleaning', 'click', 'clinic', 'clothing', 'club', 'cm', 'cn',
+    'co', 'coach', 'codes', 'coffee', 'college', 'cologne', 'com',
+    'community', 'company', 'computer', 'condos', 'construction',
+    'consulting', 'contractors', 'cooking', 'cool', 'coop',
+    'country', 'cr', 'credit', 'creditcard', 'cricket', 'crs',
+    'cruises', 'cu', 'cuisinella', 'cv', 'cw', 'cx', 'cy', 'cymru',
     'cz',
-    'de',
-    'dj',
-    'dk',
-    'dm',
-    'do',
-    'dz',
-    'ec',
-    'edu',
-    'ee',
-    'eg',
-    'eh',
-    'er',
-    'es',
-    'et',
-    'eu',
-    'example',
-    'fi',
-    'fj',
-    'fk',
-    'fm',
-    'fo',
-    'fr',
-    'ga',
-    'gb',
-    'gd',
-    'ge',
-    'gf',
-    'gg',
-    'gh',
-    'gi',
-    'gl',
-    'gm',
-    'gn',
-    'gov',
-    'gp',
-    'gq',
-    'gr',
-    'gs',
-    'gt',
-    'gu',
-    'gw',
-    'gy',
-    'hk',
-    'hm',
-    'hn',
-    'hr',
-    'ht',
-    'hu',
-    'id',
-    'ie',
-    'il',
-    'im',
-    'in',
-    'info',
-    'int',
-    'invalid',
-    'io',
-    'iq',
-    'ir',
-    'is',
-    'it',
-    'je',
-    'jm',
-    'jo',
-    'jobs',
-    'jp',
-    'ke',
-    'kg',
-    'kh',
-    'ki',
-    'km',
-    'kn',
-    'kp',
-    'kr',
-    'kw',
-    'ky',
-    'kz',
-    'la',
-    'lb',
-    'lc',
-    'li',
-    'lk',
-    'localhost',
-    'lr',
-    'ls',
-    'lt',
-    'lu',
-    'lv',
-    'ly',
-    'ma',
-    'mc',
-    'md',
-    'me',
-    'mf',
-    'mg',
-    'mh',
-    'mil',
-    'mk',
-    'ml',
-    'mm',
-    'mn',
-    'mo',
-    'mobi',
-    'mp',
-    'mq',
-    'mr',
-    'ms',
-    'mt',
-    'mu',
-    'museum',
-    'mv',
-    'mw',
-    'mx',
-    'my',
-    'mz',
-    'na',
-    'name',
-    'nc',
-    'ne',
-    'net',
-    'nf',
-    'ng',
-    'ni',
-    'nl',
-    'no',
-    'np',
-    'nr',
-    'nu',
-    'nz',
-    'om',
-    'org',
-    'pa',
-    'pe',
-    'pf',
-    'pg',
-    'ph',
-    'pk',
-    'pl',
-    'pm',
-    'pn',
-    'pr',
-    'pro',
-    'ps',
-    'pt',
-    'pw',
-    'py',
-    'qa',
-    're',
-    'ro',
-    'rs',
-    'ru',
-    'rw',
-    'sa',
-    'sb',
-    'sc',
-    'sd',
-    'se',
-    'sg',
-    'sh',
-    'si',
-    'sj',
-    'sk',
-    'sl',
-    'sm',
-    'sn',
-    'so',
-    'sr',
-    'st',
-    'su',
-    'sv',
-    'sy',
-    'sz',
-    'tc',
-    'td',
-    'tel',
-    'test',
-    'tf',
-    'tg',
-    'th',
-    'tj',
-    'tk',
-    'tl',
-    'tm',
-    'tn',
-    'to',
-    'tp',
-    'tr',
-    'travel',
-    'tt',
-    'tv',
-    'tw',
+    # d
+    'dabur', 'dad', 'dance', 'dating', 'day', 'dclk', 'de', 'deals',
+    'degree', 'delivery', 'democrat', 'dental', 'dentist', 'desi',
+    'design', 'dev', 'diamonds', 'diet', 'digital', 'direct',
+    'directory', 'discount', 'dj', 'dk', 'dm', 'dnp', 'do', 'docs',
+    'domains', 'doosan', 'durban', 'dvag', 'dz',
+    # e
+    'eat', 'ec', 'edu', 'education', 'ee', 'eg', 'email', 'emerck',
+    'energy', 'engineer', 'engineering', 'enterprises', 'equipment',
+    'er', 'es', 'esq', 'estate', 'et', 'eu', 'eurovision', 'eus',
+    'events', 'everbank', 'exchange', 'expert', 'exposed',
+    # f
+    'fail', 'fans', 'farm', 'fashion', 'feedback', 'fi', 'finance',
+    'financial', 'firmdale', 'fish', 'fishing', 'fit', 'fitness',
+    'fj', 'fk', 'flights', 'florist', 'flowers', 'flsmidth', 'fly',
+    'fm', 'fo', 'foo', 'football', 'forsale', 'foundation', 'fr',
+    'frl', 'frogans', 'fund', 'furniture', 'futbol',
+    # g
+    'ga', 'gal', 'gallery', 'garden', 'gb', 'gbiz', 'gd', 'gdn',
+    'ge', 'gent', 'gf', 'gg', 'ggee', 'gh', 'gi', 'gift', 'gifts',
+    'gives', 'gl', 'glass', 'gle', 'global', 'globo', 'gm', 'gmail',
+    'gmo', 'gmx', 'gn', 'goldpoint', 'goog', 'google', 'gop', 'gov',
+    'gp', 'gq', 'gr', 'graphics', 'gratis', 'green', 'gripe', 'gs',
+    'gt', 'gu', 'guide', 'guitars', 'guru', 'gw', 'gy',
+    # h
+    'hamburg', 'hangout', 'haus', 'healthcare', 'help', 'here',
+    'hermes', 'hiphop', 'hiv', 'hk', 'hm', 'hn', 'holdings',
+    'holiday', 'homes', 'horse', 'host', 'hosting', 'house', 'how',
+    'hr', 'ht', 'hu',
+    # i
+    'ibm', 'id', 'ie', 'ifm', 'il', 'im', 'immo', 'immobilien', 'in',
+    'industries', 'info', 'ing', 'ink', 'institute', 'insure', 'int',
+    'international', 'investments', 'io', 'iq', 'ir', 'irish', 'is',
+    'it', 'iwc',
+    # j
+    'jcb', 'je', 'jetzt', 'jm', 'jo', 'jobs', 'joburg', 'jp',
+    'juegos',
+    # k
+    'kaufen', 'kddi', 'ke', 'kg', 'kh', 'ki', 'kim', 'kitchen',
+    'kiwi', 'km', 'kn', 'koeln', 'kp', 'kr', 'krd', 'kred', 'kw',
+    'ky', 'kyoto', 'kz',
+    # l
+    'la', 'lacaixa', 'land', 'lat', 'latrobe', 'lawyer', 'lb', 'lc',
+    'lds', 'lease', 'legal', 'lgbt', 'li', 'lidl', 'life',
+    'lighting', 'limited', 'limo', 'link', 'lk', 'loans',
+    'localhost', 'london', 'lotte', 'lotto', 'lr', 'ls', 'lt',
+    'ltda', 'lu', 'luxe', 'luxury', 'lv', 'ly',
+    # m
+    'ma', 'madrid', 'maison', 'management', 'mango', 'market',
+    'marketing', 'marriott', 'mc', 'md', 'me', 'media', 'meet',
+    'melbourne', 'meme', 'memorial', 'menu', 'mg', 'mh', 'miami',
+    'mil', 'mini', 'mk', 'ml', 'mm', 'mn', 'mo', 'mobi', 'moda',
+    'moe', 'monash', 'money', 'mormon', 'mortgage', 'moscow',
+    'motorcycles', 'mov', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu',
+    'museum', 'mv', 'mw', 'mx', 'my', 'mz',
+    # n
+    'na', 'nagoya', 'name', 'navy', 'nc', 'ne', 'net', 'network',
+    'neustar', 'new', 'nexus', 'nf', 'ng', 'ngo', 'nhk', 'ni',
+    'nico', 'ninja', 'nl', 'no', 'np', 'nr', 'nra', 'nrw', 'ntt',
+    'nu', 'nyc', 'nz',
+    # o
+    'okinawa', 'om', 'one', 'ong', 'onl', 'ooo', 'org', 'organic',
+    'osaka', 'otsuka', 'ovh',
+    # p
+    'pa', 'paris', 'partners', 'parts', 'party', 'pe', 'pf', 'pg',
+    'ph', 'pharmacy', 'photo', 'photography', 'photos', 'physio',
+    'pics', 'pictures', 'pink', 'pizza', 'pk', 'pl', 'place',
+    'plumbing', 'pm', 'pn', 'pohl', 'poker', 'porn', 'post', 'pr',
+    'praxi', 'press', 'pro', 'prod', 'productions', 'prof',
+    'properties', 'property', 'ps', 'pt', 'pub', 'pw', 'py',
+    # q
+    'qa', 'qpon', 'quebec',
+    # r
+    're', 'realtor', 'recipes', 'red', 'rehab', 'reise', 'reisen',
+    'reit', 'ren', 'rentals', 'repair', 'report', 'republican',
+    'rest', 'restaurant', 'reviews', 'rich', 'rio', 'rip', 'ro',
+    'rocks', 'rodeo', 'rs', 'rsvp', 'ru', 'ruhr', 'rw', 'ryukyu',
+    # s
+    'sa', 'saarland', 'sale', 'samsung', 'sarl', 'saxo', 'sb', 'sc',
+    'sca', 'scb', 'schmidt', 'school', 'schule', 'schwarz',
+    'science', 'scot', 'sd', 'se', 'services', 'sew', 'sexy', 'sg',
+    'sh', 'shiksha', 'shoes', 'shriram', 'si', 'singles', 'sj', 'sk',
+    'sky', 'sl', 'sm', 'sn', 'so', 'social', 'software', 'sohu',
+    'solar', 'solutions', 'soy', 'space', 'spiegel', 'sr', 'st',
+    'style', 'su', 'supplies', 'supply', 'support', 'surf',
+    'surgery', 'suzuki', 'sv', 'sx', 'sy', 'sydney', 'systems', 'sz',
+    # t
+    'taipei', 'tatar', 'tattoo', 'tax', 'tc', 'td', 'technology',
+    'tel', 'temasek', 'tennis', 'tf', 'tg', 'th', 'tienda', 'tips',
+    'tires', 'tirol', 'tj', 'tk', 'tl', 'tm', 'tn', 'to', 'today',
+    'tokyo', 'tools', 'top', 'toshiba', 'town', 'toys', 'tp', 'tr',
+    'trade', 'training', 'travel', 'trust', 'tt', 'tui', 'tv', 'tw',
     'tz',
-    'ua',
-    'ug',
-    'uk',
-    'um',
-    'us',
-    'uy',
-    'uz',
-    'va',
-    'vc',
-    've',
-    'vg',
-    'vi',
-    'vn',
-    'vu',
-    'wf',
-    'ws',
-    'xn--0zwm56d',
-    'xn--11b5bs3a9aj6g',
-    'xn--80akhbyknj4f',
-    'xn--9t4b11yi5a',
-    'xn--deba0ad',
-    'xn--g6w251d',
-    'xn--hgbk6aj7f53bba',
-    'xn--hlcj6aya9esc7a',
-    'xn--jxalpdlp',
-    'xn--kgbechtv',
-    'xn--p1ai',
-    'xn--zckzah',
-    'ye',
-    'yt',
-    'yu',
-    'za',
-    'zm',
-    'zw',
+    # u
+    'ua', 'ug', 'uk', 'university', 'uno', 'uol', 'us', 'uy', 'uz',
+    # v
+    'va', 'vacations', 'vc', 've', 'vegas', 'ventures',
+    'versicherung', 'vet', 'vg', 'vi', 'viajes', 'video', 'villas',
+    'vision', 'vlaanderen', 'vn', 'vodka', 'vote', 'voting', 'voto',
+    'voyage', 'vu',
+    # w
+    'wales', 'wang', 'watch', 'webcam', 'website', 'wed', 'wedding',
+    'wf', 'whoswho', 'wien', 'wiki', 'williamhill', 'wme', 'work',
+    'works', 'world', 'ws', 'wtc', 'wtf',
+    # x
+    'xn--1qqw23a', 'xn--3bst00m', 'xn--3ds443g', 'xn--3e0b707e',
+    'xn--45brj9c', 'xn--45q11c', 'xn--4gbrim', 'xn--55qw42g',
+    'xn--55qx5d', 'xn--6frz82g', 'xn--6qq986b3xl', 'xn--80adxhks',
+    'xn--80ao21a', 'xn--80asehdb', 'xn--80aswg', 'xn--90a3ac',
+    'xn--90ais', 'xn--b4w605ferd', 'xn--c1avg', 'xn--cg4bki',
+    'xn--clchc0ea0b2g2a9gcd', 'xn--czr694b', 'xn--czrs0t',
+    'xn--czru2d', 'xn--d1acj3b', 'xn--d1alf', 'xn--fiq228c5hs',
+    'xn--fiq64b', 'xn--fiqs8s', 'xn--fiqz9s', 'xn--flw351e',
+    'xn--fpcrj9c3d', 'xn--fzc2c9e2c', 'xn--gecrj9c', 'xn--h2brj9c',
+    'xn--hxt814e', 'xn--i1b6b1a6a2e', 'xn--io0a7i', 'xn--j1amh',
+    'xn--j6w193g', 'xn--kprw13d', 'xn--kpry57d', 'xn--kput3i',
+    'xn--l1acc', 'xn--lgbbat1ad8j', 'xn--mgb9awbf',
+    'xn--mgba3a4f16a', 'xn--mgbaam7a8h', 'xn--mgbab2bd',
+    'xn--mgbayh7gpa', 'xn--mgbbh1a71e', 'xn--mgbc0a9azcg',
+    'xn--mgberp4a5d4ar', 'xn--mgbx4cd0ab', 'xn--ngbc5azd',
+    'xn--node', 'xn--nqv7f', 'xn--nqv7fs00ema', 'xn--o3cw4h',
+    'xn--ogbpf8fl', 'xn--p1acf', 'xn--p1ai', 'xn--pgbs0dh',
+    'xn--q9jyb4c', 'xn--qcka1pmc', 'xn--rhqv96g', 'xn--s9brj9c',
+    'xn--ses554g', 'xn--unup4y', 'xn--vermgensberater-ctb',
+    'xn--vermgensberatung-pwb', 'xn--vhquv', 'xn--wgbh1c',
+    'xn--wgbl6a', 'xn--xhq521b', 'xn--xkc2al3hye2a',
+    'xn--xkc2dl3a5ee0h', 'xn--yfro4i67o', 'xn--ygbi2ammx',
+    'xn--zfr164b', 'xxx', 'xyz',
+    # y
+    'yachts', 'yandex', 'ye', 'yodobashi', 'yoga', 'yokohama',
+    'youtube', 'yt',
+    # z
+    'za', 'zip', 'zm', 'zone', 'zuerich', 'zw'
 ]
 
 
@@ -1936,6 +1809,7 @@ class IS_HTTP_URL(Validator):
         error_message='Enter a valid URL',
         allowed_schemes=None,
         prepend_scheme='http',
+        allowed_tlds=None
     ):
 
         self.error_message = error_message
@@ -1943,6 +1817,10 @@ class IS_HTTP_URL(Validator):
             self.allowed_schemes = http_schemes
         else:
             self.allowed_schemes = allowed_schemes
+        if allowed_tlds is None:
+            self.allowed_tlds = official_top_level_domains
+        else:
+            self.allowed_tlds = allowed_tlds
         self.prepend_scheme = prepend_scheme
 
         for i in self.allowed_schemes:
@@ -1986,7 +1864,7 @@ class IS_HTTP_URL(Validator):
                         if domainMatch:
                             # if the top-level domain really exists
                             if domainMatch.group(5).lower()\
-                                    in official_top_level_domains:
+                                    in self.allowed_tlds:
                                 # Then this HTTP URL is valid
                                 return (value, None)
                 else:
@@ -2111,13 +1989,18 @@ class IS_URL(Validator):
         mode='http',
         allowed_schemes=None,
         prepend_scheme='http',
+        allowed_tlds=None
     ):
 
         self.error_message = error_message
         self.mode = mode.lower()
-        if not self.mode in ['generic', 'http']:
+        if self.mode not in ['generic', 'http']:
             raise SyntaxError("invalid mode '%s' in IS_URL" % self.mode)
         self.allowed_schemes = allowed_schemes
+        if allowed_tlds is None:
+            self.allowed_tlds = official_top_level_domains
+        else:
+            self.allowed_tlds = allowed_tlds
 
         if self.allowed_schemes:
             if prepend_scheme not in self.allowed_schemes:
@@ -2150,7 +2033,8 @@ class IS_URL(Validator):
         elif self.mode == 'http':
             subMethod = IS_HTTP_URL(error_message=self.error_message,
                                     allowed_schemes=self.allowed_schemes,
-                                    prepend_scheme=self.prepend_scheme)
+                                    prepend_scheme=self.prepend_scheme,
+                                    allowed_tlds=self.allowed_tlds)
         else:
             raise SyntaxError("invalid mode '%s' in IS_URL" % self.mode)
 

scripts/parse_top_level_domains.py

@@ -0,0 +1,41 @@
+#!/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Use to update official_top_level_domains in gluon/validators.py
+"""
+
+import itertools
+import operator
+import urllib2
+
+LIMIT = 70
+PREFIX = '    '
+TLDS_URL = 'http://data.iana.org/TLD/tlds-alpha-by-domain.txt'
+
+resp = urllib2.urlopen(TLDS_URL)
+content = resp.read()
+
+valid_lines = [a.strip().lower() for a in content.split('\n') if a.strip() and a.strip()[0] != '#']
+valid_lines += ['localhost']
+
+print 'Fetched TLDs are %s' % len(valid_lines)
+
+results = [list(g) for k, g in itertools.groupby(sorted(valid_lines), key=operator.itemgetter(0))]
+
+output = []
+line = "'%s', "
+
+for a in results:
+    output.append('%s# %s' % (PREFIX, a[-1][0]))
+    thisline = PREFIX
+    for c in a:
+        newline = thisline + line % c
+        if len(newline) > 70:
+            output.append(thisline[:-1])
+            thisline = PREFIX + line % c
+        else:
+            thisline += line % c
+    if thisline:
+        output.append(thisline[:-1])
+
+print '[\n' + '\n'.join(output)[:-1] + '\n]'