dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:web2py/web2py

Browse Source
This commit is contained in:
mdipierro
2015-03-01 11:04:53 -06:00
parent 5bed152a0d 9ee8fe25fe
commit befb5b286a
7 changed files with 254 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
applications/admin/controllers/default.py
View File

@@ -1427,14 +1427,13 @@ def create_file():
if request.vars.dir:
response.flash = result
response.headers['web2py-component-content'] = 'append'
response.headers['web2py-component-command'] = """
$.web2py.invalidate('#files_menu');
load_file('%s');
$.web2py.enableElement($('#form form').find($.web2py.formInputClickSelector));
""" % URL('edit', args=[app,request.vars.dir,filename])
response.headers['web2py-component-command'] = "%s %s %s" % (
"$.web2py.invalidate('#files_menu');",
"load_file('%s');" % URL('edit', args=[app,request.vars.dir,filename]),
"$.web2py.enableElement($('#form form').find($.web2py.formInputClickSelector));")
return ''
else:
redirect(request.vars.sender + anchor)
redirect(request.vars.sender + anchor)
def listfiles(app, dir, regexp='.*\.py$'):

6
applications/welcome/static/js/web2py.js
View File

@@ -490,7 +490,7 @@
* and prevent clicking on it */
disableElement: function(el) {
el.addClass('disabled');
var method = el.is('button') ? 'html' : 'val';
var method = el.is('input') ? 'val' : 'html';
//method = el.attr('name') ? 'html' : 'val';
var disable_with_message = (typeof w2p_ajax_disable_with_message != 'undefined') ? w2p_ajax_disable_with_message : "Working...";
/*store enabled state if not already disabled */
@@ -515,7 +515,7 @@
/* restore element to its original state which was disabled by 'disableElement' above*/
enableElement: function(el) {
var method = el.is('button') ? 'html' : 'val';
var method = el.is('input') ? 'val' : 'html';
if(el.data('w2p_enable_with') !== undefined) {
/* set to old enabled state */
el[method](el.data('w2p_enable_with'));
@@ -730,4 +730,4 @@ web2py_event_handlers = jQuery.web2py.event_handlers;
web2py_trap_link = jQuery.web2py.trap_link;
web2py_calc_entropy = jQuery.web2py.calc_entropy;
*/
/* compatibility code - end*/
/* compatibility code - end*/

2
gluon/scheduler.py
View File

@@ -310,7 +310,7 @@ def executor(queue, task, out):
from gluon import current
current.W2P_TASK = W2P_TASK
globals().update(_env)
args = loads(task.args)
args = _decode_list(loads(task.args))
vars = loads(task.vars, object_hook=_decode_dict)
result = dumps(_function(*args, **vars))
else:

28
gluon/tests/test_is_url.py
View File

@@ -498,7 +498,7 @@ class TestIsHttpUrl(unittest.TestCase):
'http://1234567890.com.',
'http://1234567890.com./path',
'http://google.com./path',
'http://domain.xn--0zwm56d',
'http://domain.xn--d1acj3b',
'http://127.123.0.256',
'http://127.123.0.256/document/drawer',
'127.123.0.256/document/',
@@ -669,5 +669,31 @@ class TestUnicode(unittest.TestCase):
# ##############################################################################
class TestSimple(unittest.TestCase):
def test_IS_URL(self):
rtn = IS_URL()('abc.com')
self.assertEqual(rtn, ('http://abc.com', None))
rtn = IS_URL(mode='generic')('abc.com')
self.assertEqual(rtn, ('abc.com', None))
rtn = IS_URL(allowed_schemes=['https'], prepend_scheme='https')('https://abc.com')
self.assertEqual(rtn, ('https://abc.com', None))
rtn = IS_URL(prepend_scheme='https')('abc.com')
self.assertEqual(rtn, ('https://abc.com', None))
rtn = IS_URL(mode='generic', allowed_schemes=['ftps', 'https'], prepend_scheme='https')('https://abc.com')
self.assertEqual(rtn, ('https://abc.com', None))
rtn = IS_URL(mode='generic', allowed_schemes=['ftps', 'https', None], prepend_scheme='https')('abc.com')
self.assertEqual(rtn, ('abc.com', None))
# regression test for issue 773
rtn = IS_URL()('domain.ninja')
self.assertEqual(rtn, ('http://domain.ninja', None))
# addition of allowed_tlds
rtn = IS_URL(allowed_tlds=['com', 'net', 'org'])('domain.ninja')
self.assertEqual(rtn, ('domain.ninja', 'Enter a valid URL'))
# mode = 'generic' doesn't consider allowed_tlds
rtn = IS_URL(mode='generic', allowed_tlds=['com', 'net', 'org'])('domain.ninja')
self.assertEqual(rtn, ('domain.ninja', None))
if __name__ == '__main__':
unittest.main()

15
gluon/tests/test_validators.py
View File

@@ -583,8 +583,6 @@ class TestValidators(unittest.TestCase):
rtn = IS_LENGTH(6)(a)
self.assertEqual(rtn, (a, None))
def test_IS_LOWER(self):
rtn = IS_LOWER()('ABC')
self.assertEqual(rtn, ('abc', None))
@@ -769,19 +767,6 @@ class TestValidators(unittest.TestCase):
rtn = IS_UPPER()('ñ')
self.assertEqual(rtn, ('\xc3\x91', None))
def test_IS_URL(self):
rtn = IS_URL()('abc.com')
self.assertEqual(rtn, ('http://abc.com', None))
rtn = IS_URL(mode='generic')('abc.com')
self.assertEqual(rtn, ('abc.com', None))
rtn = IS_URL(allowed_schemes=['https'], prepend_scheme='https')('https://abc.com')
self.assertEqual(rtn, ('https://abc.com', None))
rtn = IS_URL(prepend_scheme='https')('abc.com')
self.assertEqual(rtn, ('https://abc.com', None))
rtn = IS_URL(mode='generic', allowed_schemes=['ftps', 'https'], prepend_scheme='https')('https://abc.com')
self.assertEqual(rtn, ('https://abc.com', None))
rtn = IS_URL(mode='generic', allowed_schemes=['ftps', 'https', None], prepend_scheme='https')('abc.com')
self.assertEqual(rtn, ('abc.com', None))
def test_IS_JSON(self):
rtn = IS_JSON()('{"a": 100}')

470
gluon/validators.py
View File

@@ -1579,300 +1579,173 @@ class IS_GENERIC_URL(Validator):
# else the URL is not valid
return (value, translate(self.error_message))
# Sources (obtained 2008-Nov-11):
# http://en.wikipedia.org/wiki/Top-level_domain
# http://www.iana.org/domains/root/db/
# Sources (obtained 2015-Feb-24):
# http://data.iana.org/TLD/tlds-alpha-by-domain.txt
# see scripts/parse_top_level_domains.py for an easy update
official_top_level_domains = [
'ac',
'ad',
'ae',
'aero',
'af',
'ag',
'ai',
'al',
'am',
'an',
'ao',
'aq',
'ar',
'arpa',
'as',
'asia',
'at',
'au',
'aw',
'ax',
'az',
'ba',
'bb',
'bd',
'be',
'bf',
'bg',
'bh',
'bi',
'biz',
'bj',
'bl',
'bm',
'bn',
'bo',
'br',
'bs',
'bt',
'bv',
'bw',
'by',
'bz',
'ca',
'cat',
'cc',
'cd',
'cf',
'cg',
'ch',
'ci',
'ck',
'cl',
'cm',
'cn',
'co',
'com',
'coop',
'cr',
'cu',
'cv',
'cx',
'cy',
# a
'abogado', 'ac', 'academy', 'accountants', 'active', 'actor',
'ad', 'adult', 'ae', 'aero', 'af', 'ag', 'agency', 'ai',
'airforce', 'al', 'allfinanz', 'alsace', 'am', 'amsterdam', 'an',
'android', 'ao', 'apartments', 'aq', 'aquarelle', 'ar', 'archi',
'army', 'arpa', 'as', 'asia', 'associates', 'at', 'attorney',
'au', 'auction', 'audio', 'autos', 'aw', 'ax', 'axa', 'az',
# b
'ba', 'band', 'bank', 'bar', 'barclaycard', 'barclays',
'bargains', 'bayern', 'bb', 'bd', 'be', 'beer', 'berlin', 'best',
'bf', 'bg', 'bh', 'bi', 'bid', 'bike', 'bingo', 'bio', 'biz',
'bj', 'black', 'blackfriday', 'bloomberg', 'blue', 'bm', 'bmw',
'bn', 'bnpparibas', 'bo', 'boo', 'boutique', 'br', 'brussels',
'bs', 'bt', 'budapest', 'build', 'builders', 'business', 'buzz',
'bv', 'bw', 'by', 'bz', 'bzh',
# c
'ca', 'cab', 'cal', 'camera', 'camp', 'cancerresearch', 'canon',
'capetown', 'capital', 'caravan', 'cards', 'care', 'career',
'careers', 'cartier', 'casa', 'cash', 'casino', 'cat',
'catering', 'cbn', 'cc', 'cd', 'center', 'ceo', 'cern', 'cf',
'cg', 'ch', 'channel', 'chat', 'cheap', 'christmas', 'chrome',
'church', 'ci', 'citic', 'city', 'ck', 'cl', 'claims',
'cleaning', 'click', 'clinic', 'clothing', 'club', 'cm', 'cn',
'co', 'coach', 'codes', 'coffee', 'college', 'cologne', 'com',
'community', 'company', 'computer', 'condos', 'construction',
'consulting', 'contractors', 'cooking', 'cool', 'coop',
'country', 'cr', 'credit', 'creditcard', 'cricket', 'crs',
'cruises', 'cu', 'cuisinella', 'cv', 'cw', 'cx', 'cy', 'cymru',
'cz',
'de',
'dj',
'dk',
'dm',
'do',
'dz',
'ec',
'edu',
'ee',
'eg',
'eh',
'er',
'es',
'et',
'eu',
'example',
'fi',
'fj',
'fk',
'fm',
'fo',
'fr',
'ga',
'gb',
'gd',
'ge',
'gf',
'gg',
'gh',
'gi',
'gl',
'gm',
'gn',
'gov',
'gp',
'gq',
'gr',
'gs',
'gt',
'gu',
'gw',
'gy',
'hk',
'hm',
'hn',
'hr',
'ht',
'hu',
'id',
'ie',
'il',
'im',
'in',
'info',
'int',
'invalid',
'io',
'iq',
'ir',
'is',
'it',
'je',
'jm',
'jo',
'jobs',
'jp',
'ke',
'kg',
'kh',
'ki',
'km',
'kn',
'kp',
'kr',
'kw',
'ky',
'kz',
'la',
'lb',
'lc',
'li',
'lk',
'localhost',
'lr',
'ls',
'lt',
'lu',
'lv',
'ly',
'ma',
'mc',
'md',
'me',
'mf',
'mg',
'mh',
'mil',
'mk',
'ml',
'mm',
'mn',
'mo',
'mobi',
'mp',
'mq',
'mr',
'ms',
'mt',
'mu',
'museum',
'mv',
'mw',
'mx',
'my',
'mz',
'na',
'name',
'nc',
'ne',
'net',
'nf',
'ng',
'ni',
'nl',
'no',
'np',
'nr',
'nu',
'nz',
'om',
'org',
'pa',
'pe',
'pf',
'pg',
'ph',
'pk',
'pl',
'pm',
'pn',
'pr',
'pro',
'ps',
'pt',
'pw',
'py',
'qa',
're',
'ro',
'rs',
'ru',
'rw',
'sa',
'sb',
'sc',
'sd',
'se',
'sg',
'sh',
'si',
'sj',
'sk',
'sl',
'sm',
'sn',
'so',
'sr',
'st',
'su',
'sv',
'sy',
'sz',
'tc',
'td',
'tel',
'test',
'tf',
'tg',
'th',
'tj',
'tk',
'tl',
'tm',
'tn',
'to',
'tp',
'tr',
'travel',
'tt',
'tv',
'tw',
# d
'dabur', 'dad', 'dance', 'dating', 'day', 'dclk', 'de', 'deals',
'degree', 'delivery', 'democrat', 'dental', 'dentist', 'desi',
'design', 'dev', 'diamonds', 'diet', 'digital', 'direct',
'directory', 'discount', 'dj', 'dk', 'dm', 'dnp', 'do', 'docs',
'domains', 'doosan', 'durban', 'dvag', 'dz',
# e
'eat', 'ec', 'edu', 'education', 'ee', 'eg', 'email', 'emerck',
'energy', 'engineer', 'engineering', 'enterprises', 'equipment',
'er', 'es', 'esq', 'estate', 'et', 'eu', 'eurovision', 'eus',
'events', 'everbank', 'exchange', 'expert', 'exposed',
# f
'fail', 'fans', 'farm', 'fashion', 'feedback', 'fi', 'finance',
'financial', 'firmdale', 'fish', 'fishing', 'fit', 'fitness',
'fj', 'fk', 'flights', 'florist', 'flowers', 'flsmidth', 'fly',
'fm', 'fo', 'foo', 'football', 'forsale', 'foundation', 'fr',
'frl', 'frogans', 'fund', 'furniture', 'futbol',
# g
'ga', 'gal', 'gallery', 'garden', 'gb', 'gbiz', 'gd', 'gdn',
'ge', 'gent', 'gf', 'gg', 'ggee', 'gh', 'gi', 'gift', 'gifts',
'gives', 'gl', 'glass', 'gle', 'global', 'globo', 'gm', 'gmail',
'gmo', 'gmx', 'gn', 'goldpoint', 'goog', 'google', 'gop', 'gov',
'gp', 'gq', 'gr', 'graphics', 'gratis', 'green', 'gripe', 'gs',
'gt', 'gu', 'guide', 'guitars', 'guru', 'gw', 'gy',
# h
'hamburg', 'hangout', 'haus', 'healthcare', 'help', 'here',
'hermes', 'hiphop', 'hiv', 'hk', 'hm', 'hn', 'holdings',
'holiday', 'homes', 'horse', 'host', 'hosting', 'house', 'how',
'hr', 'ht', 'hu',
# i
'ibm', 'id', 'ie', 'ifm', 'il', 'im', 'immo', 'immobilien', 'in',
'industries', 'info', 'ing', 'ink', 'institute', 'insure', 'int',
'international', 'investments', 'io', 'iq', 'ir', 'irish', 'is',
'it', 'iwc',
# j
'jcb', 'je', 'jetzt', 'jm', 'jo', 'jobs', 'joburg', 'jp',
'juegos',
# k
'kaufen', 'kddi', 'ke', 'kg', 'kh', 'ki', 'kim', 'kitchen',
'kiwi', 'km', 'kn', 'koeln', 'kp', 'kr', 'krd', 'kred', 'kw',
'ky', 'kyoto', 'kz',
# l
'la', 'lacaixa', 'land', 'lat', 'latrobe', 'lawyer', 'lb', 'lc',
'lds', 'lease', 'legal', 'lgbt', 'li', 'lidl', 'life',
'lighting', 'limited', 'limo', 'link', 'lk', 'loans',
'localhost', 'london', 'lotte', 'lotto', 'lr', 'ls', 'lt',
'ltda', 'lu', 'luxe', 'luxury', 'lv', 'ly',
# m
'ma', 'madrid', 'maison', 'management', 'mango', 'market',
'marketing', 'marriott', 'mc', 'md', 'me', 'media', 'meet',
'melbourne', 'meme', 'memorial', 'menu', 'mg', 'mh', 'miami',
'mil', 'mini', 'mk', 'ml', 'mm', 'mn', 'mo', 'mobi', 'moda',
'moe', 'monash', 'money', 'mormon', 'mortgage', 'moscow',
'motorcycles', 'mov', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu',
'museum', 'mv', 'mw', 'mx', 'my', 'mz',
# n
'na', 'nagoya', 'name', 'navy', 'nc', 'ne', 'net', 'network',
'neustar', 'new', 'nexus', 'nf', 'ng', 'ngo', 'nhk', 'ni',
'nico', 'ninja', 'nl', 'no', 'np', 'nr', 'nra', 'nrw', 'ntt',
'nu', 'nyc', 'nz',
# o
'okinawa', 'om', 'one', 'ong', 'onl', 'ooo', 'org', 'organic',
'osaka', 'otsuka', 'ovh',
# p
'pa', 'paris', 'partners', 'parts', 'party', 'pe', 'pf', 'pg',
'ph', 'pharmacy', 'photo', 'photography', 'photos', 'physio',
'pics', 'pictures', 'pink', 'pizza', 'pk', 'pl', 'place',
'plumbing', 'pm', 'pn', 'pohl', 'poker', 'porn', 'post', 'pr',
'praxi', 'press', 'pro', 'prod', 'productions', 'prof',
'properties', 'property', 'ps', 'pt', 'pub', 'pw', 'py',
# q
'qa', 'qpon', 'quebec',
# r
're', 'realtor', 'recipes', 'red', 'rehab', 'reise', 'reisen',
'reit', 'ren', 'rentals', 'repair', 'report', 'republican',
'rest', 'restaurant', 'reviews', 'rich', 'rio', 'rip', 'ro',
'rocks', 'rodeo', 'rs', 'rsvp', 'ru', 'ruhr', 'rw', 'ryukyu',
# s
'sa', 'saarland', 'sale', 'samsung', 'sarl', 'saxo', 'sb', 'sc',
'sca', 'scb', 'schmidt', 'school', 'schule', 'schwarz',
'science', 'scot', 'sd', 'se', 'services', 'sew', 'sexy', 'sg',
'sh', 'shiksha', 'shoes', 'shriram', 'si', 'singles', 'sj', 'sk',
'sky', 'sl', 'sm', 'sn', 'so', 'social', 'software', 'sohu',
'solar', 'solutions', 'soy', 'space', 'spiegel', 'sr', 'st',
'style', 'su', 'supplies', 'supply', 'support', 'surf',
'surgery', 'suzuki', 'sv', 'sx', 'sy', 'sydney', 'systems', 'sz',
# t
'taipei', 'tatar', 'tattoo', 'tax', 'tc', 'td', 'technology',
'tel', 'temasek', 'tennis', 'tf', 'tg', 'th', 'tienda', 'tips',
'tires', 'tirol', 'tj', 'tk', 'tl', 'tm', 'tn', 'to', 'today',
'tokyo', 'tools', 'top', 'toshiba', 'town', 'toys', 'tp', 'tr',
'trade', 'training', 'travel', 'trust', 'tt', 'tui', 'tv', 'tw',
'tz',
'ua',
'ug',
'uk',
'um',
'us',
'uy',
'uz',
'va',
'vc',
've',
'vg',
'vi',
'vn',
'vu',
'wf',
'ws',
'xn--0zwm56d',
'xn--11b5bs3a9aj6g',
'xn--80akhbyknj4f',
'xn--9t4b11yi5a',
'xn--deba0ad',
'xn--g6w251d',
'xn--hgbk6aj7f53bba',
'xn--hlcj6aya9esc7a',
'xn--jxalpdlp',
'xn--kgbechtv',
'xn--p1ai',
'xn--zckzah',
'ye',
'yt',
'yu',
'za',
'zm',
'zw',
# u
'ua', 'ug', 'uk', 'university', 'uno', 'uol', 'us', 'uy', 'uz',
# v
'va', 'vacations', 'vc', 've', 'vegas', 'ventures',
'versicherung', 'vet', 'vg', 'vi', 'viajes', 'video', 'villas',
'vision', 'vlaanderen', 'vn', 'vodka', 'vote', 'voting', 'voto',
'voyage', 'vu',
# w
'wales', 'wang', 'watch', 'webcam', 'website', 'wed', 'wedding',
'wf', 'whoswho', 'wien', 'wiki', 'williamhill', 'wme', 'work',
'works', 'world', 'ws', 'wtc', 'wtf',
# x
'xn--1qqw23a', 'xn--3bst00m', 'xn--3ds443g', 'xn--3e0b707e',
'xn--45brj9c', 'xn--45q11c', 'xn--4gbrim', 'xn--55qw42g',
'xn--55qx5d', 'xn--6frz82g', 'xn--6qq986b3xl', 'xn--80adxhks',
'xn--80ao21a', 'xn--80asehdb', 'xn--80aswg', 'xn--90a3ac',
'xn--90ais', 'xn--b4w605ferd', 'xn--c1avg', 'xn--cg4bki',
'xn--clchc0ea0b2g2a9gcd', 'xn--czr694b', 'xn--czrs0t',
'xn--czru2d', 'xn--d1acj3b', 'xn--d1alf', 'xn--fiq228c5hs',
'xn--fiq64b', 'xn--fiqs8s', 'xn--fiqz9s', 'xn--flw351e',
'xn--fpcrj9c3d', 'xn--fzc2c9e2c', 'xn--gecrj9c', 'xn--h2brj9c',
'xn--hxt814e', 'xn--i1b6b1a6a2e', 'xn--io0a7i', 'xn--j1amh',
'xn--j6w193g', 'xn--kprw13d', 'xn--kpry57d', 'xn--kput3i',
'xn--l1acc', 'xn--lgbbat1ad8j', 'xn--mgb9awbf',
'xn--mgba3a4f16a', 'xn--mgbaam7a8h', 'xn--mgbab2bd',
'xn--mgbayh7gpa', 'xn--mgbbh1a71e', 'xn--mgbc0a9azcg',
'xn--mgberp4a5d4ar', 'xn--mgbx4cd0ab', 'xn--ngbc5azd',
'xn--node', 'xn--nqv7f', 'xn--nqv7fs00ema', 'xn--o3cw4h',
'xn--ogbpf8fl', 'xn--p1acf', 'xn--p1ai', 'xn--pgbs0dh',
'xn--q9jyb4c', 'xn--qcka1pmc', 'xn--rhqv96g', 'xn--s9brj9c',
'xn--ses554g', 'xn--unup4y', 'xn--vermgensberater-ctb',
'xn--vermgensberatung-pwb', 'xn--vhquv', 'xn--wgbh1c',
'xn--wgbl6a', 'xn--xhq521b', 'xn--xkc2al3hye2a',
'xn--xkc2dl3a5ee0h', 'xn--yfro4i67o', 'xn--ygbi2ammx',
'xn--zfr164b', 'xxx', 'xyz',
# y
'yachts', 'yandex', 'ye', 'yodobashi', 'yoga', 'yokohama',
'youtube', 'yt',
# z
'za', 'zip', 'zm', 'zone', 'zuerich', 'zw'
]
@@ -1936,6 +1809,7 @@ class IS_HTTP_URL(Validator):
error_message='Enter a valid URL',
allowed_schemes=None,
prepend_scheme='http',
allowed_tlds=None
):
self.error_message = error_message
@@ -1943,6 +1817,10 @@ class IS_HTTP_URL(Validator):
self.allowed_schemes = http_schemes
else:
self.allowed_schemes = allowed_schemes
if allowed_tlds is None:
self.allowed_tlds = official_top_level_domains
else:
self.allowed_tlds = allowed_tlds
self.prepend_scheme = prepend_scheme
for i in self.allowed_schemes:
@@ -1986,7 +1864,7 @@ class IS_HTTP_URL(Validator):
if domainMatch:
# if the top-level domain really exists
if domainMatch.group(5).lower()\
in official_top_level_domains:
in self.allowed_tlds:
# Then this HTTP URL is valid
return (value, None)
else:
@@ -2111,13 +1989,18 @@ class IS_URL(Validator):
mode='http',
allowed_schemes=None,
prepend_scheme='http',
allowed_tlds=None
):
self.error_message = error_message
self.mode = mode.lower()
if not self.mode in ['generic', 'http']:
if self.mode not in ['generic', 'http']:
raise SyntaxError("invalid mode '%s' in IS_URL" % self.mode)
self.allowed_schemes = allowed_schemes
if allowed_tlds is None:
self.allowed_tlds = official_top_level_domains
else:
self.allowed_tlds = allowed_tlds
if self.allowed_schemes:
if prepend_scheme not in self.allowed_schemes:
@@ -2150,7 +2033,8 @@ class IS_URL(Validator):
elif self.mode == 'http':
subMethod = IS_HTTP_URL(error_message=self.error_message,
allowed_schemes=self.allowed_schemes,
prepend_scheme=self.prepend_scheme)
prepend_scheme=self.prepend_scheme,
allowed_tlds=self.allowed_tlds)
else:
raise SyntaxError("invalid mode '%s' in IS_URL" % self.mode)

41
scripts/parse_top_level_domains.py Normal file
View File

@@ -0,0 +1,41 @@
#!/bin/env python
# -*- coding: utf-8 -*-
"""
Use to update official_top_level_domains in gluon/validators.py
"""
import itertools
import operator
import urllib2
LIMIT = 70
PREFIX = ' '
TLDS_URL = 'http://data.iana.org/TLD/tlds-alpha-by-domain.txt'
resp = urllib2.urlopen(TLDS_URL)
content = resp.read()
valid_lines = [a.strip().lower() for a in content.split('\n') if a.strip() and a.strip()[0] != '#']
valid_lines += ['localhost']
print 'Fetched TLDs are %s' % len(valid_lines)
results = [list(g) for k, g in itertools.groupby(sorted(valid_lines), key=operator.itemgetter(0))]
output = []
line = "'%s', "
for a in results:
output.append('%s# %s' % (PREFIX, a[-1][0]))
thisline = PREFIX
for c in a:
newline = thisline + line % c
if len(newline) > 70:
output.append(thisline[:-1])
thisline = PREFIX + line % c
else:
thisline += line % c
if thisline:
output.append(thisline[:-1])
print '[\n' + '\n'.join(output)[:-1] + '\n]'