From: Steven Black Date: Sat, 21 Nov 2015 03:10:49 +0000 (-0500) Subject: Refactor: whitespace for consistency and readability. X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=d372865698563a7352e6ea5c618391aea832ca65;p=stevenblack-hosts.git Refactor: whitespace for consistency and readability. --- diff --git a/updateHostsFile.py b/updateHostsFile.py index 3d29f63ab..07cd70472 100755 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -3,7 +3,7 @@ # Script by Ben Limmer # https://github.com/l1m5 # -# This simple Python script will combine all the host files you provide +# This Python script will combine all the host files you provide # as sources into one, unique host file to keep you internet browsing happy. # Making Python 2 compatible with Python 3 @@ -35,108 +35,106 @@ except ImportError: def getFileByUrl(url): try: f = urlopen(url) - return f.read().decode("UTF-8") + return f.read().decode( "UTF-8" ) except: - print ("Problem getting file: ", url); + print ( "Problem getting file: ", url ); # raise - # In Python 3 "print" is a function, braces are added everywhere # Detecting Python 3 for version-dependent implementations -Python3=False; +Python3 = False; cur_version = sys.version_info -if cur_version >= (3, 0): - Python3=True; +if cur_version >= ( 3, 0 ): + Python3 = True; # This function works in both Python 2 and Python 3 -def myInput(msg=""): +def myInput( msg = "" ): if Python3: - return input(msg); + return input( msg ); else: - return raw_input(msg); + return raw_input( msg ); # Cross-python writing function -def writeData(f, data): +def writeData( f, data ): if Python3: - f.write(bytes(data, 'UTF-8')) + f.write( bytes( data, 'UTF-8' )) else: - f.write(str(data).encode('UTF-8')) + f.write( str( data ).encode( 'UTF-8' )) # This function doesn't list hidden files -def listdir_nohidden(path): - return glob.glob(os.path.join(path, '*')) +def listdir_nohidden( path ): + return glob.glob( os.path.join( path, '*' )) # Project Settings -BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__)) -DATA_PATH = os.path.join(BASEDIR_PATH, 'data') -DATA_FILENAMES = 'hosts' +BASEDIR_PATH = os.path.dirname( os.path.realpath( __file__ )) +DATA_PATH = os.path.join( BASEDIR_PATH, 'data' ) +DATA_FILENAMES = 'hosts' UPDATE_URL_FILENAME = 'update.info' -SOURCES = listdir_nohidden(DATA_PATH) -README_TEMPLATE = os.path.join(BASEDIR_PATH, 'readme_template.md') -README_FILE = os.path.join(BASEDIR_PATH, 'readme.md') -TARGET_HOST = '0.0.0.0' -WHITELIST_FILE = os.path.join(BASEDIR_PATH, 'whitelist') +SOURCES = listdir_nohidden( DATA_PATH ) +README_TEMPLATE = os.path.join( BASEDIR_PATH, 'readme_template.md' ) +README_FILE = os.path.join( BASEDIR_PATH, 'readme.md' ) +TARGET_HOST = '0.0.0.0' +WHITELIST_FILE = os.path.join( BASEDIR_PATH, 'whitelist' ) # Exclusions EXCLUSION_PATTERN = '([a-zA-Z\d-]+\.){0,}' #append domain the end -EXCLUSIONS = [] +EXCLUSIONS = [] # Common domains to exclude COMMON_EXCLUSIONS = ['hulu.com'] # Global vars exclusionRegexs = [] -numberOfRules = 0 +numberOfRules = 0 def main(): promptForUpdate() promptForExclusions() mergeFile = createInitialFile() removeOldHostsFile() - finalFile = removeDupsAndExcl(mergeFile) - finalizeFile(finalFile) - updateReadme(numberOfRules) - printSuccess('Success! Your shiny new hosts file has been prepared.\nIt contains ' + "{:,}".format( numberOfRules ) + ' unique entries.') + finalFile = removeDupsAndExcl( mergeFile ) + finalizeFile( finalFile ) + updateReadme( numberOfRules ) + printSuccess( 'Success! Your new hosts file has been prepared.\nIt contains ' + "{:,}".format( numberOfRules ) + ' unique entries.' ) - promptForMove(finalFile) + promptForMove( finalFile ) # Prompt the User def promptForUpdate(): - # Create hosts file if it doesn't exists - if not os.path.isfile(os.path.join(BASEDIR_PATH, 'hosts')): + if not os.path.isfile( os.path.join(BASEDIR_PATH, 'hosts' )): try: - file = open(os.path.join(BASEDIR_PATH, 'hosts'), 'w+').close() + file = open( os.path.join( BASEDIR_PATH, 'hosts' ), 'w+' ).close() except: - printFailure("ERROR: No 'hosts' file in the folder, try creating one manually") - - response = query_yes_no("Do you want to update all data sources?") - if (response == "yes"): + printFailure( "ERROR: No 'hosts' file in the folder, try creating one manually" ) + + response = query_yes_no( "Do you want to update all data sources?" ) + if ( response == "yes" ): updateAllSources() else: - print ('OK, we\'ll stick with what we\'ve got locally.') + print ( 'OK, we\'ll stick with what we\'ve got locally.' ) def promptForExclusions(): - response = query_yes_no("Do you want to exclude any domains?\n" + + response = query_yes_no( "Do you want to exclude any domains?\n" + "For example, hulu.com video streaming must be able to access " + - "its tracking and ad servers in order to play video.") - if (response == "yes"): + "its tracking and ad servers in order to play video." ) + if ( response == "yes" ): displayExclusionOptions() else: - print ('OK, we\'ll only exclude domains in the whitelist.') + print ( 'OK, we\'ll only exclude domains in the whitelist.' ) def promptForMoreCustomExclusions(): - response = query_yes_no("Do you have more domains you want to enter?") - if (response == "yes"): + response = query_yes_no( "Do you have more domains you want to enter?" ) + if ( response == "yes" ): return True else: return False -def promptForMove(finalFile): - response = query_yes_no("Do you want to replace your existing hosts file with the newly generated file?") - if (response == "yes"): - moveHostsFileIntoPlace(finalFile) +def promptForMove( finalFile ): + response = query_yes_no( "Do you want to replace your existing hosts file with the newly generated file?" ) + if ( response == "yes" ): + moveHostsFileIntoPlace( finalFile ) else: return False # End Prompt the User @@ -144,31 +142,31 @@ def promptForMove(finalFile): # Exclusion logic def displayExclusionOptions(): for exclusionOption in COMMON_EXCLUSIONS: - response = query_yes_no("Do you want to exclude the domain " + exclusionOption + " ?") - if (response == "yes"): + response = query_yes_no( "Do you want to exclude the domain " + exclusionOption + " ?" ) + if ( response == "yes" ): excludeDomain(exclusionOption) else: continue - response = query_yes_no("Do you want to exclude any other domains?") - if (response == "yes"): + response = query_yes_no( "Do you want to exclude any other domains?" ) + if ( response == "yes" ): gatherCustomExclusions() def gatherCustomExclusions(): while True: # Cross-python Input - domainFromUser = myInput("Enter the domain you want to exclude (e.g. facebook.com): ") - if (isValidDomainFormat(domainFromUser)): - excludeDomain(domainFromUser) - if (promptForMoreCustomExclusions() == False): + domainFromUser = myInput( "Enter the domain you want to exclude (e.g. facebook.com): " ) + if (isValidDomainFormat( domainFromUser )): + excludeDomain( domainFromUser ) + if ( promptForMoreCustomExclusions() == False ): return -def excludeDomain(domain): - exclusionRegexs.append(re.compile(EXCLUSION_PATTERN + domain)) +def excludeDomain( domain ): + exclusionRegexs.append( re.compile( EXCLUSION_PATTERN + domain )) -def matchesExclusions(strippedRule): +def matchesExclusions( strippedRule ): strippedDomain = strippedRule.split()[1] for exclusionRegex in exclusionRegexs: - if exclusionRegex.search(strippedDomain): + if exclusionRegex.search( strippedDomain ): return True return False # End Exclusion Logic @@ -176,29 +174,29 @@ def matchesExclusions(strippedRule): # Update Logic def updateAllSources(): for source in SOURCES: - updateURL = getUpdateURLFromFile(source) - if (updateURL == None): + updateURL = getUpdateURLFromFile( source ) + if ( updateURL == None ): continue; - print ('Updating source ' + source + ' from ' + updateURL) + print ( 'Updating source ' + source + ' from ' + updateURL ) # Cross-python call - updatedFile = getFileByUrl(updateURL); - updatedFile = updatedFile.replace('\r', '') #get rid of carriage-return symbols + updatedFile = getFileByUrl( updateURL ); + updatedFile = updatedFile.replace( '\r', '' ) #get rid of carriage-return symbols # This is cross-python code - dataFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'wb') - writeData(dataFile, updatedFile); + dataFile = open( os.path.join( DATA_PATH, source, DATA_FILENAMES ), 'wb' ) + writeData( dataFile, updatedFile ); dataFile.close() -def getUpdateURLFromFile(source): - pathToUpdateFile = os.path.join(DATA_PATH, source, UPDATE_URL_FILENAME) - if os.path.exists(pathToUpdateFile): - updateFile = open(pathToUpdateFile, 'r') - retURL = updateFile.readline().strip() +def getUpdateURLFromFile( source ): + pathToUpdateFile = os.path.join( DATA_PATH, source, UPDATE_URL_FILENAME ) + if os.path.exists( pathToUpdateFile ): + updateFile = open( pathToUpdateFile, 'r' ) + retURL = updateFile.readline().strip() updateFile.close() else: retURL = None - printFailure('Warning: Can\'t find the update file for source ' + source + '\n' + - 'Make sure that there\'s a file at ' + pathToUpdateFile) + printFailure( 'Warning: Can\'t find the update file for source ' + source + '\n' + + 'Make sure that there\'s a file at ' + pathToUpdateFile ) return retURL # End Update Logic @@ -206,48 +204,48 @@ def getUpdateURLFromFile(source): def createInitialFile(): mergeFile = tempfile.NamedTemporaryFile() for source in SOURCES: - curFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'r') + curFile = open( os.path.join( DATA_PATH, source, DATA_FILENAMES ), 'r' ) #Done in a cross-python way - writeData(mergeFile, curFile.read()) + writeData( mergeFile, curFile.read() ) return mergeFile -def removeDupsAndExcl(mergeFile): +def removeDupsAndExcl( mergeFile ): global numberOfRules - if os.path.isfile(WHITELIST_FILE): - with open(WHITELIST_FILE, "r") as ins: + if os.path.isfile( WHITELIST_FILE ): + with open( WHITELIST_FILE, "r" ) as ins: for line in ins: - EXCLUSIONS.append(line) + EXCLUSIONS.append( line ) # Another mode is required to read and write the file in Python 3 - finalFile = open(os.path.join(BASEDIR_PATH, 'hosts'), 'r+b') - mergeFile.seek(0) # reset file pointer + finalFile = open( os.path.join( BASEDIR_PATH, 'hosts' ), 'r+b' ) + mergeFile.seek( 0 ) # reset file pointer hostnames = set() - hostnames.add("localhost") + hostnames.add( "localhost" ) for line in mergeFile.readlines(): write = 'true' # Explicit encoding - line = line.decode("UTF-8") + line = line.decode( "UTF-8" ) # Testing the first character doesn't require startswith if line[0] == '#' or re.match(r'^\s*$', line[0]): # Cross-python write - writeData(finalFile, line) + writeData( finalFile, line ) continue - strippedRule = stripRule(line) #strip comments - if len(strippedRule) == 0: + strippedRule = stripRule( line ) #strip comments + if len( strippedRule ) == 0: continue - if matchesExclusions(strippedRule): + if matchesExclusions( strippedRule ): continue - hostname, normalizedRule = normalizeRule(strippedRule) # normalize rule + hostname, normalizedRule = normalizeRule( strippedRule ) # normalize rule for exclude in EXCLUSIONS: - if (exclude in line): + if ( exclude in line ): write = 'false' break - if normalizedRule and (hostname not in hostnames) and (write == 'true'): - writeData(finalFile, normalizedRule) - hostnames.add(hostname) + if normalizedRule and ( hostname not in hostnames ) and ( write == 'true' ): + writeData( finalFile, normalizedRule ) + hostnames.add( hostname ) numberOfRules += 1 mergeFile.close() @@ -255,27 +253,27 @@ def removeDupsAndExcl(mergeFile): return finalFile def normalizeRule(rule): - result = re.search(r'^[ \t]*(\d+\.\d+\.\d+\.\d+)\s+([\w\.-]+)(.*)',rule) + result = re.search(r'^[ \t]*(\d+\.\d+\.\d+\.\d+)\s+([\w\.-]+)(.*)', rule ) if result: target, hostname, suffix = result.groups() hostname = hostname.lower() # explicitly lowercase hostname if suffix is not '': # add suffix as comment only, not as a separate host - return hostname, "%s %s #%s\n" % (TARGET_HOST, hostname, suffix) + return hostname, "%s %s #%s\n" % ( TARGET_HOST, hostname, suffix ) else: - return hostname, "%s %s\n" % (TARGET_HOST, hostname) - print ('==>%s<==' % rule) + return hostname, "%s %s\n" % ( TARGET_HOST, hostname ) + print ( '==>%s<==' % rule ) return None, None -def finalizeFile(finalFile): - writeOpeningHeader(finalFile) +def finalizeFile( finalFile ): + writeOpeningHeader( finalFile ) finalFile.close() # Some sources put comments around their rules, for accuracy we need to strip them # the comments are preserved in the output hosts file -def stripRule(line): +def stripRule( line ): splitLine = line.split() - if (len(splitLine) < 2) : + if ( len( splitLine ) < 2 ) : # just return blank return '' else: @@ -283,57 +281,57 @@ def stripRule(line): def writeOpeningHeader(finalFile): global numberOfRules - finalFile.seek(0) #reset file pointer + finalFile.seek( 0 ) #reset file pointer fileContents = finalFile.read(); #save content - finalFile.seek(0) #write at the top - writeData(finalFile, '# This file is a merged collection of hosts from reputable sources,\n') - writeData(finalFile, '# with a dash of crowd sourcing via Github\n#\n') - writeData(finalFile, '# Project home page: https://github.com/StevenBlack/hosts\n#\n') - writeData(finalFile, '# ===============================================================\n') - writeData(finalFile, '\n') - writeData(finalFile, '127.0.0.1 localhost\n') - writeData(finalFile, '\n') - - preamble = os.path.join(BASEDIR_PATH, "myhosts"); - if os.path.isfile(preamble): - with open(preamble, "r") as f: - writeData(finalFile, f.read()); - - finalFile.write(fileContents) - -def updateReadme(numberOfRules): - with open(README_FILE, "wt") as out: - for line in open(README_TEMPLATE): - out.write(line.replace('@NUM_ENTRIES@', "{:,}".format( numberOfRules ))) - -def moveHostsFileIntoPlace(finalFile): - if (os.name == 'posix'): - print ('Moving the file requires administrative privileges. You might need to enter your password.') - if(subprocess.call(["/usr/bin/sudo", "cp", os.path.abspath(finalFile.name), "/etc/hosts"])): - printFailure("Moving the file failed.") - print ('Flushing the DNS Cache to utilize new hosts file...') - if (platform.system() == 'Darwin'): - if(subprocess.call(["/usr/bin/sudo", "killall", "-HUP", "mDNSResponder"])): - printFailure("Flushing the DNS Cache failed.") + finalFile.seek( 0 ) #write at the top + writeData( finalFile, '# This file is a merged collection of hosts from reputable sources,\n' ) + writeData( finalFile, '# with a dash of crowd sourcing via Github\n#\n' ) + writeData( finalFile, '# Project home page: https://github.com/StevenBlack/hosts\n#\n' ) + writeData( finalFile, '# ===============================================================\n' ) + writeData( finalFile, '\n' ) + writeData( finalFile, '127.0.0.1 localhost\n' ) + writeData( finalFile, '\n' ) + + preamble = os.path.join( BASEDIR_PATH, "myhosts" ); + if os.path.isfile( preamble ): + with open( preamble, "r" ) as f: + writeData( finalFile, f.read() ); + + finalFile.write( fileContents ) + +def updateReadme( numberOfRules ): + with open( README_FILE, "wt" ) as out: + for line in open( README_TEMPLATE ): + out.write( line.replace( '@NUM_ENTRIES@', "{:,}".format( numberOfRules ))) + +def moveHostsFileIntoPlace( finalFile ): + if ( os.name == 'posix' ): + print ( 'Moving the file requires administrative privileges. You might need to enter your password.' ) + if(subprocess.call( ["/usr/bin/sudo", "cp", os.path.abspath( finalFile.name ), "/etc/hosts"] )): + printFailure( "Moving the file failed." ) + print ('Flushing the DNS Cache to utilize new hosts file...' ) + if ( platform.system() == 'Darwin' ): + if( subprocess.call( ["/usr/bin/sudo", "killall", "-HUP", "mDNSResponder"] )): + printFailure( "Flushing the DNS Cache failed." ) else: - if os.path.isfile("/etc/rc.d/init.d/nscd"): - if(subprocess.call(["/usr/bin/sudo", "/etc/rc.d/init.d/nscd", "restart"])): - printFailure("Flushing the DNS Cache failed.") - elif (os.name == 'nt'): - print ('Automatically moving the hosts file in place is not yet supported.') - print ('Please move the generated file to %SystemRoot%\system32\drivers\etc\hosts') - + if os.path.isfile( "/etc/rc.d/init.d/nscd" ): + if( subprocess.call(["/usr/bin/sudo", "/etc/rc.d/init.d/nscd", "restart"] )): + printFailure( "Flushing the DNS Cache failed." ) + elif ( os.name == 'nt' ): + print ( 'Automatically moving the hosts file in place is not yet supported.' ) + print ( 'Please move the generated file to %SystemRoot%\system32\drivers\etc\hosts' ) + def removeOldHostsFile(): # hotfix since merging with an already existing hosts file leads to artefacts and duplicates - oldFilePath=os.path.join(BASEDIR_PATH,'hosts') - open(oldFilePath, 'a').close() # create if already removed, so remove wont raise an error + oldFilePath = os.path.join( BASEDIR_PATH, 'hosts' ) + open( oldFilePath, 'a' ).close() # create if already removed, so remove wont raise an error os.remove(oldFilePath); - open(oldFilePath, 'a').close() # create new empty hostsfile + open( oldFilePath, 'a' ).close() # create new empty hostsfile # End File Logic # Helper Functions ## {{{ http://code.activestate.com/recipes/577058/ (r2) -def query_yes_no(question, default="yes"): +def query_yes_no( question, default = "yes" ): """Ask a yes/no question via raw_input() and return their answer. "question" is a string that is presented to the user. @@ -352,10 +350,10 @@ def query_yes_no(question, default="yes"): elif default == "no": prompt = " [y/N] " else: - raise ValueError("invalid default answer: '%s'" % default) + raise ValueError( "invalid default answer: '%s'" % default ) while 1: - sys.stdout.write(colorize(question, colors.PROMPT) + prompt) + sys.stdout.write( colorize( question, colors.PROMPT ) + prompt ) # Changed to be cross-python choice = myInput().lower() if default is not None and choice == '': @@ -363,17 +361,17 @@ def query_yes_no(question, default="yes"): elif choice in valid.keys(): return valid[choice] else: - printFailure("Please respond with 'yes' or 'no' "\ - "(or 'y' or 'n').\n") + printFailure( "Please respond with 'yes' or 'no' "\ + "(or 'y' or 'n').\n" ) ## end of http://code.activestate.com/recipes/577058/ }}} -def isValidDomainFormat(domain): - if (domain == ''): - print ("You didn\'t enter a domain. Try again.") +def isValidDomainFormat( domain ): + if ( domain == '' ): + print ( "You didn\'t enter a domain. Try again." ) return False - domainRegex = re.compile("www\d{0,3}[.]|https?") - if (domainRegex.match(domain)): - print ("The domain " + domain + " is not valid. Do not include www.domain.com or http(s)://domain.com. Try again.") + domainRegex = re.compile( "www\d{0,3}[.]|https?" ) + if ( domainRegex.match( domain )): + print ( "The domain " + domain + " is not valid. Do not include www.domain.com or http(s)://domain.com. Try again." ) return False else: return True @@ -385,14 +383,14 @@ class colors: FAIL = '\033[91m' ENDC = '\033[0m' -def colorize(text, color): +def colorize( text, color ): return color + text + colors.ENDC -def printSuccess(text): - print (colorize(text, colors.SUCCESS)) +def printSuccess( text ): + print ( colorize(text, colors.SUCCESS )) -def printFailure(text): - print (colorize(text, colors.FAIL)) +def printFailure( text ): + print ( colorize( text, colors.FAIL )) # End Helper Functions if __name__ == "__main__":