if subprocess.call([sys.executable, "updateReadme.py"]):
print_failure("Failed to update readme file")
+
def recursively_loop_extensions(extension, extensions, current_extensions):
"""
Helper function that recursively calls itself to prevent manually creating
Will call update_hosts_file for all combinations of extensions
"""
+
c_extensions = extensions.copy()
c_current_extensions = current_extensions.copy()
c_current_extensions.append(extension)
params = ("-a", "-n", "-o", "alternates/"+name, "-e") + tuple(c_current_extensions)
update_hosts_file(*params)
+ params = ("-a", "-n", "-s", "--nounifiedhosts", "-o", "alternates/"+name+"-only", "-e") + tuple(c_current_extensions)
+ update_hosts_file(*params)
+
while len(c_extensions) > 0:
recursively_loop_extensions(c_extensions.pop(0), c_extensions, c_current_extensions)
[](https://github.com/StevenBlack/hosts/commits/master)
[](https://github.com/StevenBlack/hosts/commits/master)
-# Unified hosts file @EXTENSIONS_HEADER@
+# @EXTENSIONS_HEADER@
This repository consolidates several reputable `hosts` files, and merges them
into a unified hosts file with duplicates removed. A variety of tailored hosts
This repository offers
[15 different host file variants](https://github.com/StevenBlack/hosts/tree/master/alternates),
-in addition to the base variant.
+in addition to the base variant, with and without the unified hosts included.
The **Non GitHub mirror** is the link to use for some hosts file managers like
[Hostsman for Windows](https://www.abelhadigital.com/hostsman/) that don't work
are generating host files with additional whitelists or blacklists and want to
keep your local checkout of this repo unmodified.
+`--nounifiedhosts`: `false` (default) or `true`, do not include the unified hosts
+file in the final hosts file. Usually used together with `--extensions`.
+
`--compress`, or `-c`: `false` (default) or `true`, _Compress_ the hosts file
ignoring non-necessary lines (empty lines and comments) and putting multiple
domains in each line. Reducing the number of lines of the hosts file improves
"keepdomaincomments": True,
"extensionspath": "foo" + self.sep + "extensions",
"extensions": [],
+ "nounifiedhosts": False,
"compress": False,
"minimise": False,
"outputsubfolder": "",
datapath=self.data_path,
extensionspath=self.extensions_path,
sourcedatafilename=self.source_data_filename,
+ nounifiedhosts=False,
)
def update_sources_data(self, sources_data, extensions):
def test_missing_keyword(self):
kwargs = dict(
- extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False
+ extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False, nounifiedhosts=False
)
for k in kwargs.keys():
def test_basic(self):
kwargs = dict(
- extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=True
+ extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=True, nounifiedhosts=False
)
write_opening_header(self.final_file, **kwargs)
def test_basic_include_static_hosts(self):
kwargs = dict(
- extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False
+ extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False, nounifiedhosts=False
)
with self.mock_property("platform.system") as obj:
obj.return_value = "Windows"
def test_basic_include_static_hosts_linux(self):
kwargs = dict(
- extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False
+ extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False, nounifiedhosts=False
)
with self.mock_property("platform.system") as system:
system.return_value = "Linux"
outputsubfolder="",
numberofrules=5,
skipstatichosts=True,
+ nounifiedhosts=False,
)
write_opening_header(self.final_file, **kwargs)
):
self.assertNotIn(expected, contents)
+ def test_no_unified_hosts(self):
+ kwargs = dict(
+ extensions=["epsilon", "gamma"],
+ outputsubfolder="",
+ numberofrules=5,
+ skipstatichosts=True,
+ nounifiedhosts=True,
+ )
+ write_opening_header(self.final_file, **kwargs)
+
+ contents = self.final_file.getvalue()
+ contents = contents.decode("UTF-8")
+
+ # Expected contents.
+ for expected in (
+ ", ".join(kwargs["extensions"]),
+ "# The unified hosts file was not used while generating this file.",
+ "# Extensions used to generate this file:",
+ "# This hosts file is a merged collection",
+ "# with a dash of crowd sourcing via GitHub",
+ "# Number of unique domains: {count}".format(count=kwargs["numberofrules"]),
+ "Fetch the latest version of this file:",
+ "Project home page: https://github.com/StevenBlack/hosts",
+ ):
+ self.assertIn(expected, contents)
+
+ # Expected non-contents.
+ for expected in (
+ "127.0.0.1 localhost",
+ "127.0.0.1 local",
+ "127.0.0.53",
+ "127.0.1.1",
+ ):
+ self.assertNotIn(expected, contents)
+
def _check_preamble(self, check_copy):
hosts_file = os.path.join(self.test_dir, "myhosts")
hosts_file += ".example" if check_copy else ""
f.write("peter-piper-picked-a-pepper")
kwargs = dict(
- extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=True
+ extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=True, nounifiedhosts=False
)
with self.mock_property("updateHostsFile.BASEDIR_PATH"):
def test_missing_keyword(self):
kwargs = dict(
- extensions="", outputsubfolder="", numberofrules="", sourcesdata=""
+ extensions="", outputsubfolder="", numberofrules="", sourcesdata="", nounifiedhosts=False
)
for k in kwargs.keys():
json.dump({"foo": "bar"}, f)
kwargs = dict(
- extensions=None, outputsubfolder="foo", numberofrules=5, sourcesdata="hosts"
+ extensions=None, outputsubfolder="foo", numberofrules=5, sourcesdata="hosts", nounifiedhosts=False
)
update_readme_data(self.readme_file, **kwargs)
sep = self.sep
expected = {
- "base": {"location": "foo" + sep, "sourcesdata": "hosts", "entries": 5},
+ "base": {"location": "foo" + sep, 'no_unified_hosts': False, "sourcesdata": "hosts", "entries": 5},
"foo": "bar",
}
json.dump({"base": "soprano"}, f)
kwargs = dict(
- extensions=None, outputsubfolder="foo", numberofrules=5, sourcesdata="hosts"
+ extensions=None, outputsubfolder="foo", numberofrules=5, sourcesdata="hosts", nounifiedhosts=False
)
update_readme_data(self.readme_file, **kwargs)
sep = self.sep
expected = {
- "base": {"location": "foo" + sep, "sourcesdata": "hosts", "entries": 5}
+ "base": {"location": "foo" + sep, 'no_unified_hosts': False, "sourcesdata": "hosts", "entries": 5},
}
with open(self.readme_file, "r") as f:
outputsubfolder="foo",
numberofrules=5,
sourcesdata="hosts",
+ nounifiedhosts=False,
+ )
+ update_readme_data(self.readme_file, **kwargs)
+
+ if platform.system().lower() == "windows":
+ sep = "/"
+ else:
+ sep = self.sep
+
+ expected = {
+ "com-org": {"location": "foo" + sep, 'no_unified_hosts': False, "sourcesdata": "hosts", "entries": 5}
+ }
+
+ with open(self.readme_file, "r") as f:
+ actual = json.load(f)
+ self.assertEqual(actual, expected)
+
+ def test_set_no_unified_hosts(self):
+ with open(self.readme_file, "w") as f:
+ json.dump({}, f)
+
+ kwargs = dict(
+ extensions=["com", "org"],
+ outputsubfolder="foo",
+ numberofrules=5,
+ sourcesdata="hosts",
+ nounifiedhosts=True,
)
update_readme_data(self.readme_file, **kwargs)
sep = self.sep
expected = {
- "com-org": {"location": "foo" + sep, "sourcesdata": "hosts", "entries": 5}
+ "com-org-only": {"location": "foo" + sep, 'no_unified_hosts': True, "sourcesdata": "hosts", "entries": 5}
}
with open(self.readme_file, "r") as f:
class TestRemoveOldHostsFile(BaseMockDir):
def setUp(self):
super(TestRemoveOldHostsFile, self).setUp()
- self.hosts_file = os.path.join(self.test_dir, "hosts")
+ self.hosts_file = "hosts"
+ self.full_hosts_path = os.path.join(self.test_dir, "hosts")
def test_remove_hosts_file(self):
old_dir_count = self.dir_count
- remove_old_hosts_file(self.hosts_file, backup=False)
+ remove_old_hosts_file(self.test_dir, self.hosts_file, backup=False)
new_dir_count = old_dir_count + 1
self.assertEqual(self.dir_count, new_dir_count)
- with open(self.hosts_file, "r") as f:
+ with open(self.full_hosts_path, "r") as f:
contents = f.read()
self.assertEqual(contents, "")
def test_remove_hosts_file_exists(self):
- with open(self.hosts_file, "w") as f:
+ with open(self.full_hosts_path, "w") as f:
f.write("foo")
old_dir_count = self.dir_count
- remove_old_hosts_file(self.hosts_file, backup=False)
+ remove_old_hosts_file(self.test_dir, self.hosts_file, backup=False)
new_dir_count = old_dir_count
self.assertEqual(self.dir_count, new_dir_count)
- with open(self.hosts_file, "r") as f:
+ with open(self.full_hosts_path, "r") as f:
contents = f.read()
self.assertEqual(contents, "")
@mock.patch("time.strftime", return_value="new")
def test_remove_hosts_file_backup(self, _):
- with open(self.hosts_file, "w") as f:
+ with open(self.full_hosts_path, "w") as f:
f.write("foo")
old_dir_count = self.dir_count
- remove_old_hosts_file(self.hosts_file, backup=True)
+ remove_old_hosts_file(self.test_dir, self.hosts_file, backup=True)
new_dir_count = old_dir_count + 1
self.assertEqual(self.dir_count, new_dir_count)
- with open(self.hosts_file, "r") as f:
+ with open(self.full_hosts_path, "r") as f:
contents = f.read()
self.assertEqual(contents, "")
- new_hosts_file = self.hosts_file + "-new"
+ new_hosts_file = self.full_hosts_path + "-new"
with open(new_hosts_file, "r") as f:
contents = f.read()
"keepdomaincomments": True,
"extensionspath": path_join_robust(BASEDIR_PATH, "extensions"),
"extensions": [],
+ "nounifiedhosts": False,
"compress": False,
"minimise": False,
"outputsubfolder": "",
nargs="*",
help="Host extensions to include in the final hosts file.",
)
+ parser.add_argument(
+ "--nounifiedhosts",
+ dest="nounifiedhosts",
+ default=False,
+ action="store_true",
+ help="Do not include the unified hosts file in the final hosts file. Usually used together with `--extensions`.",
+ )
parser.add_argument(
"--ip",
"-i",
auto = settings["auto"]
exclusion_regexes = settings["exclusionregexes"]
source_data_filename = settings["sourcedatafilename"]
+ no_unified_hosts = settings["nounifiedhosts"]
update_sources = prompt_for_update(freshen=settings["freshen"], update_auto=auto)
if update_sources:
extensions=extensions,
extensionspath=extensions_path,
sourcedatafilename=source_data_filename,
+ nounifiedhosts=no_unified_hosts,
)
- merge_file = create_initial_file()
+ merge_file = create_initial_file(
+ nounifiedhosts=no_unified_hosts,
+ )
remove_old_hosts_file(settings["outputpath"], "hosts", settings["backup"])
if settings["compress"]:
final_file = open(path_join_robust(settings["outputpath"], "hosts"), "w+b")
numberofrules=number_of_rules,
outputsubfolder=output_subfolder,
skipstatichosts=skip_static_hosts,
+ nounifiedhosts=no_unified_hosts,
)
final_file.close()
numberofrules=number_of_rules,
outputsubfolder=output_subfolder,
sourcesdata=sources_data,
+ nounifiedhosts=no_unified_hosts,
)
print_success(
2) extensions
3) extensionspath
4) sourcedatafilename
+ 5) nounifiedhosts
Returns
-------
source_data_filename = sources_params["sourcedatafilename"]
- for source in sort_sources(
- recursive_glob(sources_params["datapath"], source_data_filename)
- ):
- update_file = open(source, "r", encoding="UTF-8")
- update_data = json.load(update_file)
- sources_data.append(update_data)
- update_file.close()
+ if not sources_params["nounifiedhosts"]:
+ for source in sort_sources(
+ recursive_glob(sources_params["datapath"], source_data_filename)
+ ):
+ update_file = open(source, "r", encoding="UTF-8")
+ update_data = json.load(update_file)
+ sources_data.append(update_data)
+ update_file.close()
for source in sources_params["extensions"]:
source_dir = path_join_robust(sources_params["extensionspath"], source)
# File Logic
-def create_initial_file():
+def create_initial_file(**initial_file_params):
"""
Initialize the file in which we merge all host files for later pruning.
+
+ Parameters
+ ----------
+ header_params : kwargs
+ Dictionary providing additional parameters for populating the initial file
+ information. Currently, those fields are:
+
+ 1) nounifiedhosts
"""
merge_file = tempfile.NamedTemporaryFile()
- # spin the sources for the base file
- for source in sort_sources(
- recursive_glob(settings["datapath"], settings["hostfilename"])
- ):
+ if not initial_file_params["nounifiedhosts"]:
+ # spin the sources for the base file
+ for source in sort_sources(
+ recursive_glob(settings["datapath"], settings["hostfilename"])
+ ):
- start = "# Start {}\n\n".format(os.path.basename(os.path.dirname(source)))
- end = "\n# End {}\n\n".format(os.path.basename(os.path.dirname(source)))
+ start = "# Start {}\n\n".format(os.path.basename(os.path.dirname(source)))
+ end = "\n# End {}\n\n".format(os.path.basename(os.path.dirname(source)))
- with open(source, "r", encoding="UTF-8") as curFile:
- write_data(merge_file, start + curFile.read() + end)
+ with open(source, "r", encoding="UTF-8") as curFile:
+ write_data(merge_file, start + curFile.read() + end)
# spin the sources for extensions to the base file
for source in settings["extensions"]:
2) numberofrules
3) outputsubfolder
4) skipstatichosts
+ 5) nounifiedhosts
"""
final_file.seek(0) # Reset file pointer.
final_file.seek(0) # Write at the top.
+ no_unified_hosts = header_params["nounifiedhosts"]
+
if header_params["extensions"]:
- if len(header_params["extensions"]) > 1:
- write_data(
- final_file,
- "# Title: StevenBlack/hosts with the {0} and {1} extensions\n#\n".format(
- ", ".join(header_params["extensions"][:-1]),
- header_params["extensions"][-1],
- ),
- )
+ if no_unified_hosts:
+ if len(header_params["extensions"]) > 1:
+ write_data(
+ final_file,
+ "# Title: StevenBlack/hosts extensions {0} and {1} \n#\n".format(
+ ", ".join(header_params["extensions"][:-1]),
+ header_params["extensions"][-1],
+ ),
+ )
+ else:
+ write_data(
+ final_file,
+ "# Title: StevenBlack/hosts extension {0}\n#\n".format(
+ ", ".join(header_params["extensions"])
+ ),
+ )
else:
- write_data(
- final_file,
- "# Title: StevenBlack/hosts with the {0} extension\n#\n".format(
- ", ".join(header_params["extensions"])
- ),
- )
+ if len(header_params["extensions"]) > 1:
+ write_data(
+ final_file,
+ "# Title: StevenBlack/hosts with the {0} and {1} extensions\n#\n".format(
+ ", ".join(header_params["extensions"][:-1]),
+ header_params["extensions"][-1],
+ ),
+ )
+ else:
+ write_data(
+ final_file,
+ "# Title: StevenBlack/hosts with the {0} extension\n#\n".format(
+ ", ".join(header_params["extensions"])
+ ),
+ )
else:
write_data(final_file, "# Title: StevenBlack/hosts\n#\n")
)
if header_params["extensions"]:
- write_data(
- final_file,
- "# Extensions added to this file: "
- + ", ".join(header_params["extensions"])
- + "\n",
- )
+ if header_params["nounifiedhosts"]:
+ write_data(
+ final_file,
+ "# The unified hosts file was not used while generating this file.\n"
+ "# Extensions used to generate this file: "
+ + ", ".join(header_params["extensions"])
+ + "\n",
+ )
+ else:
+ write_data(
+ final_file,
+ "# Extensions added to this file: "
+ + ", ".join(header_params["extensions"])
+ + "\n",
+ )
write_data(
final_file,
2) sourcesdata
3) numberofrules
4) outputsubfolder
+ 5) nounifiedhosts
"""
extensions_key = "base"
extensions = readme_updates["extensions"]
+ no_unified_hosts = readme_updates["nounifiedhosts"]
if extensions:
extensions_key = "-".join(extensions)
+ if no_unified_hosts:
+ extensions_key = extensions_key + "-only"
output_folder = readme_updates["outputsubfolder"]
generation_data = {
"location": path_join_robust(output_folder, ""),
+ "no_unified_hosts": no_unified_hosts,
"entries": readme_updates["numberofrules"],
"sourcesdata": readme_updates["sourcesdata"],
}
keys = list(data.keys())
# Sort by the number of en-dashes in the key
# and then by the key string itself.
- keys.sort(key=lambda item: (item.count("-"), item))
+ keys.sort(key=lambda item: (item.replace("-only", "").count("-"), item.replace("-only", "")))
toc_rows = ""
for key in keys:
if key == "base":
data[key]["description"] = "Unified hosts = **(adware + malware)**"
else:
- data[key]["description"] = (
- "Unified hosts **+ " + key.replace("-", " + ") + "**"
- )
+ if data[key]["no_unified_hosts"]:
+ data[key]["description"] = (
+ "**" + key.replace("-only", "").replace("-", " + ") + "**"
+ )
+ else:
+ data[key]["description"] = (
+ "Unified hosts **+ " + key.replace("-", " + ") + "**"
+ )
if "\\" in data[key]["location"]:
data[key]["location"] = data[key]["location"].replace("\\", "/")
)
size_history_graph = ""
for key in keys:
- extensions = key.replace("-", ", ")
+ extensions = key.replace("-only", "").replace("-", ", ")
extensions_str = "* Extensions: **" + extensions + "**."
- extensions_header = "with " + extensions + " extensions"
+ if data[key]["no_unified_hosts"]:
+ extensions_header = "Limited to the extensions: " + extensions
+ else:
+ extensions_header = "Unified hosts file with " + extensions + " extensions"
source_rows = ""
source_list = data[key]["sourcesdata"]
git.99rst.org / stevenblack-hosts.git / commitdiff