diff --git a/README.md b/README.md index 0f9ceb1..a06fb27 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,8 @@ Please note: ``` usage: python3 hashtopolis.zip [-h] [--de-register] [--version] [--number-only] [--disable-update] [--debug] [--voucher VOUCHER] [--url URL] [--cert CERT] [--files-path FILES_PATH] [--crackers-path CRACKERS_PATH] [--hashlists-path HASHLISTS_PATH] - [--preprocessors-path PREPROCESSORS_PATH] [--zaps-path ZAPS_PATH] [--cpu-only] + [--preprocessors-path PREPROCESSORS_PATH] [--zaps-path ZAPS_PATH] [--cache-path CACHE_PATH] + [--cpu-only] Hashtopolis Client v0.7.5 @@ -57,6 +58,8 @@ options: Use given folder path as preprocessors location --zaps-path ZAPS_PATH Use given folder path as zaps location + --cache-path CACHE_PATH + Use given folder path as hashcat cache location (needs hashcat support for --cache-path) --cpu-only Force client to register as CPU only and also only reading out CPU information ``` @@ -98,8 +101,30 @@ When you run the client for the first time it will ask automatically for all the | hashlists-path | string | | Use given folder path as hashlists location | | preprocessors-path | string | | Use given folder path as preprocessors location | | zaps-path | string | | Use given folder path as zaps location | +| cache-path | string | | Folder hashcat caches everything in (`--cache-path`, see note below) | | cpu-only | boolean | false | Only send CPU information about agent (for CPU only agents) | +#### cache-path + +hashcat keeps a seek database per wordlist, which is what makes the `--skip` of every Hashtopolis +chunk cheap, plus its compiled kernels. By default those live inside the cracker folder, so they are +cold again after every cracker binary update. `cache-path` moves them to a directory of your choice. + +Seek databases are named after a hash of the wordlist they describe, so several agents pointed at +one shared mount build each database only once for the whole cluster. + +Notes: + +* Use a **local, writable** directory. hashcat tolerates a read-only location for the wordlist + caches, but a failed kernel cache write aborts the session. To share across agents, point + `cache-path` at a local directory and mount or symlink only `cache/feeds/wordlist` to the share. +* The option is only passed to crackers new enough for it. It appears in hashcat at build + `7.1.2-627-g6570a49f2`, so the agent passes it to that build and later, and to any release after + 7.1.2. Anything older, including the 7.1.2 release itself, stays silent and is unaffected. +* The agent creates the directory at startup if it is missing. If it points at a mount that is + down, that creates an empty local directory shadowing the mountpoint - check the log. +* It is not removed when the agent de-registers, since it may be shared with other agents. + ### Debug example ``` @@ -128,6 +153,7 @@ In order to use the multicast distribution for files, please make sure that the The list contains all Hashcat versions with which the client was tested and is able to work with (other versions might work): +* 7.1.2 * 7.1.1 * 7.1.0 * 7.0.0 diff --git a/__main__.py b/__main__.py index d45ffee..83ec00c 100644 --- a/__main__.py +++ b/__main__.py @@ -142,6 +142,13 @@ def init(args): if args.preprocessors_path and len(args.preprocessors_path): CONFIG.set_value('preprocessors-path', os.path.abspath(args.preprocessors_path)) + # cache-path has no default on purpose: when it is unset hashcat keeps using its own cache location + if args.cache_path and len(args.cache_path): + CONFIG.set_value('cache-path', args.cache_path) + if len(CONFIG.get_value('cache-path')) > 0: + # hashcat is started with cwd=crackers/, so a relative path would end up inside the cracker folder + CONFIG.set_value('cache-path', os.path.abspath(CONFIG.get_value('cache-path'))) + logging.info("Starting client '" + Initialize.get_version() + "'...") # check if there are running hashcat.pid files around (as we assume that nothing is running anymore if the client gets newly started) @@ -305,6 +312,7 @@ def de_register(): else: logging.info("Successfully de-registered!") # cleanup + # cache-path is left alone on purpose: it is meant to be shared between agents and is rebuildable dirs = [CONFIG.get_value('crackers-path'), CONFIG.get_value('preprocessors-path'), CONFIG.get_value('hashlists-path'), CONFIG.get_value('files-path')] files = ['config.json', '7zr.exe', '7zr'] for file in files: @@ -334,6 +342,7 @@ def de_register(): parser.add_argument('--hashlists-path', type=str, required=False, help='Use given folder path as hashlists location') parser.add_argument('--preprocessors-path', type=str, required=False, help='Use given folder path as preprocessors location') parser.add_argument('--zaps-path', type=str, required=False, help='Use given folder path as zaps location') + parser.add_argument('--cache-path', type=str, required=False, help='Use given folder path as hashcat cache location (needs hashcat support for --cache-path)') parser.add_argument('--cpu-only', action='store_true', help='Force client to register as CPU only and also only reading out CPU information') args = parser.parse_args() diff --git a/changelog.md b/changelog.md index cfafff0..f709ca9 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,15 @@ > [!NOTE] > Changelog will not be updated anymore, use the github releases section to get information about changes. +## v0.7.4 -> v0.7.5 + +### Enhancements + +* New `cache-path` config option and `--cache-path` command line argument to set the folder hashcat + caches wordlist seek databases and kernels in. Several agents can share one location. Only sent to + crackers which support `--cache-path`. + + ## v0.7.3-> v0.7.4 ### Bugfixes diff --git a/htpclient/hashcat_cracker.py b/htpclient/hashcat_cracker.py index b01bb38..95b856a 100644 --- a/htpclient/hashcat_cracker.py +++ b/htpclient/hashcat_cracker.py @@ -17,6 +17,73 @@ from htpclient.dicts import * +# Where hashcat keeps the per-wordlist seek databases that make --skip cheap, and the compiled +# kernels. +CACHE_PATH_FLAG = '--cache-path' + +# The release the option landed after, and the build on that release which first carried it. It +# appears in src/usage.c at v7.1.2-627-g6570a49f2; the build before it has no such option. Once a +# release ships with it, this becomes that release and the build number stops mattering. +CACHE_PATH_RELEASE = (7, 1, 2, 627) + +def parse_version(version_string): + """Turn the version a cracker reports into a tuple that compares the way the versions do. + + "7.1.2" becomes (7, 1, 2) and a build made after a release, "7.1.2-627-g6570a49f2", becomes + (7, 1, 2, 627); the commit hash after the build number is dropped. A release sorts before every + build made on it, so plain tuple comparison against a minimum like (5, 1, 0, 1618) works. + + Returns None when the string cannot be read as a version, so each caller decides what an + unknown version means for it. + """ + if not version_string: + return None + parts = version_string.strip().split('-') + try: + version = tuple(int(s) for s in parts[0].split('.')) + if len(parts) > 1: + version += (int(parts[1]),) + except ValueError: + return None + return version + +def version_supports_cache_path(version_string): + """Whether the version a cracker reports is one that knows --cache-path. + + The version arrives as either "7.1.2" or "7.1.2-627-g6570a49f2", which is how hashcat spells a + build made after a release, as read by parse_version(). The option landed + part way through the builds that followed 7.1.2, so on that release it is the build number that + decides, and any later release has it outright. + + Anything that cannot be read as a version is treated as too old. hashcat does not ignore an + option it does not know, it stops, so the safe answer when in doubt is to say nothing. + """ + version = parse_version(version_string) + return version is not None and version > CACHE_PATH_RELEASE + +def select_cache_arg(cache_path, version_string, isdir=os.path.isdir): + """Build the hashcat arguments for the configured cache directory. + + Returns an empty list when nothing is configured, when the cracker is too old for the option, or + when the directory is not there right now. Both an unknown option and a missing cache directory + make hashcat abort immediately, and a cache is a speed-up, never worth failing a chunk over - so + we drop the option and let hashcat fall back to its own cache location. + """ + if not cache_path: + return [] + if '"' in cache_path: + # the value is quoted, not escaped, when it goes into the shell command line + logging.debug(f"Ignoring cache-path '{cache_path}': the path must not contain a double quote") + return [] + if version_supports_cache_path(version_string) is False: + logging.debug(f"Ignoring cache-path '{cache_path}': cracker version {version_string.strip() if version_string else ''} does not support {CACHE_PATH_FLAG}") + return [] + if not isdir(cache_path): + logging.debug(f"Ignoring cache-path '{cache_path}': not a directory, hashcat will use its own cache location") + return [] + return [f'{CACHE_PATH_FLAG}="{cache_path}"'] + + class HashcatCracker: def __init__(self, cracker_id, binary_download): self.config = Config() @@ -63,26 +130,23 @@ def __init__(self, cracker_id, binary_download): self.wasStopped = False def get_outfile_format(self): - if self.version_string.find('-') == -1: - release = self.version_string.split('.') - try: - if int(str(release[0])) >= 6: - return "1,2,3,4" - except ValueError: - return "1,2,3,4" # if there is a custom version, we assume it's using the new format - return "15" # if we cannot determine the version or if the release is older than 6.0.0, we will use the old format - split = self.version_string.split('-') - if len(split) < 2: - return "15" # something is wrong with the version string, go for old format - release = str(split[0]).split('.') - commit = str(split[1]) - if int(str(release[0])) < 5: - return "15" - elif int(str(release[0])) == 5 and int(str(release[1])) < 1: - return "15" - elif int(str(release[0])) == 5 and int(str(release[1])) == 1 and int(str(release[2])) == 0 and int(commit) < 1618: - return "15" - return "1,2,3,4" # new outfile format + version = parse_version(self.version_string) + if version is None: + return "1,2,3,4" # if there is a custom version, we assume it's using the new format + if version >= (5, 1, 0, 1618): + return "1,2,3,4" # new outfile format + return "15" + + def get_cache_args(self): + """Arguments pointing hashcat at the configured cache directory, empty list if unavailable.""" + cache_path = self.config.get_value('cache-path') + if not cache_path: + return [] + return select_cache_arg(cache_path, self.version_string) + + def get_cache_args_str(self): + """get_cache_args() for the builders that concatenate strings: leading space, or empty.""" + return ''.join(f' {arg}' for arg in self.get_cache_args()) def build_command(self, task, chunk): args = [] @@ -120,6 +184,7 @@ def build_command(self, task, chunk): files = update_files(task['attackcmd']) files = files.replace(task['hashlistAlias'], f'"{hashlist_file}"') args.append(files) + args.extend(self.get_cache_args()) args.append(task['cmdpars']) @@ -136,6 +201,7 @@ def build_pipe_command(self, task, chunk): # call the command with piping pre_args = " --stdout -s " + str(chunk['skip']) + " -l " + str(chunk['length']) + ' ' pre_args += update_files(task['attackcmd']).replace(task['hashlistAlias'], '') + pre_args += self.get_cache_args_str() post_args = " --machine-readable --quiet --status --remove --restore-disable --potfile-disable --session=hashtopolis" post_args += " --status-timer " + str(task['statustimer']) post_args += " --outfile-check-timer=" + str(task['statustimer']) @@ -143,6 +209,7 @@ def build_pipe_command(self, task, chunk): post_args += " -o '" + self.config.get_value('hashlists-path') + "/" + str(task['hashlistId']) + ".out' --outfile-format=" + self.get_outfile_format() + " -p \"" + str(chr(9)) + "\"" post_args += " --remove-timer=" + str(task['statustimer']) post_args += " '" + self.config.get_value('hashlists-path') + "/" + str(task['hashlistId']) + "'" + post_args += self.get_cache_args_str() return f"'{self.callPath}'" + pre_args + " | " + f"'{self.callPath}'" + post_args + task['cmdpars'] # DEPRECATED @@ -213,6 +280,7 @@ def build_preprocessor_command(self, task, chunk, preprocessor): files = update_files(task['attackcmd']) files = files.replace(task['hashlistAlias'] + " ", "") post_args.append(files) + post_args.extend(self.get_cache_args()) post_args.append(task['cmdpars']) pre_args = ' '.join(pre_args) @@ -291,6 +359,10 @@ def run_loop(self, proc, chunk, task): piping_threshold = self.config.get_value('piping-threshold') if self.config.get_value('allow-piping') != '': enable_piping = self.config.get_value('allow-piping') + # before hashcat 7.1.2-546, the PROGRESS fields included the skip value. A version we cannot + # read is assumed to be a recent custom build, as get_outfile_format() does. + version = parse_version(self.version_string) + progress_includes_skip = version is not None and version < (7, 1, 2, 546) while True: try: # Block for 1 second. @@ -316,10 +388,6 @@ def run_loop(self, proc, chunk, task): else: identifier, line = item if identifier == 'OUT': - # "7.1.2" -> (7,1,2) - # "7.1.2-546-gc885beef" -> (7,1,2,546) - version_list = self.version_string.replace('-', '.').split('.')[:4] # Chop off anything past the fourth field (for example commit hash) - version_tuple = tuple(int(s) for s in version_list) status = HashcatStatus(line.decode()) if status.is_valid(): self.statusCount += 1 @@ -348,12 +416,11 @@ def run_loop(self, proc, chunk, task): # we need to calculate the chunk start, because progress does not start at 0 for a chunk chunk_start = int(status.get_progress_total() / (chunk['skip'] + chunk['length']) * chunk['skip']) if total > 0: - if version_tuple >= (7, 1, 2, 546): - relative_progress = int(status.get_progress() / total * 10000) - else: - # before hashcat 7.1.2-546, the PROGRESS fields included the skip value. - # Adjust for this offset + if progress_includes_skip: + # Adjust for the skip offset in the PROGRESS fields relative_progress = int((status.get_progress() - chunk_start) / float(total - chunk_start) * 10000) + else: + relative_progress = int(status.get_progress() / total * 10000) else: # this is the case when we cannot say anything about the progress relative_progress = 0 speed = status.get_speed() @@ -456,7 +523,7 @@ def measure_keyspace(self, task, chunk): files = update_files(task['attackcmd']) files = files.replace(task['hashlistAlias'] + " ", "") - full_cmd = f"{self.callPath} --keyspace --quiet {files} {task['cmdpars']}" + full_cmd = f"{self.callPath} --keyspace --quiet {files}{self.get_cache_args_str()} {task['cmdpars']}" if 'useBrain' in task and task['useBrain']: full_cmd = f"{full_cmd} -S" @@ -581,6 +648,7 @@ def run_benchmark(self, task): files = files.replace(task['hashlistAlias'], f'"{hashlist_path}"') args.append(files) + args.extend(self.get_cache_args()) args.append(task['cmdpars']) args.append('-o') args.append(f'"{hashlist_out_path}"') @@ -662,6 +730,7 @@ def run_speed_benchmark(self, task): attackcmd = attackcmd.replace(task['hashlistAlias'], f'"{hashlist_path}"') args.append(attackcmd) + args.extend(self.get_cache_args()) args.append(task['cmdpars']) if 'usePreprocessor' in task and task['usePreprocessor']: args.append('example.dict') @@ -735,6 +804,9 @@ def run_health_check(self, attack, hashlist_alias): args.append('--restore-disable') args.append('--potfile-disable') args.append('--session=health') + # this ends up in the Windows slash rewrite of full_cmd below, which is harmless: on Windows + # os.path.abspath already hands us a backslash path + args.extend(self.get_cache_args()) attackcmd = update_files(attack) hashlist_path = Path(self.config.get_value('hashlists-path'), "health_check.txt") hashlist_out_path = Path(self.config.get_value('hashlists-path'), "health_check.out") diff --git a/htpclient/initialize.py b/htpclient/initialize.py index 0e02e68..3d45a78 100644 --- a/htpclient/initialize.py +++ b/htpclient/initialize.py @@ -241,3 +241,14 @@ def __build_directories(self): os.makedirs(self.config.get_value('hashlists-path')) if not os.path.isdir(self.config.get_value('preprocessors-path')): os.makedirs(self.config.get_value('preprocessors-path')) + # hashcat does not create the cache directory it is handed and aborts when it is missing, so we + # create it here. It may be a read-only or currently unavailable mount, which must not stop the + # agent from starting - the cracker checks again per chunk and drops the option if it is gone. + # exist_ok, because this is the one directory here that several agents may be sharing: two of + # them starting together both find it missing and both go on to create it. + cache_path = self.config.get_value('cache-path') + if cache_path and not os.path.isdir(cache_path): + try: + os.makedirs(cache_path, exist_ok=True) + except OSError as e: + logging.warning("Could not create cache directory '" + cache_path + "': " + str(e)) diff --git a/tests/test_cache_path.py b/tests/test_cache_path.py new file mode 100644 index 0000000..60b4315 --- /dev/null +++ b/tests/test_cache_path.py @@ -0,0 +1,140 @@ +import unittest +from htpclient.hashcat_cracker import HashcatCracker, parse_version, select_cache_arg, version_supports_cache_path + +# Versions as hashcat reports them: a release on its own, and a build made after one, which carries +# the commit count and hash. --cache-path arrived after 7.1.2 was released. +RELEASE_WITHOUT = "7.1.2" +BUILD_WITH = "7.1.2-629-g3b4a9ea09+" + + +class TestVersionSupportsCachePath(unittest.TestCase): + def test_releases_without_the_option(self): + self.assertFalse(version_supports_cache_path("7.1.2")) + self.assertFalse(version_supports_cache_path("7.1.1")) + self.assertFalse(version_supports_cache_path("7.0.0")) + self.assertFalse(version_supports_cache_path("6.2.6")) + + def test_builds_on_that_release(self): + # builds after 627 have the option + self.assertTrue(version_supports_cache_path("7.1.2-628-g1a2b3c4d5")) + self.assertTrue(version_supports_cache_path("7.1.2-629-g3b4a9ea09+")) + self.assertTrue(version_supports_cache_path("7.1.2-658-gf975d8b")) + self.assertTrue(version_supports_cache_path("7.1.2-1200-gabcdef1")) + + def test_builds_before_the_option_landed(self): + # a self built snapshot from between the release and that commit has no such option, and + # hashcat stops on one it does not know + self.assertFalse(version_supports_cache_path("7.1.2-627-g6570a49f2")) + self.assertFalse(version_supports_cache_path("7.1.2-626-gc84986904")) + self.assertFalse(version_supports_cache_path("7.1.2-1-gdeadbee")) + self.assertFalse(version_supports_cache_path("7.1.2-gnocount")) + + def test_later_releases(self): + self.assertTrue(version_supports_cache_path("7.1.3")) + self.assertTrue(version_supports_cache_path("7.2.0")) + self.assertTrue(version_supports_cache_path("8.0.0")) + self.assertTrue(version_supports_cache_path("10.0.0")) + + def test_unreadable_versions_are_treated_as_too_old(self): + # an option hashcat does not know stops the run, so anything unclear has to say no + self.assertFalse(version_supports_cache_path("")) + self.assertFalse(version_supports_cache_path(None)) + self.assertFalse(version_supports_cache_path("latest")) + self.assertFalse(version_supports_cache_path("unknown-build")) + + def test_whitespace_and_short_versions(self): + self.assertTrue(version_supports_cache_path(" 7.2.0\n")) + self.assertFalse(version_supports_cache_path("7.1")) + self.assertTrue(version_supports_cache_path("7.2")) + + +class TestSelectCacheArg(unittest.TestCase): + def test_no_cache_path_configured(self): + # the default: nothing configured, so the command line stays exactly as it was + self.assertEqual(select_cache_arg('', BUILD_WITH), []) + self.assertEqual(select_cache_arg(None, BUILD_WITH), []) + + def test_supported_cracker(self): + self.assertEqual( + select_cache_arg('/mnt/hccache', BUILD_WITH, isdir=lambda p: True), + ['--cache-path="/mnt/hccache"'] + ) + + def test_cracker_too_old_is_skipped(self): + # 7.1.2 and older have no such option and abort on an unknown one + with self.assertLogs(level='DEBUG') as logs: + self.assertEqual(select_cache_arg('/mnt/hccache', RELEASE_WITHOUT, isdir=lambda p: True), []) + self.assertEqual(len(logs.output), 1) + self.assertIn('does not support', logs.output[0]) + self.assertIn('7.1.2', logs.output[0]) + + def test_unreadable_version_is_skipped(self): + self.assertEqual(select_cache_arg('/mnt/hccache', 'latest', isdir=lambda p: True), []) + self.assertEqual(select_cache_arg('/mnt/hccache', '', isdir=lambda p: True), []) + + def test_missing_directory_is_skipped(self): + # hashcat aborts when the cache directory it is handed does not exist, so we drop the option + with self.assertLogs(level='DEBUG') as logs: + self.assertEqual(select_cache_arg('/mnt/gone', BUILD_WITH, isdir=lambda p: False), []) + self.assertEqual(len(logs.output), 1) + self.assertIn('not a directory', logs.output[0]) + + def test_path_with_space_stays_quoted(self): + self.assertEqual( + select_cache_arg('C:\\hashcat cache', BUILD_WITH, isdir=lambda p: True), + ['--cache-path="C:\\hashcat cache"'] + ) + + def test_path_with_double_quote_is_rejected(self): + # the value is quoted, not escaped, when it lands in the shell command line + with self.assertLogs(level='DEBUG') as logs: + self.assertEqual(select_cache_arg('/mnt/a"b', BUILD_WITH, isdir=lambda p: True), []) + self.assertEqual(len(logs.output), 1) + self.assertIn('double quote', logs.output[0]) + + +class TestParseVersion(unittest.TestCase): + def test_release(self): + self.assertEqual(parse_version("7.1.2"), (7, 1, 2)) + self.assertEqual(parse_version(" 6.2.6\n"), (6, 2, 6)) + + def test_build_drops_commit_hash(self): + self.assertEqual(parse_version("7.1.2-629-g3b4a9ea09+"), (7, 1, 2, 629)) + self.assertEqual(parse_version("5.1.0-1618"), (5, 1, 0, 1618)) + + def test_release_sorts_before_its_builds(self): + self.assertLess(parse_version("7.1.2"), parse_version("7.1.2-1-gdeadbee")) + self.assertLess(parse_version("7.1.2-1200-gabcdef1"), parse_version("7.1.3")) + + def test_unreadable(self): + self.assertIsNone(parse_version("")) + self.assertIsNone(parse_version(None)) + self.assertIsNone(parse_version("latest")) + self.assertIsNone(parse_version("unknown-build")) + self.assertIsNone(parse_version("7.1.2-gnocount")) + + +class TestOutfileFormat(unittest.TestCase): + def outfile_format(self, version_string): + cracker = HashcatCracker.__new__(HashcatCracker) # skip __init__, which runs the binary + cracker.version_string = version_string + return cracker.get_outfile_format() + + def test_old_format(self): + self.assertEqual(self.outfile_format("4.2.1"), "15") + self.assertEqual(self.outfile_format("5.0.0-100-gabcdef1"), "15") + self.assertEqual(self.outfile_format("5.1.0"), "15") + self.assertEqual(self.outfile_format("5.1.0-1617-gabcdef1"), "15") + + def test_new_format(self): + self.assertEqual(self.outfile_format("5.1.0-1618-gabcdef1"), "1,2,3,4") + self.assertEqual(self.outfile_format("6.0.0"), "1,2,3,4") + self.assertEqual(self.outfile_format("7.1.2-629-g3b4a9ea09+"), "1,2,3,4") + + def test_custom_version_assumes_new_format(self): + self.assertEqual(self.outfile_format("custom"), "1,2,3,4") + self.assertEqual(self.outfile_format(""), "1,2,3,4") + + +if __name__ == '__main__': + unittest.main()