diff --git a/main.py b/main.py index f53d888..1ca455e 100644 --- a/main.py +++ b/main.py @@ -16,50 +16,70 @@ import importlib import sys +import types from absl import app from absl import flags - import tcli.tcli_lib as tcli - flags.DEFINE_string( - 'cmds', None, """ + 'cmds', + None, + """ Commands (newline separated) to send to devices in the target list. 'Prompting' commands, commands that request further input from the user before completing are discouraged and will fail. Examples to avoid: telnet, ping, reload. - Shortname: 'C'.""", short_name='C') + Shortname: 'C'.""", + short_name='C', +) flags.DEFINE_boolean( - 'interactive', False, """ + 'interactive', + False, + """ TCLI runs in interactive mode. This is the default if no cmds are supplied.' - Shortname: 'I'.""", short_name='I') -# Defaults to inventory_csv.py which contains canned data for testing. -flags.DEFINE_string('accessor_file', 'accessor_canned', - 'Name of the module that implements SendRequests.') -# Defaults to inventory_csv.py which contains canned data for testing. -flags.DEFINE_string('inventory_file', 'inventory_csv', - 'Name of the module that implements the Inventory class.') + Shortname: 'I'.""", + short_name='I', +) +flags.DEFINE_string( + 'accessor_file', + 'accessor_canned', + 'Name of the module that implements SendRequests.', +) +flags.DEFINE_string( + 'inventory_file', + 'inventory_csv', + 'Name of the module that implements the Inventory class.', +) FLAGS = flags.FLAGS -def main(_): +def _ParseFlags(argv: list[str]) -> list[str]: + """Pre-loads accessor and inventory modules before parsing all CLI flags.""" + FLAGS(argv, known_only=True) # Replace the generic Accessor function with the site specific one. tcli.accessor = importlib.import_module(f'tcli.{FLAGS.accessor_file}') # Replace the generic Inventory class with the site specific one. tcli.inventory = importlib.import_module(f'tcli.{FLAGS.inventory_file}') + return app.parse_flags_with_usage(argv) + + +def main(_): try: # If no commands supplied via flags then assume interactive mode. - interactive = FLAGS.interactive or not FLAGS.cmds + interactive = FLAGS.interactive or not FLAGS.cmds tcli_singleton = tcli.TCLI(interactive, FLAGS.cmds) - except (EOFError, tcli.TcliCmdError, - tcli.inventory.AuthError, tcli.inventory.InventoryError, - ValueError) as error_message: + except ( + EOFError, + tcli.TcliCmdError, + tcli.inventory.AuthError, + tcli.inventory.InventoryError, + ValueError, + ) as error_message: print('%s' % error_message, file=sys.stderr) sys.exit(1) if not tcli_singleton.interactive: - del tcli_singleton sys.exit(0) # Interactive prompt, setup Tab completion @@ -75,9 +95,9 @@ def main(_): except KeyboardInterrupt: continue except EOFError: - del tcli_singleton sys.exit(0) if __name__ == '__main__': - app.run(main) + app.run(main, flags_parser=_ParseFlags) + diff --git a/tcli/accessor_base.py b/tcli/accessor_base.py index d87550c..9000b41 100644 --- a/tcli/accessor_base.py +++ b/tcli/accessor_base.py @@ -13,16 +13,21 @@ # permissions and limitations under the License. """Sends commands to devices via callbacks.""" -import typing +from collections.abc import Callable + +# pylint: disable=g-importing-member from tcli.inventory_base import CmdRequest +from tcli.inventory_base import Response DEADLINE = None +# pylint: disable=unused-argument def SendRequests( - requests_callbacks: list[tuple[CmdRequest, typing.Callable]], - deadline: float | None=DEADLINE) -> None: + requests_callbacks: list[tuple[CmdRequest, Callable[[Response], None]]], + deadline: float | None = DEADLINE, +) -> None: """Submits command requests to device manager. Submit the command requests to the device manager for resolution. @@ -34,10 +39,11 @@ def SendRequests( is to be executed by the device manager. Args: - requests_callbacks: List of tuples. - Each tuple pairs a request object with a callback function. + requests_callbacks: List of tuples. Each tuple pairs a request object with a + callback function. deadline: An optional int, the deadline to set when sending the request. + Returns: None """ - raise NotImplementedError \ No newline at end of file + raise NotImplementedError diff --git a/tcli/accessor_canned.py b/tcli/accessor_canned.py index e2ada9a..f95855d 100644 --- a/tcli/accessor_canned.py +++ b/tcli/accessor_canned.py @@ -14,8 +14,8 @@ """Sends commands to devices via callbacks.""" import asyncio +from collections.abc import Callable import os -import typing from tcli import accessor_base as accessor from tcli import inventory_base as inventory @@ -23,13 +23,17 @@ DEADLINE = accessor.DEADLINE ## CHANGEME ## Where we store the canned responses. -DEFAULT_RESPONSE_DIRECTORY = os.path.join( - os.path.dirname(__file__), 'testdata', 'device_output') +DEFAULT_RESPONSE_DIRECTORY = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'testdata', 'device_output') +) def SendRequests( - requests_callbacks:list[tuple[accessor.CmdRequest, typing.Callable]], - deadline: float | None=DEADLINE) -> None: + requests_callbacks: list[ + tuple[accessor.CmdRequest, Callable[[inventory.Response], None]] + ], + deadline: float | None = DEADLINE, +) -> None: """Submits command requests to device manager. Submit the command requests to the device manager for resolution. @@ -41,28 +45,45 @@ def SendRequests( is to be executed by the device manager. Args: - requests_callbacks: List of tuples. - Each tuple pairs a request object with a callback function. + requests_callbacks: List of tuples. Each tuple pairs a request object with a + callback function. deadline: An optional int, the deadline to set when sending the request. + Returns: None """ - - for (request, callback) in requests_callbacks: + del deadline + for request, callback in requests_callbacks: asyncio.run(_ReadCannedResult(request, callback)) + async def _ReadCannedResult( - request: inventory.CmdRequest, callback: typing.Callable) -> None: + request: inventory.CmdRequest, + callback: Callable[[inventory.Response], None], +) -> None: """Reads canned result from local file.""" data, error = '', '' file_name = request.target + '_' + request.command.replace(' ', '_') - file_path = os.path.join(DEFAULT_RESPONSE_DIRECTORY, file_name) + file_path = os.path.abspath( + os.path.join(DEFAULT_RESPONSE_DIRECTORY, file_name) + ) try: - with open(file_path) as fp: + if not file_path.startswith(DEFAULT_RESPONSE_DIRECTORY + os.sep): + raise OSError('Invalid file path outside response directory.') + with open(file_path, encoding='utf-8') as fp: data = fp.read() except IOError: - error = ('Failure to retrieve response from device "%s",' - ' for command "%s".' % (request.target, request.command)) - callback(inventory.Response(uid=request.uid, device_name=request.target, - command=request.command, data=data, error=error)) \ No newline at end of file + error = ( + 'Failure to retrieve response from device "%s", for command "%s".' + % (request.target, request.command) + ) + callback( + inventory.Response( + uid=request.uid, + device_name=request.target, + command=request.command, + data=data, + error=error, + ) + ) diff --git a/tcli/accessor_canned_test.py b/tcli/accessor_canned_test.py index 62f6bed..d324b05 100644 --- a/tcli/accessor_canned_test.py +++ b/tcli/accessor_canned_test.py @@ -12,61 +12,89 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -"""Tests for tcli.accessor_files.""" +"""Unit tests for the canned file accessor module.""" import os import unittest - from unittest import mock from tcli import accessor_canned as accessor from tcli import inventory_base as inventory + class UnitTestCannedFileAccessor(unittest.TestCase): """Test the CSV inventory class.""" def testSendRequests(self): """Tests command requests are pulled from file.""" - - class requestObject(inventory.CmdRequest): + + class RequestObject(inventory.CmdRequest): uid = 0 + def __init__(self, target, command): self.target = target self.command = command - self.uid += requestObject.uid + self.uid += RequestObject.uid # Invalid target or command results in error. target = 'bogus' command = 'show version' - request = requestObject(target, command) + request = RequestObject(target, command) callback = mock.MagicMock() - accessor.SendRequests([(request, callback),]) + accessor.SendRequests([ + (request, callback), + ]) callback.assert_called_with( - inventory.Response( - uid=0, device_name=target, command=command, data='', - error='Failure to retrieve response from device ' - + f'"{target}", for command "{command}".' - ) + inventory.Response( + uid=0, + device_name=target, + command=command, + data='', + error='Failure to retrieve response from device ' + + f'"{target}", for command "{command}".', + ) ) # Result is pulled from file for valid device and command. target = 'device_a' - request = requestObject(target, command) - file_result = os.path.join(accessor.DEFAULT_RESPONSE_DIRECTORY, - f'{target}_{command}'.replace(' ', '_')) + request = RequestObject(target, command) + file_result = os.path.join( + accessor.DEFAULT_RESPONSE_DIRECTORY, + f'{target}_{command}'.replace(' ', '_'), + ) with open(file_result) as fp: data = fp.read() callback = mock.MagicMock() - accessor.SendRequests([(request, callback),]) + accessor.SendRequests([ + (request, callback), + ]) callback.assert_called_with( - inventory.Response( - uid=0, device_name=target, command=command, - data=data, - error='' - ) + inventory.Response( + uid=0, device_name=target, command=command, data=data, error='' + ) ) + # Path traversal attempts are rejected. + traversal_target = '../../etc' + traversal_command = 'passwd' + traversal_request = RequestObject(traversal_target, traversal_command) + callback = mock.MagicMock() + accessor.SendRequests([(traversal_request, callback)]) + callback.assert_called_with( + inventory.Response( + uid=0, + device_name=traversal_target, + command=traversal_command, + data='', + error=( + 'Failure to retrieve response from device ' + f'"{traversal_target}", for command "{traversal_command}".' + ), + ) + ) + + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tcli/command_completer.py b/tcli/command_completer.py index 890f7f4..110db64 100644 --- a/tcli/command_completer.py +++ b/tcli/command_completer.py @@ -14,6 +14,7 @@ """Commandline completion functions for interactive TCLI.""" +import functools import re from tcli import command_parser @@ -21,7 +22,16 @@ _COMPLETER_LIST = [] -def CmdCompleter(full_line: str, state: int, filter_engine) -> str|None: +@functools.cache +def _CompilePattern(pattern: str) -> re.Pattern[str] | None: + """Compiles a regex pattern for command completion, returning None if invalid.""" + try: + return re.compile(pattern) + except re.error: + return None + + +def CmdCompleter(full_line: str, state: int, filter_engine) -> str | None: """Commandline completion used by readline library.""" global _COMPLETER_LIST @@ -31,11 +41,11 @@ def _ScrubToken(token: str) -> str: # This could break legitimate regexps but since these are completion # candidates, That breakage is cosmetic. _reg_exp_str = r'\((?P.*)\)\?' - clean_token = re.sub(_reg_exp_str, '\g', token) # type: ignore + clean_token = re.sub(_reg_exp_str, r'\g', token) # Repeat until all nested (...)? are removed. while token != clean_token: token = clean_token - clean_token = re.sub(_reg_exp_str, '\g', clean_token) # type: ignore + clean_token = re.sub(_reg_exp_str, r'\g', clean_token) return token # First invocation, so build candidate list and cache for re-use. @@ -47,9 +57,9 @@ def _ScrubToken(token: str) -> str: # What has been typed so far. # Collapse quotes to remove any whitespace within. - cleaned_line = re.sub(r'\".+\"', '""', full_line) - cleaned_line = re.sub(r'\'.+\'', '""', cleaned_line) - # Remove double spaces etc + cleaned_line = re.sub(r'"[^"]*"', '""', full_line) + cleaned_line = re.sub(r"'[^']*'", '""', cleaned_line) + # Remove double spaces etc. cleaned_line = re.sub(r'\s+', ' ', cleaned_line) # Are we part way through typing a word, or a word boundary. @@ -68,7 +78,7 @@ def _ScrubToken(token: str) -> str: current_word = line_tokens.pop() # Compare with table of possible command matches in filter_engine. - for row in filter_engine.index.index: # type: ignore + for row in filter_engine.index.index: # Split the command match into tokens. cmd_tokens = row['Command'].split(' ') # Does the line match the partial list of tokens. @@ -76,9 +86,10 @@ def _ScrubToken(token: str) -> str: # Currently a blank line so the first token is what we want. token = cmd_tokens[0] # Does our line match this command completion candidate. - elif ( - len(cmd_tokens) > len(line_tokens) and - re.match(' '.join(cmd_tokens[:len(line_tokens)]), cleaned_line)): + elif len(cmd_tokens) > len(line_tokens): + pattern = _CompilePattern(' '.join(cmd_tokens[: len(line_tokens)])) + if not pattern or not pattern.match(cleaned_line): + continue # Take the token from off the end of the Completer command. token = cmd_tokens[len(line_tokens)] else: @@ -91,14 +102,13 @@ def _ScrubToken(token: str) -> str: _completer_set.add(token) _COMPLETER_LIST = list(_completer_set) _COMPLETER_LIST.sort() - if state < len(_COMPLETER_LIST): return _COMPLETER_LIST[state] return None -def TCLICompleter(full_line: str, state: int, cli_parser) -> str|None: +def TCLICompleter(full_line: str, state: int, cli_parser) -> str | None: """Command line completion for escape commands.""" # Remove the leading slash. @@ -107,8 +117,8 @@ def TCLICompleter(full_line: str, state: int, cli_parser) -> str|None: # We have a complete TCLI command. # Ignore get_completer_delims, we only use spaces as separators. if ' ' in full_line: - word_sep = full_line.index(' ') # Word boundary. - (cmd_name, arg_string) = (full_line[:word_sep], full_line[word_sep +1:]) + word_sep = full_line.index(' ') # Word boundary. + cmd_name, arg_string = (full_line[:word_sep], full_line[word_sep + 1 :]) # If we have a shortname, expand it before continuing. if len(cmd_name) == 1: @@ -130,7 +140,7 @@ def TCLICompleter(full_line: str, state: int, cli_parser) -> str|None: for cmd_name in cli_parser: if cmd_name.startswith(full_line): completer_list.append(cmd_name) - if (cmd_name in cli_parser and cli_parser[cmd_name].append): + if cmd_name in cli_parser and cli_parser[cmd_name].append: # Add the apend option of the command to the list completer_list.append(cmd_name + command_parser.APPEND) completer_list.sort() @@ -138,4 +148,4 @@ def TCLICompleter(full_line: str, state: int, cli_parser) -> str|None: if state < len(completer_list): # Re-apply slash to completion. return command_parser.SLASH + completer_list[state] - return None \ No newline at end of file + return None diff --git a/tcli/command_completer_test.py b/tcli/command_completer_test.py index a066c3d..0680c2f 100644 --- a/tcli/command_completer_test.py +++ b/tcli/command_completer_test.py @@ -14,17 +14,14 @@ """tests command completer functions.""" - import unittest from unittest import mock from absl import flags - +from tcli import command_completer as completer from tcli import tcli_lib as tcli from tcli.tcli_textfsm import clitable -from tcli import command_completer as completer - APPEND = tcli.command_parser.APPEND FLAGS = flags.FLAGS @@ -33,9 +30,11 @@ class UnitTestCompleter(unittest.TestCase): """Tests the TCLI class.""" def setUp(self): - # Instantiate FLAGS global var. - tcli.FLAGS([__file__,]) - return super().setUp() + # Instantiate FLAGS global var. + tcli.FLAGS([ + __file__, + ]) + return super().setUp() def testTCLICompleter(self): """Tests completing TCLI native commands.""" @@ -46,10 +45,12 @@ def testTCLICompleter(self): self.assertEqual(completer.TCLICompleter('/reco', 0, _cp), '/record') self.assertEqual( - completer.TCLICompleter('/reco', 1, _cp), f'/record{APPEND}') + completer.TCLICompleter('/reco', 1, _cp), f'/record{APPEND}' + ) self.assertEqual(completer.TCLICompleter('/reco', 2, _cp), '/recordall') self.assertEqual( - completer.TCLICompleter('/reco', 3, _cp), f'/recordall{APPEND}') + completer.TCLICompleter('/reco', 3, _cp), f'/recordall{APPEND}' + ) self.assertEqual(completer.TCLICompleter('/reco', 4, _cp), '/recordstop') self.assertIsNone(completer.TCLICompleter('/reco', 5, _cp)) # Complete on command arguments. @@ -63,7 +64,8 @@ def testCmdCompleter(self): self.tcli_obj.filter = 'default' clitable.CliTable.INDEX = {} self.tcli_obj.filter_engine = clitable.CliTable( - 'default_index', template_dir=tcli.FLAGS.template_dir) + 'default_index', template_dir=tcli.FLAGS.template_dir + ) _fe = self.tcli_obj.filter_engine # Completions are alphabetical. @@ -89,4 +91,17 @@ def testCmdCompleter(self): self.assertEqual('brief', completer.CmdCompleter('show int01 ', 0, _fe)) self.assertIsNone(completer.CmdCompleter('show int00 ', 1, _fe)) self.assertEqual('.+', completer.CmdCompleter('show int01 brief ', 0, _fe)) - self.assertIsNone(completer.CmdCompleter('show int00 brief yes', 1, _fe)) \ No newline at end of file + self.assertIsNone(completer.CmdCompleter('show int00 brief yes', 1, _fe)) + + # Multiple quoted arguments do not collapse unquoted middle tokens. + fake_fe = mock.MagicMock() + fake_fe.index.index = [ + {'Command': 'cmd ".*" middle ".*" final'}, + {'Command': '[invalid_regex next_token'}, + ] + self.assertEqual( + 'final', + completer.CmdCompleter('cmd "arg 1" middle "arg 2" ', 0, fake_fe), + ) + # Malformed regex in index is safely skipped without raising re.error. + self.assertIsNone(completer.CmdCompleter('[invalid_regex ', 0, fake_fe)) diff --git a/tcli/command_parser.py b/tcli/command_parser.py index ede4790..350e0cc 100644 --- a/tcli/command_parser.py +++ b/tcli/command_parser.py @@ -14,18 +14,17 @@ """Parses TCLI command string and calls handler methods.""" - import re import shlex import typing # Command start with a slash, inline commands a double slash SLASH = '/' -INLINE = SLASH*2 +INLINE = SLASH * 2 # Command suffix for denoting appending values. APPEND = '+' # Indent for help strings. -I = '\n' + ' '*4 +I = '\n' + ' ' * 4 class Error(Exception): @@ -71,12 +70,22 @@ def __init__(self, command_attr): # The command expects a bool and flips the value if unspecified. toggle = property(lambda self: self.attr['toggle']) - def RegisterCommand(self, command_name:str, help_str:str, short_name:str='', - min_args:int=0, max_args:int=1, default_value=None, - append:bool=False, inline:bool=False, raw_arg:bool=False, - regexp:bool=False, toggle:bool=False, - handler:typing.Callable=lambda: None, - completer:typing.Callable=lambda: None) -> None: + def RegisterCommand( + self, + command_name: str, + help_str: str, + short_name: str = '', + min_args: int = 0, + max_args: int = 1, + default_value=None, + append: bool = False, + inline: bool = False, + raw_arg: bool = False, + regexp: bool = False, + toggle: bool = False, + handler: typing.Callable[..., typing.Any] = lambda: None, + completer: typing.Callable[..., typing.Any] = lambda: None, + ) -> None: """Adds support for command to parser. Args: @@ -107,7 +116,7 @@ def RegisterCommand(self, command_name:str, help_str:str, short_name:str='', 'regexp': regexp, 'toggle': toggle, 'handler': handler, - 'completer': completer + 'completer': completer, }) def UnRegisterCommand(self, command_name: str) -> None: @@ -135,7 +144,7 @@ def _CommandExpand(self, line: str) -> tuple[str, str, bool]: and a bool for if the command had an append suffix or not. """ - def _ExpandShortCommand(short_name: str) -> str|None: + def _ExpandShortCommand(short_name: str) -> str | None: """Find full command name for a single command letter.""" for command_name in self: if self[command_name].short_name == short_name: @@ -144,21 +153,20 @@ def _ExpandShortCommand(short_name: str) -> str|None: # Short commands are non-alphabetic or capitalised, # long names are always lowercase. append = False - command_name = None command_name = _ExpandShortCommand(line[0]) # Found short command. if command_name: # Strip short command from line. line = line[1:] # Strip append symbol from input and reattach to command. - if line and line[0] is APPEND: + if line and line[0] == APPEND: command_name += APPEND line = line[1:] else: # Separate command from subsequent arguments. if ' ' in line: command_sep = line.index(' ') - (command_name, line) = (line[:command_sep], line[command_sep :]) + command_name, line = (line[:command_sep], line[command_sep:]) else: # No arguments, just the command. command_name = line @@ -173,16 +181,18 @@ def _ExpandShortCommand(short_name: str) -> str|None: return (command_name, line, append) def ExecHandler( - self, command_name:str, args:list[str], append:bool) -> str|None: + self, command_name: str, args: list[str], append: bool + ) -> str | None: """Execute the handler associated with this command.""" if not self[command_name].handler: - raise ParseError('Unable to exec handler, "%s" has no handler.' % - command_name) + raise ParseError( + 'Unable to exec handler, "%s" has no handler.' % command_name + ) return self[command_name].handler.__call__(command_name, args, append) - def ExecWithDefault(self, command_name: str) -> str|None: + def ExecWithDefault(self, command_name: str) -> str | None: """Executes command handler with the default value. Args: @@ -210,56 +220,58 @@ def ExecWithDefault(self, command_name: str) -> str|None: value = 'off' # Confirm that command expects an argument. if not self[command_name].max_args: - raise ParseError('Unable to set default, "%s" expects no arguments.' % - command_name) + raise ParseError( + 'Unable to set default, "%s" expects no arguments.' % command_name + ) return self.ExecHandler(command_name, [value], False) def ExtractInlineCommands(self, command: str) -> tuple[str, list[str]]: - f"""Separate out inline commmand overrides from command input. + """Separate out inline command overrides from command input. Converts something like: - 'cat alpha | grep abc || grep xyz {INLINE}display csv {INLINE}log bufname' + 'cat alpha | grep abc || grep xyz //display csv //log bufname' Into: command = ['cat alpha | grep ablc || grep xyz'] display = 'csv' log = 'buffername' - A sequence with '{INLINE}' that is not preceded by a space and part of a + A sequence with '//' that is not preceded by a space and part of a valid command is treated as part of the command body and no further matches are made. This determination is done from right to left. e.g. - 'show flash:{INLINE}file_name {INLINE}bogus {INLINE}log filelist' + 'show flash://file_name //bogus //log filelist' Converts into: - ('show flash:{INLINE}file_name {INLINE}bogus', ('log filelist',)) + ('show flash://file_name //bogus', ('log filelist',)) Args: command: str, command issued to target devices. Returns: - List, the command line (minus inline commands) and a tupe of inline the + List, the command line (minus inline commands) and a tuple of inline the TCLI commands extracted. """ token_list = command.split(f' {INLINE}') # Do we need to extract inline commands at all? - if len(token_list) == 1: return (command, []) + if len(token_list) == 1: + return (command, []) - (command_str, token_list) = (token_list[0], token_list[1:]) + command_str, token_list = (token_list[0], token_list[1:]) inline_commands = [] command_suffix = '' index = 0 # Work from right to left. for i in range(len(token_list), 0, -1): index = i - token = token_list[i -1] + token = token_list[i - 1] try: # 'exit' in this context stops any further inline command parsing. if token == 'exit': # Any inline commands to the left are treated as regular input. # Drop 'exit' from the inline commands, it has no other purpose here. - token_list.pop(index -1) + token_list.pop(index - 1) break # Confirm that token parses cleanly. self.ParseCommandLine(token) @@ -267,12 +279,12 @@ def ExtractInlineCommands(self, command: str) -> tuple[str, list[str]]: except (ValueError, ParseError): # If a token doesn't parse then it and all tokens to the left are # returned back to the commandline. - index += 1 # Including the current one. + index += 1 # Including the current one. break - if index > 1: # True if we broke early from the above for loop. + if index > 1: # True if we broke early from the above for loop. # Add back any remaining unparsed tokens to the command string. - command_suffix = f' {INLINE}' + f' {INLINE}'.join(token_list[:index-1]) + command_suffix = f' {INLINE}' + f' {INLINE}'.join(token_list[: index - 1]) return (command_str + command_suffix, inline_commands) @@ -298,16 +310,17 @@ def ExtractPipe(self, command: str) -> tuple[str, str]: # Split out quoted and non-quoted text. unquoted_str = """([^"']+)""" sgl_quoted_str = """('[^']*')""" - dbl_quoted_str = '''("[^"]*")''' + dbl_quoted_str = """("[^"]*")""" token_list = re.findall( - f'{unquoted_str}|{dbl_quoted_str}|{sgl_quoted_str}', command) + f'{unquoted_str}|{dbl_quoted_str}|{sgl_quoted_str}', command + ) pipes = '([|]+)' not_pipes = '([^|]+)' new_token_list = [] # Further decompose the unquoted parts of the string. for index in range(len(token_list)): - (unquoted_str, _, _) = token_list[index] + unquoted_str, _, _ = token_list[index] if unquoted_str: # Split the unquoted part further, by pipes if present. token_sublist = re.findall(f'{pipes}|{not_pipes}', unquoted_str) @@ -323,16 +336,18 @@ def ExtractPipe(self, command: str) -> tuple[str, str]: lhs_str = '' while sgl_pipe in new_token_list: pipe_index = new_token_list.index(sgl_pipe) - lhs_str += ''.join(new_token_list[:pipe_index +1]) - new_token_list = new_token_list[pipe_index +1:] + lhs_str += ''.join(new_token_list[: pipe_index + 1]) + new_token_list = new_token_list[pipe_index + 1 :] if dbl_pipe in new_token_list: pipe_index = new_token_list.index(dbl_pipe) lhs_str += ''.join(new_token_list[:pipe_index]) # Remove 1st instance of double pipe. - new_token_list = new_token_list[pipe_index +1:] + new_token_list = new_token_list[pipe_index + 1 :] # Replace all subsequent instances with a single pipe. - new_token_list = [x if x != dbl_pipe else sgl_pipe for x in new_token_list ] + new_token_list = [ + x if x != dbl_pipe else sgl_pipe for x in new_token_list + ] else: lhs_str += ''.join(new_token_list) new_token_list = [] @@ -376,10 +391,11 @@ def ParseCommandLine(self, line: str) -> tuple[str, list[str], bool]: ParseError: If command not registered, or arguments are invalid. """ - if not line: raise ParseError('Missing escape command.') - + if not line: + raise ParseError('Missing escape command.') + # Expand command into command name, args and it it is append or not. - (command_name, line, append) = self._CommandExpand(line) + command_name, line, append = self._CommandExpand(line) if command_name not in self: raise ParseError(f"Invalid escape command '{command_name}'.") @@ -390,7 +406,8 @@ def ParseCommandLine(self, line: str) -> tuple[str, list[str], bool]: if append and not self[command_name].append: raise ParseError( - f"Command '{command_name}' does not support append mode.") + f"Command '{command_name}' does not support append mode." + ) # Split remaining line into arguments. # Silently discard additional arguments. @@ -398,17 +415,20 @@ def ParseCommandLine(self, line: str) -> tuple[str, list[str], bool]: arguments = shlex.split(line) except ValueError as error_message: raise ParseError( - f"Invalid string could not be parsed into arguments: '{error_message}'") - - if (len(arguments) < self[command_name].min_args or - len(arguments) > self[command_name].max_args): + 'Invalid string could not be parsed into arguments:' + f" '{error_message}'" + ) from error_message + + if ( + len(arguments) < self[command_name].min_args + or len(arguments) > self[command_name].max_args + ): raise ParseError(f'Invalid number of arguments, found: {len(arguments)}.') # Check if a command only expects (Alpha numeric) arguments. - if (not self[command_name].regexp and - not self[command_name].raw_arg): + if not self[command_name].regexp and not self[command_name].raw_arg: for arg in arguments: if re.search(r'\W', arg): raise ParseError('Arguments with alphanumeric characters only.') - return (command_name, arguments, append) \ No newline at end of file + return (command_name, arguments, append) diff --git a/tcli/command_parser_test.py b/tcli/command_parser_test.py index 3bf4475..59abe1c 100644 --- a/tcli/command_parser_test.py +++ b/tcli/command_parser_test.py @@ -12,19 +12,18 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -"""Tests for tcli.command_parser.""" +"""Unit tests for the command_parser module.""" import unittest -from tcli import command_parser -from tcli.command_parser import INLINE +from tcli import command_parser as cp class CommandParserTest(unittest.TestCase): def setUp(self): super(CommandParserTest, self).setUp() - self.cmd_parser = command_parser.CommandParser() + self.cmd_parser = cp.CommandParser() def testRegisterCommand(self): @@ -32,23 +31,54 @@ def testRegisterCommand(self): self.cmd_parser.RegisterCommand('boo', 'A help string.', short_name='B') boo = self.cmd_parser['boo'] boo_dict = { - 'min_args': 0, 'max_args': 1, 'default_value': None, 'append': False, - 'inline': False, 'raw_arg': False, 'regexp': False, 'toggle': False, - 'handler': None} + 'min_args': 0, + 'max_args': 1, + 'default_value': None, + 'append': False, + 'inline': False, + 'raw_arg': False, + 'regexp': False, + 'toggle': False, + 'handler': None, + } # Lots on non-default self.cmd_parser.RegisterCommand( - 'hoo', 'A help string.', short_name='H', min_args=1, - max_args=2, default_value=10, append=True, - inline=True, raw_arg=True, regexp=True, toggle=True) + 'hoo', + 'A help string.', + short_name='H', + min_args=1, + max_args=2, + default_value=10, + append=True, + inline=True, + raw_arg=True, + regexp=True, + toggle=True, + ) hoo = self.cmd_parser['hoo'] hoo_dict = { - 'min_args': 1, 'max_args': 2, 'default_value': 10, 'append': True, - 'inline': True, 'raw_arg': True, 'regexp': True, 'toggle': True, - 'handler': None} - - for attr in ('min_args', 'max_args', 'default_value', 'append', - 'inline', 'raw_arg', 'regexp', 'toggle'): + 'min_args': 1, + 'max_args': 2, + 'default_value': 10, + 'append': True, + 'inline': True, + 'raw_arg': True, + 'regexp': True, + 'toggle': True, + 'handler': None, + } + + for attr in ( + 'min_args', + 'max_args', + 'default_value', + 'append', + 'inline', + 'raw_arg', + 'regexp', + 'toggle', + ): self.assertEqual(boo_dict[attr], getattr(boo, attr)) self.assertEqual(hoo_dict[attr], getattr(hoo, attr)) @@ -57,14 +87,21 @@ def testCommandExpand(self): self.cmd_parser.RegisterCommand('command', 'A help string.', short_name='C') - self.assertEqual(self.cmd_parser._CommandExpand('Cacommand'), - ('command', 'acommand', False)) - self.assertEqual(self.cmd_parser._CommandExpand('C acommand'), - ('command', 'acommand', False)) - self.assertEqual(self.cmd_parser._CommandExpand('ganoncommand'), - ('ganoncommand', '', False)) - self.assertEqual(self.cmd_parser._CommandExpand('C'), - ('command', '', False)) + self.assertEqual( + self.cmd_parser._CommandExpand('Cacommand'), + ('command', 'acommand', False), + ) + self.assertEqual( + self.cmd_parser._CommandExpand('C acommand'), + ('command', 'acommand', False), + ) + self.assertEqual( + self.cmd_parser._CommandExpand('ganoncommand'), + ('ganoncommand', '', False), + ) + self.assertEqual( + self.cmd_parser._CommandExpand('C'), ('command', '', False) + ) def testExecHandler(self): """Tests handler execution.""" @@ -73,8 +110,9 @@ def _Handler(command, args, append): return (command, args, append) self.cmd_parser.RegisterCommand('a', '', handler=_Handler) - self.assertEqual(self.cmd_parser.ExecHandler('a', ['b'], False), - ('a', ['b'], False)) + self.assertEqual( + self.cmd_parser.ExecHandler('a', ['b'], False), ('a', ['b'], False) + ) def testExecWithDefault(self): """Tests handler execution.""" @@ -82,94 +120,108 @@ def testExecWithDefault(self): def _Handler(command, args, append): return (command, args, append) - self.cmd_parser.RegisterCommand('a', '', toggle=True, - default_value=True, handler=_Handler) + self.cmd_parser.RegisterCommand( + 'a', '', toggle=True, default_value=True, handler=_Handler + ) self.assertEqual(self.cmd_parser.ExecWithDefault('a'), ('a', ['on'], False)) def testExtractInlineCmds(self) -> None: """Tests extracting inline commands from right of commandline.""" - def _testSplit(pre_cmd: str, pre_inline: str) -> None: - (post_cmd, post_inline) = self.cmd_parser.ExtractInlineCommands( - pre_cmd + pre_inline) + def _TestSplit(pre_cmd: str, pre_inline: str) -> None: + post_cmd, post_inline = self.cmd_parser.ExtractInlineCommands( + pre_cmd + pre_inline + ) self.assertEqual(pre_cmd, post_cmd) post_inline.insert(0, '') - self.assertEqual(pre_inline, f' {INLINE}'.join(post_inline)) + self.assertEqual(pre_inline, f' {cp.INLINE}'.join(post_inline)) self.cmd_parser.RegisterCommand( - 'log', inline=True, handler=lambda x:x, help_str='') + 'log', inline=True, handler=lambda x: x, help_str='' + ) self.cmd_parser.RegisterCommand( - 'display', short_name='D', inline=True, handler=lambda x:x, help_str='') + 'display', short_name='D', inline=True, handler=lambda x: x, help_str='' + ) self.cmd_parser.RegisterCommand( - 'exit', inline=True, handler=lambda x:x, help_str='') + 'exit', inline=True, handler=lambda x: x, help_str='' + ) # Command with simple inline in short form. cmd = 'cat alpha' - inline = f' {INLINE}D csv' - _testSplit(cmd, inline) + inline = f' {cp.INLINE}D csv' + _TestSplit(cmd, inline) # Reasonably complex command with piping but no inlines. cmd = 'cat alpha | grep abc || grep xyz' inline = '' # Command without inlines returns original command line. - _testSplit(cmd, inline) + _TestSplit(cmd, inline) # Command with invalid inlines are included as command line. - _testSplit(cmd + f' {INLINE}bogus', inline) + _TestSplit(cmd + f' {cp.INLINE}bogus', inline) # The inline must be preceded by a space. - _testSplit(cmd + f'{INLINE}log logfile', inline) - _testSplit(cmd, f' {INLINE}log logfile') + _TestSplit(cmd + f'{cp.INLINE}log logfile', inline) + _TestSplit(cmd, f' {cp.INLINE}log logfile') # Multiple inline commands are supported. - inline = f' {INLINE}' + f' {INLINE}'.join(('display csv', 'log logfile')) - _testSplit(cmd, inline) + inline = f' {cp.INLINE}' + f' {cp.INLINE}'.join( + ('display csv', 'log logfile') + ) + _TestSplit(cmd, inline) # 'exit' inline command will stop further processing of inline commands. - cmd += f' {INLINE}' + 'display csv' - inline = f' {INLINE}' + f' {INLINE}'.join(('exit', 'log logfile')) - (post_cmd, post_inline) = self.cmd_parser.ExtractInlineCommands( - cmd + inline) + cmd += f' {cp.INLINE}' + 'display csv' + inline = f' {cp.INLINE}' + f' {cp.INLINE}'.join(('exit', 'log logfile')) + post_cmd, post_inline = self.cmd_parser.ExtractInlineCommands(cmd + inline) self.assertEqual(cmd, post_cmd) self.assertEqual(post_inline, ['log logfile']) # Invalid inline commands, and commands to the left of it, are assumed to # be part of the regular command. - cmd = 'cat alpha{INLINE}log {INLINE}bogus' - inline = f' {INLINE}log filelist' - _testSplit(cmd, inline) + cmd = 'cat alpha{cp.INLINE}log {cp.INLINE}bogus' + inline = f' {cp.INLINE}log filelist' + _TestSplit(cmd, inline) def testExtractPipe(self) -> None: - """Tests parsing of command pipes.""" - - # A double quited string. - cmd = 'cat "alpha"' - self.assertEqual(self.cmd_parser.ExtractPipe(cmd), ('cat "alpha"', '')) - - # A single pipe. - cmd = 'cat alpha | grep abc' - self.assertEqual(self.cmd_parser.ExtractPipe(cmd), - ('cat alpha | grep abc', '')) - - # A single double pipe. - cmd = 'cat alpha || grep abc' - self.assertEqual(self.cmd_parser.ExtractPipe(cmd), - ('cat alpha', 'grep abc')) - - # A multiple double pipes. - cmd = 'cat alpha | grep abc || grep xyz || grep -v "||"' - self.assertEqual(self.cmd_parser.ExtractPipe(cmd), - ('cat alpha | grep abc', 'grep xyz | grep -v "||"')) - - # Skip qoted double pipe. - cmd = "cat alpha '||' || grep xyz || grep -v . " - self.assertEqual(self.cmd_parser.ExtractPipe(cmd), - ("cat alpha '||'", 'grep xyz | grep -v .')) - - # Double pipe rejected because a single pipe is to its right. - cmd = 'cat alpha || grep xyz | grep -v "||" || echo' - self.assertEqual(self.cmd_parser.ExtractPipe(cmd), - ('cat alpha || grep xyz | grep -v "||"', 'echo')) + """Tests parsing of command pipes.""" + + # A double quited string. + cmd = 'cat "alpha"' + self.assertEqual(self.cmd_parser.ExtractPipe(cmd), ('cat "alpha"', '')) + + # A single pipe. + cmd = 'cat alpha | grep abc' + self.assertEqual( + self.cmd_parser.ExtractPipe(cmd), ('cat alpha | grep abc', '') + ) + + # A single double pipe. + cmd = 'cat alpha || grep abc' + self.assertEqual( + self.cmd_parser.ExtractPipe(cmd), ('cat alpha', 'grep abc') + ) + + # A multiple double pipes. + cmd = 'cat alpha | grep abc || grep xyz || grep -v "||"' + self.assertEqual( + self.cmd_parser.ExtractPipe(cmd), + ('cat alpha | grep abc', 'grep xyz | grep -v "||"'), + ) + + # Skip qoted double pipe. + cmd = "cat alpha '||' || grep xyz || grep -v . " + self.assertEqual( + self.cmd_parser.ExtractPipe(cmd), + ("cat alpha '||'", 'grep xyz | grep -v .'), + ) + + # Double pipe rejected because a single pipe is to its right. + cmd = 'cat alpha || grep xyz | grep -v "||" || echo' + self.assertEqual( + self.cmd_parser.ExtractPipe(cmd), + ('cat alpha || grep xyz | grep -v "||"', 'echo'), + ) def testGetDefault(self): """Tests retrieving default values.""" @@ -187,78 +239,119 @@ def testInlineOnly(self): self.cmd_parser.RegisterCommand('b', '') self.cmd_parser.RegisterCommand('c', '', inline=True) self.cmd_parser.InlineOnly() - self.assertTrue('a' in self.cmd_parser) - self.assertFalse('b' in self.cmd_parser) - self.assertTrue('c' in self.cmd_parser) + self.assertIn('a', self.cmd_parser) + self.assertNotIn('b', self.cmd_parser) + self.assertIn('c', self.cmd_parser) def testParseCommandLine1(self): """Tests parsing a command with default arguments.""" # Vanilla command without appending, inline, regexp or toggle support. self.cmd_parser.RegisterCommand( - 'boo', 'A help string.', short_name='B', min_args=0, - max_args=2, default_value=None, append=False, - inline=False, raw_arg=False, regexp=False, toggle=False) + 'boo', + 'A help string.', + short_name='B', + min_args=0, + max_args=2, + default_value=None, + append=False, + inline=False, + raw_arg=False, + regexp=False, + toggle=False, + ) # Arguments are gathered into a list. - self.assertEqual(self.cmd_parser.ParseCommandLine('boo'), - ('boo', [], False)) - self.assertEqual(self.cmd_parser.ParseCommandLine('boo hoo'), - ('boo', ['hoo'], False)) - self.assertEqual(self.cmd_parser.ParseCommandLine('boo hello world'), - ('boo', ['hello', 'world'], False)) + self.assertEqual( + self.cmd_parser.ParseCommandLine('boo'), ('boo', [], False) + ) + self.assertEqual( + self.cmd_parser.ParseCommandLine('boo hoo'), ('boo', ['hoo'], False) + ) + self.assertEqual( + self.cmd_parser.ParseCommandLine('boo hello world'), + ('boo', ['hello', 'world'], False), + ) # Shortname is expanded. - self.assertEqual(self.cmd_parser.ParseCommandLine('Bhello world'), - ('boo', ['hello', 'world'], False)) + self.assertEqual( + self.cmd_parser.ParseCommandLine('Bhello world'), + ('boo', ['hello', 'world'], False), + ) # Missing command disallowed. - self.assertRaises(command_parser.ParseError, - self.cmd_parser.ParseCommandLine, '') + self.assertRaises(cp.ParseError, self.cmd_parser.ParseCommandLine, '') # Append disallowed. - self.assertRaises(command_parser.ParseError, - self.cmd_parser.ParseCommandLine, 'boo+ hoo') + self.assertRaises( + cp.ParseError, self.cmd_parser.ParseCommandLine, 'boo+ hoo' + ) # Regexp disallowed. - self.assertRaises(command_parser.ParseError, - self.cmd_parser.ParseCommandLine, 'boo ^.*') + self.assertRaises( + cp.ParseError, self.cmd_parser.ParseCommandLine, 'boo ^.*' + ) # Invalid \ escape terminated string. - self.assertRaises(command_parser.ParseError, - self.cmd_parser.ParseCommandLine, 'boo slash ending\\') + self.assertRaises( + cp.ParseError, + self.cmd_parser.ParseCommandLine, + 'boo slash ending\\', + ) # Command that supports append and regexp. self.cmd_parser.RegisterCommand( - 'hoo', 'A help string.', short_name='H', min_args=1, - max_args=2, default_value='on', append=True, - inline=False, raw_arg=False, regexp=True, toggle=False) + 'hoo', + 'A help string.', + short_name='H', + min_args=1, + max_args=2, + default_value='on', + append=True, + inline=False, + raw_arg=False, + regexp=True, + toggle=False, + ) # Quoted text is OK. - self.assertEqual(self.cmd_parser.ParseCommandLine('hoo "A quoted line"'), - ('hoo', ['A quoted line'], False)) + self.assertEqual( + self.cmd_parser.ParseCommandLine('hoo "A quoted line"'), + ('hoo', ['A quoted line'], False), + ) # Regexps are marked OK, quoted blocks are ok, as is append. - self.assertEqual(self.cmd_parser.ParseCommandLine('hoo+ "^.* ?"'), - ('hoo', ['^.* ?'], True)) + self.assertEqual( + self.cmd_parser.ParseCommandLine('hoo+ "^.* ?"'), + ('hoo', ['^.* ?'], True), + ) # Append still works when things are short and cramped. - self.assertEqual(self.cmd_parser.ParseCommandLine('H+hello world'), - ('hoo', ['hello', 'world'], True)) + self.assertEqual( + self.cmd_parser.ParseCommandLine('H+hello world'), + ('hoo', ['hello', 'world'], True), + ) # Minimum arguments triggers exception. - self.assertRaises( - command_parser.ParseError, self.cmd_parser.ParseCommandLine, 'hoo') + self.assertRaises(cp.ParseError, self.cmd_parser.ParseCommandLine, 'hoo') # As does the maximum. self.assertRaises( - command_parser.ParseError, self.cmd_parser.ParseCommandLine, - 'hoo the line is too long') + cp.ParseError, + self.cmd_parser.ParseCommandLine, + 'hoo the line is too long', + ) # Commnad with raw argument. self.cmd_parser.RegisterCommand( - 'foo', 'A help string.', short_name='F', raw_arg=True) + 'foo', 'A help string.', short_name='F', raw_arg=True + ) # Single argument, the remainder of the line. - self.assertEqual(self.cmd_parser.ParseCommandLine('foo hello world...'), - ('foo', ['hello world...'], False)) + self.assertEqual( + self.cmd_parser.ParseCommandLine('foo hello world...'), + ('foo', ['hello world...'], False), + ) # Short command name and regexp string. - self.assertEqual(self.cmd_parser.ParseCommandLine('F^.*$ .?'), - ('foo', ['^.*$ .?'], False)) + self.assertEqual( + self.cmd_parser.ParseCommandLine('F^.*$ .?'), + ('foo', ['^.*$ .?'], False), + ) + if __name__ == '__main__': unittest.main() diff --git a/tcli/command_register.py b/tcli/command_register.py index daebfe9..096cf82 100644 --- a/tcli/command_register.py +++ b/tcli/command_register.py @@ -15,7 +15,6 @@ """Register core TCLI command handlers.""" from absl import flags - from tcli import command_parser # Values for flags that are an enum. @@ -23,43 +22,73 @@ DISPLAY_FORMATS = ('raw', 'csv', 'tbl', 'nvp') MODE_FORMATS = ('cli', 'gated', 'http', 'shell') FLAG_COMMANDS = ( - 'color', 'color_scheme', 'display', 'filter', 'linewrap', 'mode', 'timeout') + 'color', + 'color_scheme', + 'display', + 'filter', + 'linewrap', + 'mode', + 'timeout', +) # Flag help string indentation. -I = '\n' + ' '*4 +I = '\n' + ' ' * 4 # TCLI commands that can be populated at invocation via flags. flags.DEFINE_boolean( - 'color', True, f'{I}Toggle using color when displaying results.') + 'color', True, f'{I}Toggle using color when displaying results.' +) flags.DEFINE_enum( - 'color_scheme', 'light', COLOR_SCHEMES, - f"{I}Use 'light' scheme on dark background or 'dark' otherwise.") + 'color_scheme', + 'light', + COLOR_SCHEMES, + f"{I}Use 'light' scheme on dark background or 'dark' otherwise.", +) flags.DEFINE_enum( - 'display', 'raw', DISPLAY_FORMATS, f""" + 'display', + 'raw', + DISPLAY_FORMATS, + f""" Extensible set of routines used for formatting command output. Available display formats are: {DISPLAY_FORMATS} - Shortname: 'D'.""", short_name='D') + Shortname: 'D'.""", + short_name='D', +) flags.DEFINE_string( - 'filter', 'default_index', """ + 'filter', + 'default_index', + """ Name of index file that contains the map of command output to TextFSM templates. For extracting relevant data from command output for display. - Shortname: 'F'.""", short_name='F') + Shortname: 'F'.""", + short_name='F', +) flags.DEFINE_boolean( - 'linewrap', False, f'{I}Override default line wrap behavior.') + 'linewrap', False, f'{I}Override default line wrap behavior.' +) flags.DEFINE_enum( - 'mode', 'cli', MODE_FORMATS, f""" + 'mode', + 'cli', + MODE_FORMATS, + f""" CLI mode for command. Available display formats are: {MODE_FORMATS} - Shortname: 'M'.""", short_name='M') + Shortname: 'M'.""", + short_name='M', +) flags.DEFINE_integer( - 'timeout', 45, f""" + 'timeout', + 45, + """ Period (in seconds) to wait for outstanding command responses. - Shortname: 'O'.""", short_name='O') + Shortname: 'O'.""", + short_name='O', +) FLAGS = flags.FLAGS @@ -72,186 +101,294 @@ def SetFlagDefaults(cli_parser: command_parser.CommandParser) -> None: # Calling the handlers directly will not be logged. cli_parser.ExecWithDefault(command_name) + def RegisterCommands( - command_object, cli_parser:command_parser.CommandParser) -> None: + command_object, cli_parser: command_parser.CommandParser +) -> None: """Register TCLI CLI commands. - - Attributes: + + Args: command_object: The object that has callable methods for the commands. cli_parser: Object that does the registering. - indent: str, indentation level, after first line of help string. -""" - + """ + # pylint: disable=protected-access cli_parser.RegisterCommand( - 'buffer', - f'{I}Show contents of buffer.', min_args=1, - handler=command_object._CmdBuffer) - + 'buffer', + f'{I}Show contents of buffer.', + min_args=1, + handler=command_object._CmdBuffer, + ) + cli_parser.RegisterCommand( - 'bufferlist', - f'{I}Show buffers currently in use (written to and not cleared).', - max_args=0, handler=command_object._CmdBufferList) - + 'bufferlist', + f'{I}Show buffers currently in use (written to and not cleared).', + max_args=0, + handler=command_object._CmdBufferList, + ) + cli_parser.RegisterCommand( - 'clear', - f'{I}Deletes contents of named buffer.', min_args=1, - handler=command_object._CmdClear) - + 'clear', + f'{I}Deletes contents of named buffer.', + min_args=1, + handler=command_object._CmdClear, + ) + cli_parser.RegisterCommand( - 'color', - FLAGS['color'].help, inline=True, toggle=True, - default_value=FLAGS['color'].value, - handler=command_object._CmdToggleValue, completer=lambda: ['on', 'off']) - + 'color', + FLAGS['color'].help, + inline=True, + toggle=True, + default_value=FLAGS['color'].value, + handler=command_object._CmdToggleValue, + completer=lambda: ['on', 'off'], + ) + cli_parser.RegisterCommand( - 'color_scheme', - FLAGS['color_scheme'].help, inline=True, - default_value=FLAGS['color_scheme'].value, - handler=command_object._CmdColorScheme, completer=lambda: COLOR_SCHEMES) - + 'color_scheme', + FLAGS['color_scheme'].help, + inline=True, + default_value=FLAGS['color_scheme'].value, + handler=command_object._CmdColorScheme, + completer=lambda: COLOR_SCHEMES, + ) + cli_parser.RegisterCommand( - 'command', f""" - Submit command to target device's. Safe mode enforces use of + 'command', + """ + Submit command to target device's. Safe mode enforces use of 'command' for sending input to targets. - Shortname: 'C'.""", short_name='C', min_args=1, raw_arg=True, - handler=command_object._CmdCommand) - + Shortname: 'C'.""", + short_name='C', + min_args=1, + raw_arg=True, + handler=command_object._CmdCommand, + ) + cli_parser.RegisterCommand( - 'defaults', """ + 'defaults', + """ Returns environment to startup/factory defaults. Supply argument to set a specific value back to default, or 'all' to return everything to the defaults.""", - handler=command_object._CmdDefaults) - + handler=command_object._CmdDefaults, + ) + cli_parser.RegisterCommand( - 'display', - FLAGS['display'].help, short_name='D', inline=True, - default_value=FLAGS['display'].value, handler=command_object._CmdDisplay, - completer=lambda: DISPLAY_FORMATS) - + 'display', + FLAGS['display'].help, + short_name='D', + inline=True, + default_value=FLAGS['display'].value, + handler=command_object._CmdDisplay, + completer=lambda: DISPLAY_FORMATS, + ) + cli_parser.RegisterCommand( - 'env', - f'{I}Display current escape command settings.', - max_args=0, handler=command_object._CmdEnv) - + 'env', + f'{I}Display current escape command settings.', + max_args=0, + handler=command_object._CmdEnv, + ) + cli_parser.RegisterCommand( - 'exec', f""" + 'exec', + """ Execute command in shell. - Shortname: '!'.""", short_name='!', min_args=1, raw_arg=True, - handler=command_object._CmdExecShell) - + Shortname: '!'.""", + short_name='!', + min_args=1, + raw_arg=True, + handler=command_object._CmdExecShell, + ) + cli_parser.RegisterCommand( - 'exit', f'{I}Exit tcli.', - inline=True, max_args=0, handler=command_object._CmdExit) - + 'exit', + f'{I}Exit tcli.', + inline=True, + max_args=0, + handler=command_object._CmdExit, + ) + cli_parser.RegisterCommand( - 'expandtargets', """ + 'expandtargets', + """ Displays the expanded list of devices matched by 'targets' and - not matched by 'xtargets'.""", max_args=0, - handler=command_object._CmdExpandTargets) - + not matched by 'xtargets'.""", + max_args=0, + handler=command_object._CmdExpandTargets, + ) + cli_parser.RegisterCommand( - 'filter', FLAGS['filter'].help, - short_name='F', inline=True, - default_value=FLAGS['filter'].value, - handler=command_object._CmdFilter) - + 'filter', + FLAGS['filter'].help, + short_name='F', + inline=True, + default_value=FLAGS['filter'].value, + handler=command_object._CmdFilter, + ) + cli_parser.RegisterCommand( - 'help', f'{I}Display escape command online help.', - max_args=0, inline=True, handler=command_object._CmdHelp) - + 'help', + f'{I}Display escape command online help.', + max_args=0, + inline=True, + handler=command_object._CmdHelp, + ) + cli_parser.RegisterCommand( - 'inventory', """ + 'inventory', + """ Displays attributes of matched targets. - Shortname: 'V'.""", short_name='V', - max_args=0, handler=command_object._CmdInventory) - + Shortname: 'V'.""", + short_name='V', + max_args=0, + handler=command_object._CmdInventory, + ) + cli_parser.RegisterCommand( - 'linewrap', FLAGS['linewrap'].help, - inline=True, toggle=True, - default_value=FLAGS['linewrap'].value, - handler=command_object._CmdToggleValue, completer=lambda: ['on', 'off']) - + 'linewrap', + FLAGS['linewrap'].help, + inline=True, + toggle=True, + default_value=FLAGS['linewrap'].value, + handler=command_object._CmdToggleValue, + completer=lambda: ['on', 'off'], + ) + cli_parser.RegisterCommand( - 'log', """ + 'log', + """ Record commands and device output to buffer. Does not include escape commands or output from these commands.""", - append=True, inline=True, - handler=command_object._CmdLogging) - + append=True, + inline=True, + handler=command_object._CmdLogging, + ) + cli_parser.RegisterCommand( - 'logall', - f'{I}Record both commands and escape commands and output to buffer.', - append=True, inline=True, handler=command_object._CmdLogging) - + 'logall', + f'{I}Record both commands and escape commands and output to buffer.', + append=True, + inline=True, + handler=command_object._CmdLogging, + ) + cli_parser.RegisterCommand( - 'logstop', - f"{I}Stop recording or logging to named buffer (same as 'recordstop'.", - inline=True, min_args=1, handler=command_object._CmdLogStop) - + 'logstop', + f"{I}Stop recording or logging to named buffer (same as 'recordstop'.", + inline=True, + min_args=1, + handler=command_object._CmdLogStop, + ) + cli_parser.RegisterCommand( - 'mode', - FLAGS['mode'].help, short_name='M', inline=True, - default_value=FLAGS['mode'].value, - handler=command_object._CmdMode) - + 'mode', + FLAGS['mode'].help, + short_name='M', + inline=True, + default_value=FLAGS['mode'].value, + handler=command_object._CmdMode, + ) + cli_parser.RegisterCommand( - 'play', """ + 'play', + """ Play out recorded keystrokes from named buffer to target device/s. - Shortname: 'P'.""", short_name='P', min_args=1, - handler=command_object._CmdPlay) - + Shortname: 'P'.""", + short_name='P', + min_args=1, + handler=command_object._CmdPlay, + ) + cli_parser.RegisterCommand( - 'quit', f'{I}Exit by another name.', - inline=True, max_args=0, handler=command_object._CmdExit) - + 'quit', + f'{I}Exit by another name.', + inline=True, + max_args=0, + handler=command_object._CmdExit, + ) + cli_parser.RegisterCommand( - 'read', """ + 'read', + """ Read contents of file and store in buffer. File name is specified at a subsequent prompt.""", - append=True, min_args=1, max_args=2, handler=command_object._CmdRead) - + append=True, + min_args=1, + max_args=2, + handler=command_object._CmdRead, + ) + cli_parser.RegisterCommand( - 'record', f""" + 'record', + f""" Record commands to named . If command is appended with {command_parser.APPEND} then append to buffer.""", - append=True, inline=True, handler=command_object._CmdLogging) - + append=True, + inline=True, + handler=command_object._CmdLogging, + ) + cli_parser.RegisterCommand( - 'recordall', f""" + 'recordall', + f""" Record commands and escape commands to named . If command is appended with {command_parser.APPEND} then append to buffer.""", + append=True, + inline=True, + handler=command_object._CmdLogging, + ) - append=True, inline=True, handler=command_object._CmdLogging) - cli_parser.RegisterCommand( - 'recordstop', - f"{I}Stop recording or logging to named buffer (same as 'logstop').", - inline=True, min_args=1, handler=command_object._CmdLogStop) - + 'recordstop', + f"{I}Stop recording or logging to named buffer (same as 'logstop').", + inline=True, + min_args=1, + handler=command_object._CmdLogStop, + ) + cli_parser.RegisterCommand( - 'safemode', """ + 'safemode', + """ Do not forward input to 'targets' unless using 'command'." - Shortname: 'S'.""", short_name='S', inline=True, toggle=True, - handler=command_object._CmdToggleValue, completer=lambda: ['on', 'off']) - + Shortname: 'S'.""", + short_name='S', + inline=True, + toggle=True, + handler=command_object._CmdToggleValue, + completer=lambda: ['on', 'off'], + ) + cli_parser.RegisterCommand( - 'timeout', FLAGS['timeout'].help, - default_value=FLAGS['timeout'].value, - handler=command_object._CmdTimeout) - + 'timeout', + FLAGS['timeout'].help, + default_value=FLAGS['timeout'].value, + handler=command_object._CmdTimeout, + ) + cli_parser.RegisterCommand( - 'write', """ + 'write', + """ Dumps contents of buffer to file. File name is specified at a subsequent prompt.""", - append=True, min_args=1, max_args=2, handler=command_object._CmdWrite) - - cli_parser.RegisterCommand( - 'verbose', f'{I}Display extra data columns in output (for csv mode).', - inline=True, toggle=True, handler=command_object._CmdToggleValue, - completer=lambda: ['on', 'off']) - + append=True, + min_args=1, + max_args=2, + handler=command_object._CmdWrite, + ) + cli_parser.RegisterCommand( - 'vi', f'{I}Opens buffer in vi editor.', min_args=1, - handler=command_object._CmdEditor) + 'verbose', + f'{I}Display extra data columns in output (for csv mode).', + inline=True, + toggle=True, + handler=command_object._CmdToggleValue, + completer=lambda: ['on', 'off'], + ) + cli_parser.RegisterCommand( + 'vi', + f'{I}Opens buffer in vi editor.', + min_args=1, + handler=command_object._CmdEditor, + ) diff --git a/tcli/command_response.py b/tcli/command_response.py index 1e7426e..d7dc06c 100644 --- a/tcli/command_response.py +++ b/tcli/command_response.py @@ -14,17 +14,17 @@ """Stores response results and current state for device inventory Callback. - Used to collate returned responses prior to outputing to the user. +Used to collate returned responses prior to outputing to the user. """ +from collections.abc import Callable import threading - import tqdm -from absl import logging +from typing import Any +from absl import logging from tcli import inventory_base as inventory - class CmdResponse(object): """Stores response results and current state for device inventory Callback. @@ -52,46 +52,50 @@ class CmdResponse(object): """ # Class will only few (one) instance, so class-wide locking is acceptable. - _lock = threading.Lock() # Lock access to data to support async calls. + _lock = threading.Lock() # Lock access to data to support async calls. - def Synchronized(func): # type: ignore + # pylint: disable=no-self-argument + def Synchronized( + func: Callable[..., Any], + ) -> Callable[..., Any]: """Synchronization decorator.""" - def Wrapper(main_obj, *args, **kwargs): - with main_obj._lock: # pylint: disable=protected-access - return func(main_obj, *args, **kwargs) # type: ignore + def Wrapper(main_obj: 'CmdResponse', *args: Any, **kwargs: Any) -> Any: + with main_obj._lock: # pylint: disable=protected-access + return func(main_obj, *args, **kwargs) + return Wrapper - @Synchronized # type: ignore def __init__(self): """Init starting values.""" - - self._pipe = {} # Client side pipe function. Indexed on row id. - self._results = {} # All response objects indexed by uid. - self._row_index = {} # List of uid indexed by row. - self._row_response = {} # List of responses indexed by row. - self._uid_index = {} # Which row a uid response corresponds to. - self._current_row = 0 # Current row index of requests being returned. - self._response_count = 0 # Total count of received responses. - self.done = threading.Event() - # Graphic to indicate progress receiving responses. - self._progressbar = None - - @Synchronized # type: ignore + self._lock = threading.Lock() + with self._lock: + self._pipe = {} # Client side pipe function. Indexed on row id. + self._results = {} # All response objects indexed by uid. + self._row_index = {} # List of uid indexed by row. + self._row_response = {} # List of responses indexed by row. + self._uid_index = {} # Which row a uid response corresponds to. + self._current_row = 0 # Current row index of requests being returned. + self._response_count = 0 # Total count of received responses. + self.done = threading.Event() + # Graphic to indicate progress receiving responses. + self._progressbar = None + + @Synchronized def InitCommandRow(self, command_row: int, pipe: str) -> None: """Initialise data for a new row, each row corresponds to a command. Args: command_row: int, the row number for the current command. - pipe: str the rhs of the commandline that output is to be piped into. - The pipe is not sent to the target so we track it here. + pipe: str the rhs of the commandline that output is to be piped into. The + pipe is not sent to the target so we track it here. """ self._row_index[command_row] = [] self._row_response[command_row] = [] self._pipe[command_row] = pipe - @Synchronized # type: ignore + @Synchronized def SetRequest(self, command_row: int, request_uid: int) -> None: """Maps uid returned by inventory class to a row number and result. @@ -104,13 +108,13 @@ def SetRequest(self, command_row: int, request_uid: int) -> None: self._uid_index[request_uid] = command_row self._row_index[command_row].append(request_uid) - @Synchronized # type: ignore + @Synchronized def AddResponse(self, response: inventory.Response) -> bool: """Add response to results table. Args: response: obj, returned by inventory class, contains the targets response. - we only care that it has a 'uid' attribute. + we only care that it has a 'uid' attribute. Returns: True if we were expecting a response with this uid @@ -120,8 +124,9 @@ def AddResponse(self, response: inventory.Response) -> bool: if response.uid not in self._uid_index: # Response with an unexpected uid. This can be the case if outstanding # requests are interrupted - Silently discard. - logging.warning("Discarded response: '%s', not expected (stale?)", - response.uid) + logging.warning( + "Discarded response: '%s', not expected (stale?)", response.uid + ) return False # Add to the response object to results table. @@ -137,7 +142,7 @@ def AddResponse(self, response: inventory.Response) -> bool: self._progressbar.update() return True - @Synchronized # type: ignore + @Synchronized def GetRow(self) -> tuple[list[int], str]: """Return current row if fully populated with results. @@ -148,31 +153,39 @@ def GetRow(self) -> tuple[list[int], str]: """ # Have we received all responses for the current row. - if (self._current_row in self._row_response and - len(self._row_response[self._current_row]) == - len(self._row_index[self._current_row])): + if self._current_row in self._row_response and len( + self._row_response[self._current_row] + ) == len(self._row_index[self._current_row]): logging.info( - 'Row %s was complete (size %s) and returned.', - self._current_row, len(self._row_response[self._current_row])) + 'Row %s was complete (size %s) and returned.', + self._current_row, + len(self._row_response[self._current_row]), + ) # Assemble the results for the row and any corresponding pipe content. result = ( - self._row_response[self._current_row], self._pipe[self._current_row]) + self._row_response[self._current_row], + self._pipe[self._current_row], + ) # Advance the current row. self._current_row += 1 - # Reset the progress indicator as there is results to display. - if self._progressbar is not None: - self._progressbar.close() + # Reset the progress indicator when all expected responses have arrived. + if self._progressbar is not None and self._response_count >= len( + self._results + ): + self._progressbar.Finish() + self._progressbar = None # return the row. return result - + # Have we returned the last row of results, if so then we are done? if len(self._results) == self._response_count: logging.debug('GetRow: All results returned.') self.done.set() - + # Either still waiting on entries in current row, or we're done. return ([], '') + @Synchronized def GetResponse(self, uid: int) -> inventory.Response|None: """Returns response object for a given uid.""" @@ -182,6 +195,7 @@ def GetResponse(self, uid: int) -> inventory.Response|None: logging.error( 'Invalid UID: %s, possible values: %s.', uid, str(self._results)) + @Synchronized def StartIndicator(self, message: str = '#! Receiving:') -> None: """Starts a progress indicator to indicate receiving of requests.""" self._progressbar = tqdm.tqdm(list(range(len(self._results))), desc=message) diff --git a/tcli/command_response_test.py b/tcli/command_response_test.py index f2c4125..0456830 100644 --- a/tcli/command_response_test.py +++ b/tcli/command_response_test.py @@ -12,13 +12,13 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -"""Tests for tcli.command_response.""" +"""Unit tests for the command_response module.""" import unittest from unittest import mock -from tcli import inventory_base as inventory from tcli import command_response +from tcli import inventory_base as inventory class FakeLock(object): @@ -28,14 +28,14 @@ class FakeLock(object): conflict = False was_locked = False - def acquire(self): # pylint: disable=g-bad-name + def acquire(self): # pylint: disable=g-bad-name if self.locked: self.conflict = True else: self.locked = True self.was_locked = True - def release(self): # pylint: disable=g-bad-name + def release(self): # pylint: disable=g-bad-name self.locked = False @@ -112,8 +112,10 @@ def testAddResponse(self): """Tests AddResponse method.""" self.assertFalse( - self.cmd_response.AddResponse( - inventory.Response(1, 'device_name', 'command', 'data', 'error'))) + self.cmd_response.AddResponse( + inventory.Response(1, 'device_name', 'command', 'data', 'error') + ) + ) self.cmd_response.InitCommandRow(0, 'boohoo') self.cmd_response.InitCommandRow(1, 'testing') @@ -122,18 +124,26 @@ def testAddResponse(self): self.cmd_response.SetRequest(1, 2) self.assertFalse(self.cmd_response._response_count) - self.assertTrue(self.cmd_response.AddResponse( - inventory.Response(1, 'device_name', 'command', 'data', 'error'))) + self.assertTrue( + self.cmd_response.AddResponse( + inventory.Response(1, 'device_name', 'command', 'data', 'error') + ) + ) self.assertTrue(self.cmd_response._results[1]) self.assertEqual([1], self.cmd_response._row_response[1]) self.assertEqual(1, self.cmd_response._response_count) - self.assertTrue(self.cmd_response.AddResponse( - inventory.Response(2, 'device_name', 'command', 'data', 'error'))) - self.assertFalse(self.cmd_response.AddResponse( - inventory.Response(3, 'device_name', 'command', 'data', 'error'))) + self.assertTrue( + self.cmd_response.AddResponse( + inventory.Response(2, 'device_name', 'command', 'data', 'error') + ) + ) + self.assertFalse( + self.cmd_response.AddResponse( + inventory.Response(3, 'device_name', 'command', 'data', 'error') + ) + ) self.assertEqual(2, self.cmd_response._response_count) - self.assertEqual([1, 2], - self.cmd_response._row_response[1]) + self.assertEqual([1, 2], self.cmd_response._row_response[1]) # pylint: disable=g-generic-assert self.assertEqual(2, len(self.cmd_response._row_index)) @@ -156,13 +166,17 @@ def testGetRow(self): # Responses are added against the request IDs. self.cmd_response.AddResponse( - inventory.Response(1, 'device_name', 'command', 'data', 'error')) + inventory.Response(1, 'device_name', 'command', 'data', 'error') + ) self.cmd_response.AddResponse( - inventory.Response(2, 'device_name', 'command', 'data', 'error')) + inventory.Response(2, 'device_name', 'command', 'data', 'error') + ) self.cmd_response.AddResponse( - inventory.Response(3, 'device_name', 'command', 'data', 'error')) + inventory.Response(3, 'device_name', 'command', 'data', 'error') + ) self.cmd_response.AddResponse( - inventory.Response(4, 'device_name', 'command', 'data', 'error')) + inventory.Response(4, 'device_name', 'command', 'data', 'error') + ) self.assertFalse(self.cmd_response._current_row) self.assertFalse(self.cmd_response.done.is_set()) @@ -176,6 +190,29 @@ def testGetRow(self): self.assertEqual(self.cmd_response._current_row, 2) self.assertTrue(self.cmd_response.done.is_set()) + def testProgressBarMultiRowFinish(self): + """Tests that progress bar only finishes after all rows are received.""" + self.cmd_response.InitCommandRow(0, '') + self.cmd_response.InitCommandRow(1, '') + self.cmd_response.SetRequest(0, 1) + self.cmd_response.SetRequest(1, 2) + mock_pb = mock.MagicMock() + self.cmd_response._progressbar = mock_pb + + # Complete row 0 only. + self.cmd_response.AddResponse( + inventory.Response(1, 'device_name', 'cmd1', 'data', '') + ) + self.assertEqual(([1], ''), self.cmd_response.GetRow()) + mock_pb.Finish.assert_not_called() + + # Complete row 1. + self.cmd_response.AddResponse( + inventory.Response(2, 'device_name', 'cmd2', 'data', '') + ) + self.assertEqual(([2], ''), self.cmd_response.GetRow()) + mock_pb.Finish.assert_called_once() + if __name__ == '__main__': unittest.main() diff --git a/tcli/display.py b/tcli/display.py index b99791f..edf1cf9 100644 --- a/tcli/display.py +++ b/tcli/display.py @@ -17,8 +17,8 @@ # Colour mapping depending on colour scheme. import sys -from textfsm import terminal from tcli import command_register +from textfsm import terminal PROMPT_HDR = '#! <%s[%s]%s> !#' PROMPT_STR = '#! ' @@ -37,8 +37,10 @@ # Display color combinations. COLOR_SCHEMES = command_register.COLOR_SCHEMES + class Display(object): - """ + """Manages CLI output formatting, prompts, and ANSI color schemes. + Attributes: color_scheme: Whether to use 'dark' on light background or vice versa. system_color: Color for system strings. @@ -46,12 +48,13 @@ class Display(object): warning_color: Color for warning strings. """ - def __init__(self) -> None: self.color_scheme: str | None = None self.system_color = self.title_color = self.warning_color = '' - def setColorScheme(self, scheme) -> None: + # pylint: disable=invalid-name + def setColorScheme(self, scheme: str) -> None: + """Sets the active ANSI color scheme.""" if scheme not in COLOR_SCHEMES: raise ValueError(f"Error: Unknown color scheme: '{scheme}'") @@ -74,31 +77,34 @@ def getPrompt(self, targets: str, devices: list[str], safemode: bool) -> str: safe = '*' if safemode else '' - (_, width) = terminal.TerminalSize() + _, width = terminal.TerminalSize() # Expand the targets displayed in prompt, if it fits in the terminal. - if (width > len(PROMPT_HDR % (targets, len(devices), safe))): + if width > len(PROMPT_HDR % (targets, len(devices), safe)): target_str = targets else: # Truncate prompt if too long to fit in terminal. target_str = '#####' return PROMPT_HDR % ( - terminal.AnsiText(target_str, self.system_color), - terminal.AnsiText(len(devices), self.warning_color), - terminal.AnsiText(safe, self.title_color)) + terminal.AnsiText(target_str, self.system_color), + terminal.AnsiText(len(devices), self.warning_color), + terminal.AnsiText(safe, self.title_color), + ) def printOut(self, msg, color, linewrap, msgtype='default') -> None: """Prints (and logs) outputs.""" # Format for width of display. - if linewrap: msg = terminal.LineWrap(msg) + if linewrap: + msg = terminal.LineWrap(msg) # Colourise depending on nature of message. if color: msg_color = f'{msgtype}_color' - if hasattr(self, msg_color) and type(getattr(self, msg_color) is str): - msg = terminal.AnsiText(msg, getattr(self, msg_color)) + color_spec = getattr(self, msg_color, None) + if color_spec: + msg = terminal.AnsiText(msg, color_spec) # Warnings go to stderr. if msgtype == 'warning': print(msg, file=sys.stderr) else: - print(msg) \ No newline at end of file + print(msg) diff --git a/tcli/display_test.py b/tcli/display_test.py index f8560f2..14e58f3 100644 --- a/tcli/display_test.py +++ b/tcli/display_test.py @@ -12,21 +12,23 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -"""Tests for tcli.display.""" +"""Unit tests for the display module.""" +import io import unittest +from unittest import mock from tcli import display +from textfsm import terminal + class UnitTestDisplay(unittest.TestCase): """Tests the TCLI Display class.""" def setUp(self): + super().setUp() self.d = display.Display() - def tearDown(self): - pass - def testColorScheme(self): self.assertIsNone(self.d.color_scheme) # Valid color_scheme accepted @@ -36,5 +38,35 @@ def testColorScheme(self): self.assertEqual('dark', self.d.color_scheme) self.assertRaises(ValueError, self.d.setColorScheme, 'bogus') + def testPrintOutColorHandling(self): + """Tests printOut with and without an active color scheme.""" + with mock.patch('sys.stdout', new_callable=io.StringIO) as mock_stdout: + # Without setColorScheme called, color=True does not wrap in ANSI reset. + self.d.printOut( + 'plain message', color=True, linewrap=False, msgtype='system' + ) + self.assertEqual('plain message\n', mock_stdout.getvalue()) + + self.d.setColorScheme('light') + with mock.patch('sys.stdout', new_callable=io.StringIO) as mock_stdout: + self.d.printOut( + 'colored system', color=True, linewrap=False, msgtype='system' + ) + expected = ( + terminal.AnsiText('colored system', display.LIGHT_SYSTEM_COLOR) + '\n' + ) + self.assertEqual(expected, mock_stdout.getvalue()) + + with mock.patch('sys.stderr', new_callable=io.StringIO) as mock_stderr: + self.d.printOut( + 'colored warning', color=True, linewrap=False, msgtype='warning' + ) + expected = ( + terminal.AnsiText('colored warning', display.LIGHT_WARNING_COLOR) + + '\n' + ) + self.assertEqual(expected, mock_stderr.getvalue()) + + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tcli/end_to_end_test.py b/tcli/end_to_end_test.py index 9a980f1..9e0b523 100644 --- a/tcli/end_to_end_test.py +++ b/tcli/end_to_end_test.py @@ -20,25 +20,21 @@ from unittest import mock from absl import flags - from tcli import tcli_lib as tcli - tcli.inventory = importlib.import_module('tcli.inventory_csv') tcli.accessor = importlib.import_module('tcli.accessor_canned') FLAGS = flags.FLAGS # Test outputs. HEADER = '#!# cat %s #!#' -OUTPUT_A = ('Host, ColAa, ColAb\n' - 'device_a, hello, world\n' - 'device_b, HELLO, WORLD\n') -OUTPUT_B = ('Host, ColBa, ColBb\n' - 'device_a, foo, bar\n' - 'device_b, FOO, BAR\n') +OUTPUT_A = ( + 'Host, ColAa, ColAb\ndevice_a, hello, world\ndevice_b, HELLO, WORLD\n' +) +OUTPUT_B = 'Host, ColBa, ColBb\ndevice_a, foo, bar\ndevice_b, FOO, BAR\n' -#TODO(harro): Expand tests here to make this an automated UAT. +# TODO: Expand tests here to make this an automated UAT. class UnitTestTCLIEndToEnd(unittest.TestCase): @@ -46,21 +42,26 @@ class UnitTestTCLIEndToEnd(unittest.TestCase): @classmethod def setUpClass(cls): super(UnitTestTCLIEndToEnd, cls).setUpClass() - tcli.FLAGS([__file__,]) + tcli.FLAGS([ + __file__, + ]) # Stub out as little as possible. tcli.command_response.tqdm = mock.MagicMock() @classmethod def tearDownClass(cls): + tcli.inventory = importlib.import_module('tcli.inventory_base') super(UnitTestTCLIEndToEnd, cls).tearDownClass() def setUp(self): super(UnitTestTCLIEndToEnd, self).setUp() tcli.FLAGS.template_dir = os.path.join( - os.path.dirname(__file__), 'testdata') + os.path.dirname(__file__), 'testdata' + ) # Read some runtime commands from there. tcli.FLAGS.config_file = os.path.join( - os.path.dirname(__file__), 'testdata', 'rc_file') + os.path.dirname(__file__), 'testdata', 'rc_file' + ) def testSendReceiveCommandInteractive(self): @@ -81,13 +82,15 @@ def testSendReceiveCommandInteractive(self): self.assertListEqual(['device_a', 'device_b'], tcli_obj.device_list) tcli_obj._ParseCommands('cat a') tcli_obj._ParseCommands('/X ^') - self.assertListEqual(['device_a', 'device_b', 'device_c'], - tcli_obj.device_list) + self.assertListEqual( + ['device_a', 'device_b', 'device_c'], tcli_obj.device_list + ) mock_print.assert_has_calls([ mock.call("Invalid escape command 'bogus'.", msgtype='warning'), mock.call(HEADER % 'a', 'title'), - mock.call(OUTPUT_A)]) + mock.call(OUTPUT_A), + ]) def testSendReceiveCommandNonInteractive(self): @@ -97,29 +100,35 @@ def testSendReceiveCommandNonInteractive(self): with mock.patch.object(tcli.TCLI, '_Print') as mock_print: tcli_obj = tcli.TCLI(interactive=False, commands='/display csv\ncat a') - mock_print.assert_has_calls([ - mock.call(HEADER % 'a', 'title'), mock.call(OUTPUT_A)]) + mock_print.assert_has_calls( + [mock.call(HEADER % 'a', 'title'), mock.call(OUTPUT_A)] + ) # RC script ignored. Logging would be on otherwise. self.assertEqual(tcli_obj.log, '') # Commands as an arg of the TCLI object. with mock.patch.object(tcli.TCLI, '_Print') as mock_print: - tcli_obj = tcli.TCLI(interactive=False, - commands='/display csv\ncat a\ncat b') + tcli.TCLI( + interactive=False, commands='/display csv\ncat a\ncat b' + ) mock_print.assert_has_calls([ - mock.call(HEADER % 'a', 'title'), mock.call(OUTPUT_A), - mock.call(HEADER % 'b', 'title'), mock.call(OUTPUT_B)]) + mock.call(HEADER % 'a', 'title'), + mock.call(OUTPUT_A), + mock.call(HEADER % 'b', 'title'), + mock.call(OUTPUT_B), + ]) # With inline TCLI commands. with mock.patch.object(tcli.TCLI, '_Print') as mock_print: - tcli_obj = tcli.TCLI(interactive=False, - commands='/display raw\ncat a //D csv') + tcli.TCLI( + interactive=False, commands='/display raw\ncat a //D csv' + ) - mock_print.assert_has_calls([ - mock.call(HEADER % 'a', 'title'), - mock.call(OUTPUT_A)]) + mock_print.assert_has_calls( + [mock.call(HEADER % 'a', 'title'), mock.call(OUTPUT_A)] + ) if __name__ == '__main__': diff --git a/tcli/inventory_base.py b/tcli/inventory_base.py index 4cbee29..72b299d 100644 --- a/tcli/inventory_base.py +++ b/tcli/inventory_base.py @@ -32,8 +32,7 @@ from absl import flags from absl import logging - -from tcli.command_parser import APPEND, CommandParser +import tcli.command_parser as cp # Global vars so flags.FLAGS has inventroy intelligence in the main program. @@ -53,30 +52,42 @@ # Data format of response. Response = collections.namedtuple( - 'CmdResponse', ['uid', 'device_name', 'command', 'data', 'error']) + 'CmdResponse', ['uid', 'device_name', 'command', 'data', 'error'] +) FLAGS = flags.FLAGS flags.DEFINE_integer( - 'maxtargets', DEFAULT_MAXTARGETS, """ + 'maxtargets', + DEFAULT_MAXTARGETS, + """ High water mark to prevent accidentally making changes to large numbers of - targets. A value of 0, removes the restriction.""") + targets. A value of 0, removes the restriction.""", +) flags.DEFINE_string( - 'targets', '', f""" + 'targets', + '', + f""" Set target devices to receive commands (hostnames or IP addresses, comma separated, no spaces). If a target is prefixed with a '^' then it is treated as a regexp and implicitly ends with '.*$'. Regexp target list is - expanded against devices. If command is appended with {APPEND} then adds to + expanded against devices. If command is appended with {cp.APPEND} then adds to current targets value." A target of '^' resets the list to 'none'. - Shortname: 'T'.""", short_name='T') + Shortname: 'T'.""", + short_name='T', +) flags.DEFINE_string( - 'xtargets', DEFAULT_XTARGETS, f""" + 'xtargets', + DEFAULT_XTARGETS, + f""" Omit targets that match a string or regexp from this list. - If command is appended with {APPEND} then adds to current xtargets value. - Shortname: 'X'.""", short_name='X') + If command is appended with {cp.APPEND} then adds to current xtargets value. + Shortname: 'X'.""", + short_name='X', +) class Error(Exception): @@ -95,12 +106,17 @@ class Attribute(object): """Object for device attributes to be used as cli commands for filtering.""" def __init__( - self, name:str, default_value: str='', valid_values:list[str]=[], - help_str:str='', display_case='lower'): + self, + name: str, + default_value: str = '', + valid_values: list[str] | None = None, + help_str: str = '', + display_case='lower', + ): self._default = default_value self.help_str = help_str self.name = name - self.valid_values = valid_values + self.valid_values = valid_values if valid_values is not None else [] self.display_case = display_case @property @@ -124,21 +140,22 @@ class CmdRequest(object): Some devices support multiple commandline modes or CLI interpretors e.g. router CLI and sub-system unix cli. - Args: - target: device to send command to. - command: string single commandline to send to each device. - mode: commandline mode on device to submit the command to. - Returns: - List of request objects. + Attributes: + uid: Unique identifier for the request. + target: Device to send command to. + command: String single commandline to send to each device. + mode: Commandline mode on device to submit the command to. """ - UID = 0 # Each request has an identifier + UID = 0 # Each request has an identifier. + _uid_lock = threading.Lock() def __init__(self, target: str, command: str, mode: str = 'cli') -> None: - CmdRequest.UID += 1 - self.uid = CmdRequest.UID + with CmdRequest._uid_lock: + CmdRequest.UID += 1 + self.uid = CmdRequest.UID self.target = target - self.command = bytes(command, 'utf-8').decode('unicode_escape') + self.command = command self.mode = mode @@ -149,16 +166,20 @@ class Inventory(object): Source specific instantiations of the class should replace private methods. Attributes: + batch: Bool indicating that the inventory is being used non-interactively. devices: A dict (keyed by string) of Device objects. Read-only. device_list: Array of device names satisfying the filters. source: String identifier of source of device inventory. """ + SOURCE = 'unknown' + batch: bool = False - def __init__(self): + def __init__(self, batch: bool = False) -> None: """Initialise thread safe access to data to support async calls.""" - self._getter_lock = threading.Lock() + self.batch: bool = batch + self._getter_lock = threading.RLock() self._load_lock = threading.Lock() self._loaded = threading.Event() self._loaded.set() @@ -172,26 +193,44 @@ def __init__(self): self._attributes: dict[str, Attribute] = {} # Filters and exclusions added by this library. logging.debug( - f'Device attributes globally defined: "{DEVICE_ATTRIBUTES}".') + 'Device attributes globally defined: "%s".', DEVICE_ATTRIBUTES + ) - #TODO(#35): Compare against reserved words and raise exception if a dup. + # TODO(#35): Compare against reserved words and raise exception if a dup. for attr in DEVICE_ATTRIBUTES: # TODO(#36): Add support for filtering on flag values. - if attr == 'flags': continue + if attr == 'flags': + continue self._inclusions[attr] = '' self._exclusions['x' + attr] = '' self._attributes[attr] = DEVICE_ATTRIBUTES[attr] - # Devices keyed on device name. + # Devices keyed on device name. # If we have already loaded devices e.g. copy, don't do it again. if not hasattr(self, '_devices'): self._devices = {} - # Load device inventory from external source. - self.Load() + if not batch: + # Load device inventory from external source. + self.Load() # List of device names. - self._device_list: list[str]|None = None + self._device_list: list[str] | None = None + + def __copy__(self) -> 'Inventory': + """Returns a shallow copy of the inventory with independent filters.""" + inv_copy = type(self).__new__(type(self)) + inv_copy.__dict__.update(self.__dict__) + inv_copy._getter_lock = threading.RLock() + inv_copy._inclusions = self._inclusions.copy() + inv_copy._exclusions = self._exclusions.copy() + inv_copy._filters = self._filters.copy() + inv_copy._attributes = self._attributes.copy() + inv_copy._device_list = ( + self._device_list[:] if self._device_list is not None else None + ) + return inv_copy @property def attributes(self) -> dict[str, Attribute]: + """Returns mapping of device attribute names to Attribute objects.""" return self._attributes @property @@ -206,13 +245,19 @@ def devices(self) -> dict[str, typing.NamedTuple]: ... ), ... } + + Raises: + InventoryError: If device inventory data failed to load or is empty. """ with self._getter_lock: + if self.batch and not self._devices: + self.Load() # Wait for any pending device loading. self._loaded.wait() if not self._devices: raise InventoryError( - 'Device inventory data failed to load or no devices found.') + 'Device inventory data failed to load or no devices found.' + ) return self._devices @property @@ -226,17 +271,20 @@ def device_list(self) -> list[str]: if self._device_list is None: # If still None, then there was an error. raise InventoryError( - 'Device inventory data failed to load or no devices found.') + 'Device inventory data failed to load or no devices found.' + ) return self._device_list @property def inclusions(self) -> dict[str, str]: + """Returns mapping of inclusion filter names to filter strings.""" return self._inclusions @property def exclusions(self) -> dict[str, str]: + """Returns mapping of exclusion filter names to filter strings.""" return self._exclusions - + # pylint: disable=protected-access targets = property(lambda self: self._inclusions['targets']) @@ -249,45 +297,75 @@ def Load(self) -> None: self._loaded.clear() # Collect result in a thread so it can complete in the background. # Threading here is helpful if device inventory is large. - self._devices_thread = threading.Thread(name='Device loader', - target=self._AsyncLoad, - daemon=True) + self._devices_thread = threading.Thread( + name='Device loader', target=self._AsyncLoad, daemon=True + ) self._devices_thread.start() - def RegisterCommands(self, cmd_register: CommandParser) -> None: + def RegisterCommands(self, cmd_register: cp.CommandParser) -> None: """Add module specific command support to TCLI.""" # Register commands common to any inventory source. - cmd_register.RegisterCommand('targets', FLAGS['targets'].help, - append=True, regexp=True, inline=True, - short_name='T', default_value=FLAGS.targets, - handler=self._CmdFilter) - cmd_register.RegisterCommand('xtargets', FLAGS['xtargets'].help, - append=True, regexp=True, inline=True, - short_name='X', default_value=FLAGS.xtargets, - handler=self._CmdFilter) - cmd_register.RegisterCommand('maxtargets', FLAGS['maxtargets'].help, - default_value=FLAGS.maxtargets, - handler=self._CmdMaxTargets) - - cmd_register.RegisterCommand('attributes', f""" + cmd_register.RegisterCommand( + 'targets', + FLAGS['targets'].help, + append=True, + regexp=True, + inline=True, + short_name='T', + default_value=FLAGS.targets, + handler=self._CmdFilter, + ) + cmd_register.RegisterCommand( + 'xtargets', + FLAGS['xtargets'].help, + append=True, + regexp=True, + inline=True, + short_name='X', + default_value=FLAGS.xtargets, + handler=self._CmdFilter, + ) + cmd_register.RegisterCommand( + 'maxtargets', + FLAGS['maxtargets'].help, + default_value=FLAGS.maxtargets, + handler=self._CmdMaxTargets, + ) + + cmd_register.RegisterCommand( + 'attributes', + f""" Filter targets based on an attribute of the device. First argument is attribute name, the second is the value. If a attribute value is prefixed with a '^' then it is treated as a regexp and implicitly ends with '.*$'. - If command is appended with {APPEND} then adds to current attribute value. + If command is appended with {cp.APPEND} then adds to current attribute value. A value of '^' resets the attribute filter to 'none'. - Shortname: 'A'.""", short_name='A', - max_args=2, append=True, regexp=True, inline=True, - handler=self._AttributeFilter, completer=self._CmdFilterCompleter) + Shortname: 'A'.""", + short_name='A', + max_args=2, + append=True, + regexp=True, + inline=True, + handler=self._AttributeFilter, + completer=self._CmdFilterCompleter, + ) cmd_register.RegisterCommand( - 'xattributes', f""" + 'xattributes', + f""" Omit targets that have the attribute of the same name as the first argument that matches a string or regexp from second argument. - If command is appended with {APPEND} then adds to current xattributes value. - Shortname: 'E'.""", short_name='E', - max_args=2, append=True, regexp=True, inline=True, - handler=self._AttributeFilter, completer=self._CmdFilterCompleter) + If command is appended with {cp.APPEND} then adds to current xattributes value. + Shortname: 'E'.""", + short_name='E', + max_args=2, + append=True, + regexp=True, + inline=True, + handler=self._AttributeFilter, + completer=self._CmdFilterCompleter, + ) # Register commands specific to this inventory source. for attribute in self._attributes.values(): @@ -295,29 +373,38 @@ def RegisterCommands(self, cmd_register: CommandParser) -> None: default_value = getattr(FLAGS, attribute.name) else: default_value = attribute.default_value - cmd_register.RegisterCommand(attribute.name, - attribute.help_str, - default_value=default_value, - append=True, inline=True, regexp=True, - handler=self._CmdFilter) - cmd_register.RegisterCommand('x' + attribute.name, - attribute.help_str, default_value='', - append=True, inline=True, regexp=True, - handler=self._CmdFilter) - - def SetFiltersFromDefaults(self, cmd_register: CommandParser) -> None: + cmd_register.RegisterCommand( + attribute.name, + attribute.help_str, + default_value=default_value, + append=True, + inline=True, + regexp=True, + handler=self._CmdFilter, + ) + cmd_register.RegisterCommand( + 'x' + attribute.name, + attribute.help_str, + default_value='', + append=True, + inline=True, + regexp=True, + handler=self._CmdFilter, + ) + + def SetFiltersFromDefaults(self, cmd_register: cp.CommandParser) -> None: """Set/Reset filters to default values.""" for attr in self.inclusions: cmd_register.ExecWithDefault(attr) - + for attr in self.exclusions: cmd_register.ExecWithDefault(attr) def ShowEnv(self) -> str: """Show inventory attribute filter settings.""" - indent = ' '*2 + indent = ' ' * 2 # Add headline to indicate this display section is from this module. display_string = ['Inventory:'] display_string.append(f'{indent}Max Targets: {self._maxtargets}') @@ -327,17 +414,18 @@ def ShowEnv(self) -> str: for incl, excl in zip(sorted(self._inclusions), sorted(self._exclusions)): # Create paired entries like 'Targets: ..., XTargets: ...' display_string.append( - f'{indent*2}' + - self._FormatLabelAndValue(incl, self._inclusions[incl]) - + ', ' - + self._FormatLabelAndValue(excl, self._exclusions[excl], caps=2) - ) + f'{indent*2}' + + self._FormatLabelAndValue(incl, self._inclusions[incl]) + + ', ' + + self._FormatLabelAndValue(excl, self._exclusions[excl], caps=2) + ) return '\n'.join(display_string) + '\n' # Command handlers have identical arguments. def _CmdFilterCompleter( - self, word_list:list[str], state:int) -> list[str]|None: + self, word_list: list[str], state: int + ) -> list[str] | None: """Returns a command completion list for valid attribute completions.""" # Only complete on a single word, the attribute name. @@ -361,12 +449,15 @@ def _CmdFilterCompleter( return None def _AttributeFilter( - self, command_name:str, args:list[str], append:bool=False) -> str: + self, command_name: str, args: list[str], append: bool = False + ) -> str: """Updates or displays the inventory inclusions or exclusions (filters). Args: command_name: 'attributes' or 'xattributes'. args: list of positional args. + append: bool indicating that filter args are to be appended. + Returns: String to display. Raises: @@ -375,13 +466,13 @@ def _AttributeFilter( if command_name not in ('attributes', 'xattributes'): raise ValueError(f'Command "{command_name}" invalid.') - + # Display values of all device attribute filters. if not args: result = '' if command_name == 'attributes': attr_list = self._inclusions - else: # xattributes + else: # xattributes attr_list = self._exclusions for attr in attr_list: result += self._CmdFilter(attr, [], append) @@ -389,79 +480,97 @@ def _AttributeFilter( # Update attribute filter/s. if command_name == 'attributes': - self._CmdFilter(args[0], args[1 :], append) + self._CmdFilter(args[0], args[1:], append) else: # 'xattributes' so add 'x' prefix to corresponding attribute. - self._CmdFilter('x' + args[0], args[1 :], append) + self._CmdFilter('x' + args[0], args[1:], append) return '' - + def _CmdFilter( - self, command_name:str, args:list[str], append:bool=False) -> str: + self, command_name: str, args: list[str], append: bool = False + ) -> str: """Updates or displays target device inventory filter. Args: command_name: Command entered by the user (minus slash prefix). args: list of positional args after the command. append: bool indicating that filters args are to be appended. + Returns: String to display. Raises: ValueError: If called on unknown attribute. """ - if (command_name not in self._inclusions and - command_name not in self._exclusions): + if ( + command_name not in self._inclusions + and command_name not in self._exclusions + ): raise ValueError(f'Command "{command_name}" invalid.') # Filter may be inclusive, or exclusive. if command_name in self._inclusions: filters = self._inclusions - caps = 1 # Capitalise the first character. + caps = 1 # Capitalise the first character. else: filters = self._exclusions - caps = 2 # Capitalise the two characters. + caps = 2 # Capitalise the two characters. # No args, so display current value/s. if not args: return self._FormatLabelAndValue( - command_name, filters[command_name], caps=caps) + command_name, filters[command_name], caps=caps + ) - filter_string = args[0] # TODO(harro): Raise exception is args >1 ? + filter_string = args[0] # TODO: Raise exception is args >1 ? if filter_string in ('^', '^$'): filters[command_name] = '' - # Clear device list to trigger re-application of filter. - self._BuildDeviceList() + self._filters.pop(command_name, None) + if self.batch: + self._devices = {} + self._device_list = None + else: + # Clear device list to trigger re-application of filter. + self._BuildDeviceList() return '' # Appending a new filter string to an existing filter. if append and filter_string and filters[command_name]: filter_string = ','.join([filters[command_name], filter_string]) - #TODO(harro): Pass in ignorecase flag, add to class __ini__. + # TODO: Pass in ignorecase flag, add to class __ini__. _filter = FilterMatch(filter_string) if not self.ValidFilter(command_name, _filter.filters[0]): raise ValueError( - f'Non-regexp filter entry "{_filter.filters[0]}" is not valid.') + f'Non-regexp filter entry "{_filter.filters[0]}" is not valid.' + ) self._filters[command_name] = _filter filters[command_name] = filter_string - # Clear device list to trigger re-application of filter. - self._BuildDeviceList() + if self.batch: + self._devices = {} + self._device_list = None + else: + # Clear device list to trigger re-application of filter. + self._BuildDeviceList() return '' def _CmdMaxTargets( - self, command_name:str, args:list[str], append=False) -> str: + self, command_name: str, args: list[str], append: bool = False + ) -> str: """Updates or displays maxtargets filter. Args: command_name: str command name entered by user (minus slash prefix). args: list of positional args. append: bool indicating that command had a suffix to append data. + Returns: String to display Raises: ValueError: If argument cannot be interpreted as a cardinal. Or is less than the current device list. """ + del append if not args: return self._FormatLabelAndValue(command_name, str(self._maxtargets)) @@ -469,13 +578,18 @@ def _CmdMaxTargets( maxtargets = int(args[0]) if maxtargets < 0: raise ValueError - except ValueError: - raise ValueError(f"Max Targets is a non-cardinal value: '{args[0]}.'") + except ValueError as exc: + raise ValueError( + f"Max Targets is a non-cardinal value: '{args[0]}.'" + ) from exc self._maxtargets = maxtargets return '' - def _Flatten(self, container: list|tuple|set) -> typing.Iterator[str]: + def _Flatten( + self, + container: list[typing.Any] | tuple[typing.Any, ...] | set[typing.Any], + ) -> typing.Iterator[str]: """Flattens arbitrarily deeply nested lists.""" for i in container: @@ -494,33 +608,40 @@ def ValidFilter(self, filter_name: str, literals: list[str]) -> bool: Args: filter_name: str filter or exclusion name. literals: list of literals in the filter. + Raises: ValueError: If literal device name specified and device is unknown. Returns: Bool The filter string 'arg'. """ - #TODO(#37): Add support for regexp validation. - if not literals: return True + # TODO(#37): Add support for regexp validation. + if not literals: + return True attribute = filter_name # Trim off the 'x' prefix for attribute name. if filter_name.startswith('x'): - attribute = filter_name[1 :] + attribute = filter_name[1:] - validate_list: list|set = [] + validate_list: list[typing.Any] | set[typing.Any] = [] if attribute == 'targets': + if self.batch: + return True # Warn user if literal is unknown. validate_list = list(self.devices) elif attribute in self._attributes: if self._attributes[attribute].valid_values: - validate_list = self._attributes[attribute].valid_values # type: ignore + validate_list = self._attributes[attribute].valid_values + elif self.batch: + return True else: # Without a specific list of valid values, check that at least one # device matches. - # TODO(harro): For filter responsiveness reasons we may drop this. + # TODO: For filter responsiveness reasons we may drop this. validate_list = [ - getattr(dev, attribute, None) for dev in self.devices.values()] + getattr(dev, attribute, None) for dev in self.devices.values() + ] validate_list = set(self._Flatten(validate_list)) validate_list = [value.lower() for value in validate_list] @@ -534,19 +655,23 @@ def _FormatLabelAndValue(self, label: str, value: str, caps: int = 1) -> str: caps = min(caps, len(label)) # Capitalise the prefix. - label = label[:caps].upper() + label[caps :] + label = label[:caps].upper() + label[caps:] return f'{label}: {value}' - #TODO(harro): If we flip the exclude/include logic, is this cleaner? - def _FilterMatch(self, devicename:str, device_attrs:typing.NamedTuple, - exclude:bool=False) -> bool: + # TODO: If we flip the exclude/include logic, is this cleaner? + def _FilterMatch( + self, + devicename: str, + device_attrs: typing.NamedTuple, + exclude: bool = False, + ) -> bool: """Returns true if device matches the inclusion/exclusion filter.""" - filter = self._exclusions if exclude else self._inclusions + active_filter = self._exclusions if exclude else self._inclusions prefix = 'x' if exclude else '' - for attr in filter: + for attr in active_filter: # Blank filters are ignored. - if not filter[attr]: + if not active_filter[attr]: continue # For xtargets we match on device name as the attributes value. @@ -554,7 +679,7 @@ def _FilterMatch(self, devicename:str, device_attrs:typing.NamedTuple, attr_value = devicename else: # Strip 'x' attribute prefix if an exclusion. - stripped_attr = attr[1 :] if exclude else attr + stripped_attr = attr[1:] if exclude else attr attr_value = getattr(device_attrs, stripped_attr, None) # Devices without this attribute are ignored. if not attr_value: @@ -595,7 +720,7 @@ def _BuildDeviceList(self) -> list[str]: return self._device_list d_list = [] - for (devicename, d) in self.devices.items(): + for devicename, d in self.devices.items(): # Skip devices that match any non-blank exclusions. if self._FilterMatch(devicename, d, exclude=True): continue @@ -607,11 +732,12 @@ def _BuildDeviceList(self) -> list[str]: # Raise error if number of matches exceeds the maximum set by user. if self._maxtargets and len(d_list) > self._maxtargets: raise ValueError( - f'Target list exceeded Maximum targets limit of: {self._maxtargets}.') + f'Target list exceeded Maximum targets limit of: {self._maxtargets}.' + ) # Cache the result. self._device_list = d_list - logging.debug(f'Device List length: {len(self._device_list)}') + logging.debug('Device List length: %s', len(self._device_list)) return self._device_list def _AsyncLoad(self) -> None: @@ -626,7 +752,7 @@ def _AsyncLoad(self) -> None: self._loaded.set() def _FetchDevices(self) -> None | NotImplementedError: - """Fetches Devices from external store .""" + """Fetches Devices from external store.""" raise NotImplementedError @@ -638,7 +764,7 @@ def __init__(self, filter_string: str, ignorecase: bool = True) -> None: self._Set(filter_string, ignorecase) @property - def filters(self) -> tuple: + def filters(self) -> tuple[list[str], list[re.Pattern[str]]]: return (self._literals, self._compiled) def _Set(self, filter_string: str, ignore_case: bool) -> None: @@ -649,19 +775,23 @@ def _Set(self, filter_string: str, ignore_case: bool) -> None: Args: filter_string: str to use as the basis of the filters. + ignore_case: bool to canonicalize to lowercase or ignore case in regexps. """ # Split the string into literal and regexp elements. - (self._literals, self._compiled) = self._DecomposeString( - filter_string, ignore_case) + self._literals, self._compiled = self._DecomposeString( + filter_string, ignore_case + ) def _DecomposeString( - self, filter_string:str, ignore_case:bool) -> tuple: + self, filter_string: str, ignore_case: bool + ) -> tuple[list[str], list[re.Pattern[str]]]: """Returns a tuple of lists of compiled and literal strings for matching. Args: filter_string: str comma separated substrings to use for filtering. ignore_case: bool to cononalise to lowercase or regexp ignores case. + Raises: ValueError if a substring is indicated as a regexp but is not valid. Returns: @@ -686,8 +816,10 @@ def _DecomposeString( re_substrs.append(re.compile(substring, re.IGNORECASE)) else: re_substrs.append(re.compile(substring)) - except re.error: - raise ValueError(f'The filter regexp "{substring}" is invalid.') + except re.error as exc: + raise ValueError( + f'The filter regexp "{substring}" is invalid.' + ) from exc else: if ignore_case: # Canonalise to all lowercase. @@ -697,7 +829,7 @@ def _DecomposeString( return (literal_substrs, re_substrs) - def Match(self, value: str|list[str]|list[list[str]]) -> bool: + def Match(self, value: str | list[str] | list[list[str]]) -> bool: """Returns if a value matches the filter.""" # If we have a list of attributes, recurse down to find match. @@ -709,7 +841,7 @@ def Match(self, value: str|list[str]|list[list[str]]) -> bool: return False # Is there a literal for this value? - if (self._literals and value in self._literals): + if self._literals and value in self._literals: return True # Regular expressions are held separately as compiled expressions. @@ -719,3 +851,4 @@ def Match(self, value: str|list[str]|list[list[str]]) -> bool: return True return False + diff --git a/tcli/inventory_base_test.py b/tcli/inventory_base_test.py index f8826b6..6361707 100644 --- a/tcli/inventory_base_test.py +++ b/tcli/inventory_base_test.py @@ -12,10 +12,11 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -"""Tests for tcli.inventory_base.""" +"""Unit tests for the inventory_base module.""" import collections import unittest +from unittest import mock from tcli import inventory_base @@ -24,23 +25,35 @@ class InventoryBaseTest(unittest.TestCase): def _ClearFilters(self, inv_obj): """Clear all filters for targets etc.""" - for x in inv_obj._inclusions: inv_obj._inclusions[x] = '' - for y in inv_obj._exclusions: inv_obj._exclusions[y] = '' + for x in inv_obj._inclusions: + inv_obj._inclusions[x] = '' + for y in inv_obj._exclusions: + inv_obj._exclusions[y] = '' @classmethod def setUpClass(cls): super(InventoryBaseTest, cls).setUpClass() - inventory_base.FLAGS([__file__,]) + inventory_base.FLAGS([ + __file__, + ]) + def _FetchDevicesStub(self): # Fake inventory data, same columns as the CSV file. _d = collections.namedtuple('Device', ('pop', 'realm', 'vendor', 'flags')) self._devices = { - 'device01': _d(vendor='juniper', realm='prod', pop='abc01', flags=['f1']), - 'device02': _d(vendor='cisco', realm='prod', pop='xyz01', flags=['f1', 'f2']), - 'device03': _d(vendor='juniper', realm='lab', pop='abc01', flags=[]), - 'device04': _d(vendor='juniper', realm='lab', pop='abc02', - flags=['f1', 'f2'])} + 'device01': _d( + vendor='juniper', realm='prod', pop='abc01', flags=['f1'] + ), + 'device02': _d( + vendor='cisco', realm='prod', pop='xyz01', flags=['f1', 'f2'] + ), + 'device03': _d(vendor='juniper', realm='lab', pop='abc01', flags=[]), + 'device04': _d( + vendor='juniper', realm='lab', pop='abc02', flags=['f1', 'f2'] + ), + } return + inventory_base.Inventory._FetchDevices = _FetchDevicesStub @classmethod @@ -50,16 +63,35 @@ def tearDownClass(cls): def setUp(self): super(InventoryBaseTest, self).setUp() self.dev_inv_orig = inventory_base.DEVICE_ATTRIBUTES + inventory_base.DEVICE_ATTRIBUTES = { + 'pop': inventory_base.Attribute( + 'pop', '', [], '\n Limit device lists to specific pop/s' + ), + 'realm': inventory_base.Attribute( + 'realm', + 'lab', + ['prod', 'lab'], + '\n Limit the device list to a specific realm/s.', + ), + 'vendor': inventory_base.Attribute( + 'vendor', + '', + ['cisco', 'juniper'], + '\n Limit device lists to a specific vendor/s', + display_case='title', + ), + } self.inv = inventory_base.Inventory() self._ClearFilters(self.inv) def tearDown(self): + super().tearDown() inventory_base.DEVICE_ATTRIBUTES = self.dev_inv_orig ############################################################################ # Thread safe public methods and properties. # ############################################################################ - + def testCmdRequestPresentation(self): """Test building commands requests to send to device connection service.""" @@ -73,12 +105,18 @@ def testCmdRequestPresentation(self): self.assertEqual('device02', request.target) self.assertEqual('shell', request.mode) + # Non-ASCII UTF-8 and raw backslashes in commands are preserved intact. + utf8_cmd = r'show run | include \x00 café \u123' + request = inventory_base.CmdRequest('device01', utf8_cmd, 'cli') + self.assertEqual(utf8_cmd, request.command) + def testGetDevices(self): """Test building dict of Device objects.""" self.assertListEqual( - list(self.inv.devices.keys()), - ['device01', 'device02', 'device03', 'device04']) + list(self.inv.devices.keys()), + ['device01', 'device02', 'device03', 'device04'], + ) def testGetDeviceList(self): """Tests returning a list of Device names.""" @@ -95,33 +133,39 @@ def testRegisterCommands(self): pass def testShowEnv(self): - self.assertEqual(self.inv.ShowEnv(), - ('Inventory:\n' - ' Max Targets: 50\n' - ' Filters:\n' - ' Pop: , XPop: \n' - ' Realm: , XRealm: \n' - ' Targets: , XTargets: \n' - ' Vendor: , XVendor: \n')) - + self.assertEqual( + self.inv.ShowEnv(), + ( + 'Inventory:\n' + ' Max Targets: 50\n' + ' Filters:\n' + ' Pop: , XPop: \n' + ' Realm: , XRealm: \n' + ' Targets: , XTargets: \n' + ' Vendor: , XVendor: \n' + ), + ) + ############################################################################ # Methods related to registering/executing TCLI CLI command extensions. # ############################################################################ - + def testCmdFilterCompleter(self): """Test command completion list for valid attribute completions.""" # Completer matches against the global device attributes. - self.inv._attributes = {'apple': inventory_base.Attribute('apple'), - 'pear': inventory_base.Attribute('pear')} + self.inv._attributes = { + 'apple': inventory_base.Attribute('apple'), + 'pear': inventory_base.Attribute('pear'), + } # A blank word will match all entries, state 0 and state 1. self.assertEqual(self.inv._CmdFilterCompleter([''], 0), 'apple') self.assertEqual(self.inv._CmdFilterCompleter([''], 1), 'pear') # An empty word list gives a 'None' result. - self.assertEqual( - self.inv._CmdFilterCompleter([''], len(self.inv._attributes)), - None) + self.assertIsNone( + self.inv._CmdFilterCompleter([''], len(self.inv._attributes)) + ) # Valid single match. self.assertEqual(self.inv._CmdFilterCompleter(['p'], 0), 'pear') self.assertEqual(self.inv._CmdFilterCompleter(['pe'], 0), 'pear') @@ -135,8 +179,10 @@ def testMaxTargets(self): command = 'maxtargets' # Display - self.assertEqual(self.inv._CmdMaxTargets(command, []), - f'Maxtargets: {inventory_base.DEFAULT_MAXTARGETS}') + self.assertEqual( + self.inv._CmdMaxTargets(command, []), + f'Maxtargets: {inventory_base.DEFAULT_MAXTARGETS}', + ) # Update self.inv._CmdMaxTargets(command, ['10']) self.assertEqual(self.inv._maxtargets, 10) @@ -198,8 +244,9 @@ def testChangeDeviceList(self): self.assertEqual(self.inv.device_list, ['device01', 'device03', 'device04']) self.inv._CmdFilter('vendor', ['cisco,juniper']) - self.assertEqual(self.inv.device_list, - ['device01', 'device02', 'device03', 'device04']) + self.assertEqual( + self.inv.device_list, ['device01', 'device02', 'device03', 'device04'] + ) # Realm and vendor filters. self.inv._CmdFilter('realm', ['prod']) @@ -215,24 +262,26 @@ def testChangeDeviceListMatches(self): self.assertListEqual(self.inv.device_list, ['device01', 'device02']) self.inv._CmdFilter('realm', ['^$']) - self.assertListEqual(self.inv.device_list, - ['device01', 'device02', 'device03', 'device04']) - + self.assertListEqual( + self.inv.device_list, ['device01', 'device02', 'device03', 'device04'] + ) + # Remove one device with xtargets. self.inv._CmdFilter('xtargets', ['device02']) - self.assertListEqual(self.inv.device_list, - ['device01', 'device03', 'device04']) + self.assertListEqual( + self.inv.device_list, ['device01', 'device03', 'device04'] + ) # Remove another device with additoinal xtargets. # Add some whitespace, which should be ignored. self.inv._CmdFilter('xtargets', [' device02 , device03 ']) self.assertListEqual(self.inv.device_list, ['device01', 'device04']) - # Remove two with regular matching and + # Remove two with regular matching and # two more (overlapping) via a regexp. self.inv._CmdFilter('xtargets', ['device02, device03, ^.*0[34]$']) self.assertListEqual(self.inv.device_list, ['device01']) - + # Set the realm to "lab" - there are no matches. self.inv._CmdFilter('realm', ['lab']) self.assertListEqual(self.inv.device_list, []) @@ -244,9 +293,10 @@ def testChangeDeviceListMatches(self): self._ClearFilters(self.inv) self.inv._CmdFilter('targets', ['^.*']) self.inv._CmdFilter('realm', ['^prod|lab']) - self.assertListEqual(self.inv.device_list, - ['device01', 'device02', 'device03', 'device04']) - + self.assertListEqual( + self.inv.device_list, ['device01', 'device02', 'device03', 'device04'] + ) + # Use the attributes indirect command rather than the 'targets' et al. self._ClearFilters(self.inv) self.inv._AttributeFilter('attributes', ['targets', 'device01']) @@ -267,27 +317,53 @@ def testFormatLabelAndValue(self): self.assertEqual(self.inv._FormatLabelAndValue('abc', 'xyz', 2), 'ABc: xyz') self.assertEqual(self.inv._FormatLabelAndValue('abc', 'xyz', 4), 'ABC: xyz') + def testBatchModeDefersLoad(self): + """Tests that batch mode defers loading devices until accessed.""" + with mock.patch.object( + inventory_base.Inventory, + '_FetchDevices', + autospec=True, + side_effect=inventory_base.Inventory._FetchDevices, + ) as mock_fetch: + batch_inv = inventory_base.Inventory(batch=True) + mock_fetch.assert_not_called() + batch_inv._CmdFilter('targets', ['device01']) + batch_inv._CmdFilter('realm', ['prod']) + mock_fetch.assert_not_called() + self.assertEqual(['device01'], batch_inv.device_list) + mock_fetch.assert_called_once() + + class AttributeFilterTest(unittest.TestCase): + def testDecomposeFilter(self): """Test deriving the compiled/literal filters from the string.""" - (literals, re_match) = inventory_base.FilterMatch('a,b,^c').filters + literals, re_match = inventory_base.FilterMatch('a,b,^c').filters # Filtering is split into literals and regexp entries - self.assertEqual((literals, [x.pattern for x in re_match]), - (['a', 'b'], ['^c$'])) - (literals, re_match) = inventory_base.FilterMatch( - '^a.*,b,^c$,d,e').filters - self.assertEqual((literals, [x.pattern for x in re_match]), - (['b', 'd', 'e'], ['^a.*$', '^c$'])) - (literals, re_match) = inventory_base.FilterMatch( - '^a.*,b, ^c$, d , "e"').filters - self.assertEqual((literals, [x.pattern for x in re_match]), - (['b', 'd', '"e"'], ['^a.*$', '^c$'])) - (literals, re_match) = inventory_base.FilterMatch( - '^A.*,B, ^C$, D , "E"').filters + self.assertEqual( + (literals, [x.pattern for x in re_match]), (['a', 'b'], ['^c$']) + ) + literals, re_match = inventory_base.FilterMatch('^a.*,b,^c$,d,e').filters + self.assertEqual( + (literals, [x.pattern for x in re_match]), + (['b', 'd', 'e'], ['^a.*$', '^c$']), + ) + literals, re_match = inventory_base.FilterMatch( + '^a.*,b, ^c$, d , "e"' + ).filters + self.assertEqual( + (literals, [x.pattern for x in re_match]), + (['b', 'd', '"e"'], ['^a.*$', '^c$']), + ) + literals, re_match = inventory_base.FilterMatch( + '^A.*,B, ^C$, D , "E"' + ).filters # Ignorecase, literals are changed to lowercase. Regexp are unchanged. - self.assertEqual((literals, [x.pattern for x in re_match]), - (['b', 'd', '"e"'], ['^A.*$', '^C$'])) + self.assertEqual( + (literals, [x.pattern for x in re_match]), + (['b', 'd', '"e"'], ['^A.*$', '^C$']), + ) # Bogus regular expressions raise an error. self.assertRaises(ValueError, inventory_base.FilterMatch, '^A.**') self.assertRaises(ValueError, inventory_base.FilterMatch, '^[A') @@ -297,10 +373,10 @@ def testMatch(self): _filter = inventory_base.FilterMatch('hello, World, ^b.*') self.assertTrue(_filter.Match('hello')) - self.assertTrue(_filter.Match('world')) # Cononical form is lowercase. + self.assertTrue(_filter.Match('world')) # Cononical form is lowercase. self.assertFalse(_filter.Match('World')) # Match on only lowercase. self.assertTrue(_filter.Match('beehive')) - self.assertTrue(_filter.Match('Beehive')) # Regexp case insensitive + self.assertTrue(_filter.Match('Beehive')) # Regexp case insensitive # Flip the ignorecase logic. _filter = inventory_base.FilterMatch('hello, World, ^b.*', False) @@ -315,5 +391,6 @@ def testMatch(self): self.assertFalse(_filter.Match(['a', 'b', 'c'])) self.assertTrue(_filter.Match(['d1', 'd2', 'd3'])) + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tcli/inventory_csv.py b/tcli/inventory_csv.py index 1be1e9d..c887740 100644 --- a/tcli/inventory_csv.py +++ b/tcli/inventory_csv.py @@ -41,9 +41,18 @@ from absl import logging from tcli import inventory_base +# pylint: disable=g-importing-member +from tcli.inventory_base import AuthError from tcli.inventory_base import CmdRequest -from tcli.inventory_base import AuthError # pylint: disable=unused-import -from tcli.inventory_base import InventoryError # pylint: disable=unused-import +from tcli.inventory_base import InventoryError + +__all__ = [ + 'Inventory', + 'AuthError', + 'CmdRequest', + 'CmdResponse', + 'InventoryError', +] ## CHANGEME ## The fields in the CSV that we want content to be filtered on. @@ -55,14 +64,21 @@ ## prefixed with an 'x' to be used for inverse matching (exclusions). DEVICE_ATTRIBUTES = inventory_base.DEVICE_ATTRIBUTES DEVICE_ATTRIBUTES['pop'] = inventory_base.Attribute( - 'pop', '', [], - '\n Limit device lists to specific pop/s') + 'pop', '', [], '\n Limit device lists to specific pop/s' +) DEVICE_ATTRIBUTES['realm'] = inventory_base.Attribute( - 'realm', 'lab', ['prod', 'lab'], - '\n Limit the device list to a specific realm/s.') + 'realm', + 'lab', + ['prod', 'lab'], + '\n Limit the device list to a specific realm/s.', +) DEVICE_ATTRIBUTES['vendor'] = inventory_base.Attribute( - 'vendor', '', ['cisco', 'juniper'], - '\n Limit device lists to a specific vendor/s', display_case='title') + 'vendor', + '', + ['cisco', 'juniper'], + '\n Limit device lists to a specific vendor/s', + display_case='title', +) # Set here if known. # In the case here this will be overwritten once the CSV content is known. @@ -71,39 +87,46 @@ ## CHANGEME ## To get up and running we use test data. Replace this path with the location ## of the device inventory CSV file that you wish to use. -DEFAULT_CSV_FILE = os.path.join(os.path.dirname(__file__), - 'testdata', 'test_devices') +DEFAULT_CSV_FILE = os.path.join( + os.path.dirname(__file__), 'testdata', 'test_devices' +) ## CHANGEME ## Any devices to exclude by default for all users, should be defined here. ## Add to the end of the list any additional expressions that should be ## implicitly filtered by default. -DEFAULT_XTARGETS = ','.join([inventory_base.DEFAULT_XTARGETS,]) +DEFAULT_XTARGETS = ','.join([ + inventory_base.DEFAULT_XTARGETS, +]) FLAGS = flags.FLAGS flags.DEFINE_string( - 'inventory', DEFAULT_CSV_FILE, - 'Location of file containing the CSV list of devices.') + 'inventory', + DEFAULT_CSV_FILE, + 'Location of file containing the CSV list of devices.', +) -# TODO(#38): Support using a different string for separating flags. -flags.DEFINE_string('separator', ', ', - 'String sequence that separates entries in the CSV file.') +# TODO(harro): Support using a different string for separating flags. +flags.DEFINE_string( + 'separator', ', ', 'String sequence that separates entries in the CSV file.' +) CmdResponse = inventory_base.Response + class Inventory(inventory_base.Inventory): """CSV Inventory Class. - Attributes: - source: String identifier of source of device inventory. + Attributes: + source: String identifier of source of device inventory. """ SOURCE = 'csv' def _ParseDevicesFromCsv( - self, buf:typing.TextIO, - separator:str=',') -> dict[str, typing.NamedTuple]: + self, buf: typing.TextIO, separator: str = ',' + ) -> dict[str, typing.NamedTuple]: """Parses buffer into tabular format. Strips off comments (preceded by '#'). @@ -154,37 +177,40 @@ def _ParseDevicesFromCsv( header_list = [l.strip() for l in header_list] if header_list[0] != 'device': raise ValueError( - 'Column named "device" must be first column of header.\n' - 'Found: "%s".' % header_str) + 'Column named "device" must be first column of header.\nFound: "%s".' + % header_str + ) # Strip device, it will be used for the index. - header_list = header_list[1 :] + header_list = header_list[1:] row_length = len(header_list) # Data format for a single device entry. - DEVICE = collections.namedtuple('Device', header_list) + device_cls = collections.namedtuple('Device', header_list) - # pylint: enable=invalid-name devices = {} # xreadlines would be better but not supported by StringIO for testing. - for row in buf: + for line in buf: # Support commented lines, provide '#' is first character of line. - row = row.strip() - if not row or row.startswith('#'): + line = line.strip() + if not line or line.startswith('#'): continue - row = row.split(separator) - # Strip excess whitespace. - row = [l.strip() for l in row] - (device_name, row) = (row[0], row[1 :]) + fields: list[typing.Any] = [l.strip() for l in line.split(separator)] + device_name, row_values = (fields[0], fields[1:]) if header_list[-1] == 'flags': # Provided the last header is 'flags' then accept extra columns. # Entries that trail on the rhs are gathered into a list under flags. - (device_flags, row) = (row[row_length -1:], row[:row_length -1]) - row.append(device_flags) # type: ignore + device_flags, row_values = ( + row_values[row_length - 1 :], + row_values[: row_length - 1], + ) + row_values.append(device_flags) try: - devices[device_name] = DEVICE(*row) - except TypeError: - raise ValueError('Final column header must be "flags" if' - ' rows are to be variable length.\n' - 'Found: %s' % header_str) + devices[device_name] = device_cls(*row_values) + except TypeError as exc: + raise ValueError( + 'Final column header must be "flags" if' + ' rows are to be variable length.\n' + 'Found: %s' % header_str + ) from exc logging.debug('Parsed "%s" devices from CSV file.', len(devices)) return devices @@ -193,4 +219,4 @@ def _FetchDevices(self): with open(FLAGS.inventory) as csv_file: logging.debug('Reading device inventory for file "%s".', FLAGS.inventory) - self._devices = self._ParseDevicesFromCsv(csv_file) \ No newline at end of file + self._devices = self._ParseDevicesFromCsv(csv_file) diff --git a/tcli/inventory_csv_test.py b/tcli/inventory_csv_test.py index e1e0698..ac94ad6 100644 --- a/tcli/inventory_csv_test.py +++ b/tcli/inventory_csv_test.py @@ -12,12 +12,11 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -"""Tests for tcli.inventory_csv.""" +"""Unit tests for the inventory_csv module.""" -import os -from io import StringIO # pylint: disable=g-importing-member +from io import StringIO # pylint: disable=g-importing-member +import typing import unittest -from unittest import mock from tcli import inventory_csv as inventory @@ -27,8 +26,10 @@ class UnitTestCSVInventory(unittest.TestCase): def _ClearFilters(self, inv_obj): """Clear all filters for targets etc.""" - for x in inv_obj._inclusions: inv_obj._inclusions[x] = '' - for y in inv_obj._exclusions: inv_obj._exclusions[y] = '' + for x in inv_obj._inclusions: + inv_obj._inclusions[x] = '' + for y in inv_obj._exclusions: + inv_obj._exclusions[y] = '' @classmethod def setUpClass(cls): @@ -38,61 +39,75 @@ def setUp(self): super(UnitTestCSVInventory, self).setUp() self.dev_inv_orig = inventory.inventory_base.DEVICE_ATTRIBUTES # Trigger FLAGS initialisation before referencing maxtargets flag. - inventory.FLAGS([__file__,]) + inventory.FLAGS([ + __file__, + ]) self.inv = inventory.Inventory() # Clear all filters for targets etc. self._ClearFilters(self.inv) - + def tearDown(self): + super().tearDown() inventory.inventory_base.DEVICE_ATTRIBUTES = self.dev_inv_orig def testParseDevicesFromCsv(self): """Tests parsing valid CSV data from string buffer.""" - - csv_text = ('# comment at start\n' - 'device, bb, ccc, flags\n' - '# between header and rows\n' - '# variable amounts of white space between fields.\n' - 'device_a,B , C , f1,f2, f3\n' - '# comment between rows\n' - '\n' - 'device_b, , CC, fx, fy, fz') - result = self.inv._ParseDevicesFromCsv(StringIO(csv_text)) - # Row matches dictionary key - self.assertEqual(result['device_a'].bb, 'B') # type: ignore + + csv_text = ( + '# comment at start\n' + 'device, bb, ccc, flags\n' + '# between header and rows\n' + '# variable amounts of white space between fields.\n' + 'device_a,B , C , f1,f2, f3\n' + '# comment between rows\n' + '\n' + 'device_b, , CC, fx, fy, fz' + ) + result: dict[str, typing.Any] = self.inv._ParseDevicesFromCsv( + StringIO(csv_text) + ) + # Row matches dictionary key. + self.assertEqual(result['device_a'].bb, 'B') # Spaces trim from entries. - self.assertEqual(result['device_a'].ccc, 'C') # type: ignore + self.assertEqual(result['device_a'].ccc, 'C') # Flags are a list. - self.assertEqual(result['device_a'].flags, ['f1', 'f2', 'f3'])# type: ignore + self.assertEqual(result['device_a'].flags, ['f1', 'f2', 'f3']) # Last entry is present and null column respected. - self.assertEqual(result['device_b'].ccc, 'CC') # type: ignore + self.assertEqual(result['device_b'].ccc, 'CC') def testParseDevicesFromCsvFail(self): """Tests parsing invalid CSV data.""" # Data without 'device' column.""" - csv_text = ('bogus, bb, ccc, flags\n' - 'device_a, B , C ,flag_1,flag_2,flag_3\n' - 'device_b,,CC,flag_x,flag_yy,flag_zzz') + csv_text = ( + 'bogus, bb, ccc, flags\n' + 'device_a, B , C ,flag_1,flag_2,flag_3\n' + 'device_b,,CC,flag_x,flag_yy,flag_zzz' + ) # First column of header should be 'device'. - self.assertRaises(ValueError, self.inv._ParseDevicesFromCsv, - StringIO(csv_text)) + self.assertRaises( + ValueError, self.inv._ParseDevicesFromCsv, StringIO(csv_text) + ) # Data without 'flag' column. - csv_text = ('device,bb,ccc,bogus\n' - 'device_a,B , C ,flag_1,flag_2,flag_3\n' - 'device_b,,CC,flag_x,flag_yy,flag_zzz') + csv_text = ( + 'device,bb,ccc,bogus\n' + 'device_a,B , C ,flag_1,flag_2,flag_3\n' + 'device_b,,CC,flag_x,flag_yy,flag_zzz' + ) # Last column of header should be 'flags'. - self.assertRaises(ValueError, self.inv._ParseDevicesFromCsv, - StringIO(csv_text)) + self.assertRaises( + ValueError, self.inv._ParseDevicesFromCsv, StringIO(csv_text) + ) # Data from empty buffer. - csv_text = ('') + csv_text = '' # Empty buffer - self.assertRaises(ValueError, self.inv._ParseDevicesFromCsv, - StringIO(csv_text)) + self.assertRaises( + ValueError, self.inv._ParseDevicesFromCsv, StringIO(csv_text) + ) # Data from empty table. - csv_text = ('device,bb,ccc,flags\n') + csv_text = 'device,bb,ccc,flags\n' # Empty table result = self.inv._ParseDevicesFromCsv(StringIO(csv_text)) self.assertEqual(result, {}) @@ -103,5 +118,6 @@ def testFetchDevices(self): self.inv._FetchDevices() self.assertTrue(self.inv._devices) + if __name__ == '__main__': unittest.main() diff --git a/tcli/tcli_lib.py b/tcli/tcli_lib.py index 7e42110..171eff2 100644 --- a/tcli/tcli_lib.py +++ b/tcli/tcli_lib.py @@ -22,34 +22,36 @@ import copy import os import subprocess +import shutil import sys import tempfile import threading from absl import flags from absl import logging -try: - # For windows platform. - from pyreadline3.rlmain import Readline # type: ignore - readline = Readline() -except(ImportError): - import readline +# inventory import will be overridden in main.py +from tcli import accessor_base as accessor from tcli import command_completer as completer from tcli import command_parser from tcli import command_register from tcli import command_response from tcli import display -# inventory import will be overridden in main.py -from tcli import accessor_base as accessor from tcli import inventory_base as inventory from tcli import text_buffer -from tcli.command_parser import ParseError, I from tcli.tcli_textfsm import clitable -from tcli.tcli_textfsm.clitable import CliTableError from textfsm import terminal from textfsm import texttable -from textfsm.texttable import TableError + +# pylint: disable=g-import-not-at-top +try: + # For windows platform. + from pyreadline3.rlmain import Readline + + readline = Readline() +except ImportError: + import readline +# pylint: enable=g-import-not-at-top # Formats for displaying to the user. DISPLAY_FORMATS = command_register.DISPLAY_FORMATS @@ -60,11 +62,12 @@ # TCLI (local) command prefix. SLASH = command_parser.SLASH # pylint: disable=redefined-builtin -if __doc__: __doc__ = __doc__.replace('%s', SLASH) +if __doc__: + __doc__ = __doc__.replace('%s', SLASH) # Banner message to display at program start. BANNER_WIDTH = 76 -MOTD = f"""#!{'#'*BANNER_WIDTH}!#' +MOTD = f"""#!{'#'*BANNER_WIDTH}!#' #! TCLI - Tokenized Command Line Interface #! Note: Beta code, use with caution. #! @@ -72,7 +75,7 @@ #! To disable color: '{SLASH}color off'. #! #! For more guidance see: -#! https://github.com/harro/tcli +#! https://github.com/google/tcli #! #! Note: #! Interactive TCLI starts in safe mode (indicated by '*' in the prompt). @@ -87,21 +90,38 @@ FLAGS = flags.FLAGS flags.DEFINE_string( - 'config_file', DEFAULT_CONFIGFILE, """ + 'config_file', + DEFAULT_CONFIGFILE, + """ Configuration file to read. Lines in this file will be read into ' buffer "startup" and played.' - Skipped if file name is the string 'None|none'""", short_name='R') + Skipped if file name is the string 'None|none'""", + short_name='R', +) flags.DEFINE_boolean( - 'dry_run', False, - f'{I}Display commands and targets, without submitting commands.') + 'dry_run', + False, + f'{command_parser.I}Display commands and targets, without submitting' + ' commands.', +) -flags.DEFINE_boolean('sorted', False, f'{I}Sort device entries in output.') +flags.DEFINE_boolean( + 'restricted', + False, + f'{command_parser.I}Disable shell execution, editor, and file writes.', +) -flags.DEFINE_string( - 'template_dir', os.path.join(os.path.dirname(__file__), 'testdata'), - f'{I}Path where command templates are located', short_name='t') +flags.DEFINE_boolean( + 'sorted', False, f'{command_parser.I}Sort device entries in output.' +) +flags.DEFINE_string( + 'template_dir', + os.path.join(os.path.dirname(__file__), 'testdata'), + f'{command_parser.I}Path where command templates are located', + short_name='t', +) class Error(Exception): """Base class for errors.""" @@ -126,6 +146,7 @@ class TCLI(object): cmd_response: Object that handles collating device requests/responses. color: If output uses ANSI escape characters for color. display: String denoting how to format output data. + dsp: Display object for formatting output and prompts. filter: String denoting how to extract data from device command output. filter_engine: Object that parses device command data. interactive: Boolean, is the user running this in interactive mode. @@ -140,21 +161,23 @@ class TCLI(object): recordall: Similar to 'record' but stores escape commands too. safemode: Boolean, control if raw commands are forwarded to targets or not. targets: String of device names and regular expressions to match against. - device_list: Sorted list of device names, the union of targets, - excluding matches with xtargets. Read-only. + device_list: Sorted list of device names, the union of targets, excluding + matches with xtargets. Read-only. timeout: Integer, Duration (in seconds) to wait for outstanding commands - responses to complete. + responses to complete. verbose: Boolean, if to display all columns or suppress Verbose ones. xtargets: String of device names and regular expressions to exclude. """ - def __init__(self, interactive=False, commands='', inventory=None) -> None: + def __init__( + self, interactive: bool = False, commands: str = '', inv=None + ) -> None: # Also see copy method when modifying/adding attributes. # Async callback. self._lock = threading.Lock() self.interactive = interactive - self.inventory = inventory + self.inventory = inv self.filter_engine = None self.playback = None self.safemode = False @@ -176,6 +199,8 @@ def __init__(self, interactive=False, commands='', inventory=None) -> None: self.dsp = display.Display() if not self.inventory: self._InitInventory() + elif hasattr(self.inventory, 'RegisterCommands'): + self.inventory.RegisterCommands(self.cli_parser) command_register.RegisterCommands(self, self.cli_parser) command_register.SetFlagDefaults(self.cli_parser) # If interactive then the user will input furhter commands. @@ -188,13 +213,17 @@ def __init__(self, interactive=False, commands='', inventory=None) -> None: if commands: self._ParseCommands(commands) - def __copy__(self) -> "TCLI": + def __copy__(self) -> 'TCLI': """Copies attributes from self to new tcli object.""" + inv_copy = ( + copy.copy(self.inventory) + if self.inventory is not None else None + ) # Create new instance with existing inventory. - tcli_obj = type(self)(inventory=self.inventory) - #TODO(harro): Why does this break ParseCommands? + tcli_obj = type(self)(inv=inv_copy) + # TODO: Why does this break ParseCommands? # tcli_obj.__dict__.update(self.__dict__) - + tcli_obj.buffers = self.buffers tcli_obj.cmd_response = self.cmd_response tcli_obj.color = self.color @@ -218,26 +247,28 @@ def __copy__(self) -> "TCLI": device_list = property(lambda self: self.inventory.device_list) # pylint: disable=unused-argument - def Completer(self, word: str, state: int) -> str|None: + def Completer(self, word: str, state: int) -> str | None: """Command line completion used by readline library.""" # The readline completer is not stateful. So we read the full line and pass # that to the respective completer functions. # Silently discard leading whitespace on cli. - full_line = readline.get_line_buffer().lstrip() # type: ignore + full_line = readline.get_line_buffer().lstrip() if full_line and full_line.startswith(SLASH): - if not self.cli_parser: return None + if not self.cli_parser: + return None # Return completions for matching TCLI commands. return completer.TCLICompleter(full_line, state, self.cli_parser) - # If not a TCLI command, or an empty prompt, then return completions for + # If not a TCLI command, or an empty prompt, then return completions for # known remote device commands that we support with TextFSM. - if not self.filter_engine: return None + if not self.filter_engine: + return None return completer.CmdCompleter(full_line, state, self.filter_engine) - def Motd(self) ->None: + def Motd(self) -> None: """Display message of the day.""" self._Print(MOTD, msgtype='system') @@ -248,8 +279,13 @@ def Prompt(self) -> None: self.cmd_response = command_response.CmdResponse() # Print the main prompt, so the ASCII escape sequences work. # Displays the target string, count of targets and if safe mode is on. - print(self.dsp.getPrompt(self.inventory.targets, # type: ignore - self.device_list, self.safemode)) + print( + self.dsp.getPrompt( + self.inventory.targets, + self.device_list, + self.safemode, + ) + ) # Stripped ASCII escape from here, as they are not interpreted in PY3. self._ParseCommands(input(display.PROMPT_STR)) @@ -258,13 +294,16 @@ def _BufferInUse(self, buffername: str) -> bool: if buffername in (self.record, self.recordall, self.log, self.logall): self._Print( - f"Buffer: '{buffername}', already open for writing.", msgtype='warning') + f"Buffer: '{buffername}', already open for writing.", + msgtype='warning', + ) return True if buffername == self.playback: self._Print( - f"Buffer: '{self.playback}', already open by 'play' command.", - msgtype='warning') + f"Buffer: '{self.playback}', already open by 'play' command.", + msgtype='warning', + ) return True return False @@ -278,13 +317,17 @@ def _Callback(self, response: inventory.Response) -> None: self.cmd_response.AddResponse(response) # If we have all responses for current row/command then display. - (row, pipe) = self.cmd_response.GetRow() + row, pipe = self.cmd_response.GetRow() while row: self._FormatRow(row, pipe) - (row, pipe) = self.cmd_response.GetRow() - - def _CmdRequests(self, device_list:list[str], command_list:list[str], - explicit_cmd:bool=False) -> None: + row, pipe = self.cmd_response.GetRow() + + def _CmdRequests( + self, + device_list: list[str], + command_list: list[str], + explicit_cmd: bool = False, + ) -> None: """Submits command list to devices and waits on responses. Args: @@ -320,7 +363,7 @@ def _CmdRequests(self, device_list:list[str], command_list:list[str], # The set of responses for a given command is known as a 'row'. for cmd_row_id, command in enumerate(command_list): # Split off client side pipe command. - (command, pipe) = self.cli_parser.ExtractPipe(command) + command, pipe = self.cli_parser.ExtractPipe(command) logging.debug("Extracted command and pipe: '%s' & '%s'.", command, pipe) self.cmd_response.InitCommandRow(cmd_row_id, pipe) @@ -343,15 +386,18 @@ def _CmdRequests(self, device_list:list[str], command_list:list[str], # Wait for all callbacks to complete. # We add a 5 seconds pad to allow requests to timeout and be included in the # results before cleanup and reporting results. - if not self.cmd_response.done.wait(self.timeout +5): + if not self.cmd_response.done.wait(self.timeout + 5): # If we had a timeout then clear pending responses. self.cmd_response = command_response.CmdResponse() - self._Print('Timeout: timer exceeded while waiting for responses.', - msgtype='warning') + self._Print( + 'Timeout: timer exceeded while waiting for responses.', + msgtype='warning', + ) logging.debug('CmdRequests: All callbacks completed.') def _DisplayFormatted( - self, result: clitable.CliTable, pipe: str = '') -> None: + self, result: clitable.CliTable, pipe: str = '' + ) -> None: """Displays output in tabular form.""" if self.display == 'csv': @@ -363,20 +409,19 @@ def _DisplayFormatted( self._Print(self._Pipe(result.LabelValueTable(), pipe=pipe)) elif self.display == 'tbl': - (_, width) = terminal.TerminalSize() # type: ignore + _, width = shutil.get_terminal_size() try: self._Print(self._Pipe(result.FormattedTable(width), pipe=pipe)) - except TableError as error_message: + except texttable.TableError: width *= 2 # Try again allowing text to wrap once. try: self._Print(self._Pipe(result.FormattedTable(width), pipe=pipe)) - except TableError as error_message: + except texttable.TableError as error_message: self._Print(str(error_message), msgtype='warning') else: # Problem with parsing of display command if we reach here. - raise TcliCmdError('Unsupported display format: %s.' % - repr(self.display)) + raise TcliCmdError('Unsupported display format: %s.' % repr(self.display)) def _DisplayRaw(self, response: inventory.Response, pipe: str = '') -> None: """Display response in raw format.""" @@ -397,17 +442,30 @@ def _FormatRow(self, response_uid_list: list[int], pipe: str = '') -> None: # Filter required if display format is not 'raw'. if self.display != 'raw' and not self.filter: - self._Print(f"No filter set, cannot display in '{self.display}' format", - 'warning') + self._Print( + f"No filter set, cannot display in '{self.display}' format", 'warning' + ) return + # Sort by device name first to ensure consistent output for parallel + # queries. + response_uid_list = sorted( + response_uid_list, + key=lambda uid: ( + (response.device_name or '') + if (response := self.cmd_response.GetResponse(uid)) + else '' + ), + ) + result = {} for response_uid in response_uid_list: response = self.cmd_response.GetResponse(response_uid) if not response: self._Print( - 'Invalid or missing response: Some output could not be displayed.', - msgtype='warning') + 'Invalid or missing response: Some output could not be displayed.', + msgtype='warning', + ) continue # If response includes an error then print that. @@ -420,35 +478,39 @@ def _FormatRow(self, response_uid_list: list[int], pipe: str = '') -> None: continue # Build a CliTable attribute filter from the response. - filter_attr = {'Command': response.command, - 'Hostname': response.device_name} + filter_attr = { + 'Command': response.command, + 'Hostname': response.device_name, + } device = self.devices[response.device_name] - for attr in self.inventory.attributes: # type: ignore + for attr in self.inventory.attributes: # Some attributes are a list rather than a string, such as flags. # These are not supported by Clitable attribute matching # and we silently drop them here. - if (not getattr(device, attr) or - isinstance(getattr(device, attr), list)): + if not getattr(device, attr) or isinstance(getattr(device, attr), list): continue # The filter index uses capitilised first letter for column names. # For some values we capitilise part of the value. - if self.inventory.attributes[attr].display_case == 'title': # type: ignore + if self.inventory.attributes[attr].display_case == 'title': filter_attr[attr.title()] = getattr(device, attr).title() - elif self.inventory.attributes[attr].display_case == 'upper': # type: ignore + elif self.inventory.attributes[attr].display_case == 'upper': filter_attr[attr.title()] = getattr(device, attr).upper() else: filter_attr[attr.title()] = getattr(device, attr) try: logging.debug('Parse response with attributes "%s".', filter_attr) - if not self.filter_engine: raise(CliTableError) + if not self.filter_engine: + raise (clitable.CliTableError) self.filter_engine.ParseCmd( - response.data, attributes=filter_attr, verbose=self.verbose) - except CliTableError as error_message: - logging.debug('Parsing engine failed for device "%s".', - response.device_name) + response.data, attributes=filter_attr, verbose=self.verbose + ) + except clitable.CliTableError as error_message: + logging.debug( + 'Parsing engine failed for device "%s".', response.device_name + ) self._Print(error_message, msgtype='warning') # If unable to parse then output as raw. self._DisplayRaw(response, pipe=pipe) @@ -466,7 +528,9 @@ def _FormatRow(self, response_uid_list: list[int], pipe: str = '') -> None: if str(self.filter_engine.header) not in result: # Copy initial command result, then append the rest as rows. # There will be a separate table for each unique set of table columns. - result[str(self.filter_engine.header)] = copy.deepcopy(self.filter_engine) + result[str(self.filter_engine.header)] = copy.deepcopy( + self.filter_engine + ) else: result[str(self.filter_engine.header)] += self.filter_engine @@ -483,13 +547,13 @@ def _InitInventory(self) -> None: """Inits inventory and triggers async load of device data.""" try: - self.inventory = inventory.Inventory() + self.inventory = inventory.Inventory(batch=not self.interactive) # Add additional command support for Inventory library. self.inventory.RegisterCommands(self.cli_parser) self.inventory.SetFiltersFromDefaults(self.cli_parser) except inventory.AuthError as error_message: self._Print(str(error_message), msgtype='warning') - raise inventory.AuthError() + raise inventory.AuthError() from error_message def _ParseCommands(self, commands: str) -> None: """Parses commands and executes them. @@ -502,34 +566,37 @@ def _ParseCommands(self, commands: str) -> None: commands: String of newline separated commands. """ - def _Flush(devices, commands: list[str]) -> None: + def _Flush(commands: list[str]) -> None: """Flush any pending device commands for the backend.""" - if not commands: return + if not commands: + return logging.debug('Flush commands: %s', commands) # Pass copy rather than reference to list. - self._CmdRequests(devices, commands[:]) - + self._CmdRequests(self.device_list, commands[:]) + # Split commands into list on newlines. Build new command_list. command_list = [] for command in commands.split('\n'): command = command.strip() # Skip blank lines. - if not command: continue + if not command: + continue # TCLI commands. if command.startswith(SLASH): - _Flush(self.device_list, command_list) + _Flush(command_list) command_list = [] # Remove command prefix and submit current command to TCLI interpreter. - self._TCLICmd(command[1 :]) + self._TCLICmd(command[1:]) else: # Backend commands for sending to devices. # Look for inline TCLI commands. - (command_prefix, inline_commands - ) = self.cli_parser.ExtractInlineCommands(command) + command_prefix, inline_commands = self.cli_parser.ExtractInlineCommands( + command + ) if inline_commands: # Send any commands we have collecte so far. - _Flush(self.device_list, command_list) + _Flush(command_list) command_list = [] # Commands with inline display modifiers are submitted # to a copy of the TCLI object with the inline modifiers applied. @@ -538,13 +605,15 @@ def _Flush(devices, commands: list[str]) -> None: inline_tcli.cli_parser.InlineOnly() # Apply the inline TCLI commands to this instance. for cmd in inline_commands: - inline_tcli._TCLICmd(cmd) - inline_tcli._CmdRequests(inline_tcli.device_list, [command_prefix]) + inline_tcli._TCLICmd(cmd) # pylint: disable=protected-access + inline_tcli._CmdRequests( # pylint: disable=protected-access + inline_tcli.device_list, [command_prefix] + ) else: # Otherwise continue collecting multiple commands to send at once. command_list.append(command) - _Flush(self.device_list, command_list) + _Flush(command_list) def _ParseRCFile(self) -> None: """Reads and execs the rc file. @@ -561,27 +630,38 @@ def _ParseRCFile(self) -> None: if FLAGS.config_file.lower() != 'none': try: - with open(FLAGS.config_file) as config_file: + with open(FLAGS.config_file, encoding='utf-8') as config_file: self.buffers.Append('startup', config_file.read()) self._ParseCommands(self.buffers.GetBuffer('startup')) - except IOError: + except IOError as exc: # Silently fail if we don't find a file in the default location. # Warn the user if they supplied a file explicitly. if FLAGS.config_file != DEFAULT_CONFIGFILE: self._Print( 'Error: Reading config file: %s' % sys.exc_info()[1], - msgtype='warning') - raise EOFError() + msgtype='warning', + ) + raise EOFError() from exc - def _Pipe(self, output: str, pipe: str = '') -> str|None: + def _Pipe(self, output: str, pipe: str = '') -> str | None: """Creates pipe for filtering command output.""" - if not pipe: return output + if not pipe: + return output + if FLAGS.restricted: + self._Print( + 'Shell pipes are disabled in restricted mode.', msgtype='warning' + ) + return pipe = pipe.lstrip('|') try: p = subprocess.Popen( - [pipe], shell=True, close_fds=True, - stdin=subprocess.PIPE, stdout=subprocess.PIPE) + [pipe], + shell=True, + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) except IOError as error_message: logging.error('IOerror opening pipe.') if str(error_message): @@ -597,22 +677,23 @@ def _Pipe(self, output: str, pipe: str = '') -> str|None: logging.error('IOerror writing/reading from pipe.') return - def _Print(self, msg, msgtype='default') ->str|None: + def _Print(self, msg, msgtype='default') -> str | None: """Prints (and logs) outputs.""" - if not msg: return + if not msg: + return # Capture output in logs. for buf in (self.logall,): self.buffers.Append(buf, msg) - + self.dsp.printOut(msg, self.color, self.linewrap, msgtype) def _TCLICmd(self, line: str) -> None: - f"""TCLI configuration command. + """TCLI configuration command. Args: - line: String command for TCLI parsing. Minus the {SLASH} escape prefix. + line: String command for TCLI parsing. Minus the '/' escape prefix. Raises: EOFError: If exit is issued. @@ -620,8 +701,8 @@ def _TCLICmd(self, line: str) -> None: # Command parsing try: - (command, args, append) = self.cli_parser.ParseCommandLine(line) - except ParseError as error_message: + command, args, append = self.cli_parser.ParseCommandLine(line) + except command_parser.ParseError as error_message: self._Print(str(error_message), msgtype='warning') return @@ -639,7 +720,8 @@ def _TCLICmd(self, line: str) -> None: # Command execution. try: result = self.cli_parser.ExecHandler(command, args, append) - if result: self._Print(result, msgtype='system') + if result: + self._Print(result, msgtype='system') # pylint: disable=broad-except except ValueError as error_message: self._Print(str(error_message), msgtype='warning') @@ -656,13 +738,14 @@ def _TCLICmd(self, line: str) -> None: # pylint: disable=unused-argument def _CmdBuffer( - self, command: str, args: list[str], append: bool=False) -> None: - """"Displays buffer contents.""" + self, command: str, args: list[str], append: bool = False + ) -> None: + """Displays buffer contents.""" # Copy buffer to local var so we capture content before adding more here. try: buf = self.buffers.GetBuffer(args[0]) - except(AttributeError): + except AttributeError: self._Print(f'Invalid buffer name "{args[0]}".', msgtype='warning') return @@ -673,22 +756,25 @@ def _CmdBuffer( self._Print('#! ENDBUFFER !#', msgtype='warning') def _CmdBufferList( - self, command:str, args:list[str], append:bool=False) -> str: + self, command: str, args: list[str], append: bool = False + ) -> str: """List all buffers.""" return self.buffers.ListBuffers() def _CmdClear( - self, command: str, args: list[str], append: bool=False) -> None: + self, command: str, args: list[str], append: bool = False + ) -> None: """Clears content of the buffer.""" self.buffers.Clear(args[0]) def _CmdColorScheme( - self, command:str, args:list[str], append:bool=False) -> str|None: + self, command: str, args: list[str], append: bool = False + ) -> str | None: """Sets ANSI color escape values.""" if not args: return self.dsp.color_scheme - + scheme = args[0] self.dsp.setColorScheme(scheme) @@ -697,11 +783,13 @@ def _CmdCommand(self, command: str, args: list[str], append: bool) -> None: self._CmdRequests(self.device_list, [args[0]], True) def _CmdDefaults( - self, command:str, args:list[str], append:bool=False) -> str|None: + self, command: str, args: list[str], append: bool = False + ) -> str | None: """Reset commands to the 'at start' value.""" # Display rather than change. - if not args: return self._CmdEnv(command, args, append) + if not args: + return self._CmdEnv(command, args, append) # Change everything default = args[0] @@ -711,14 +799,15 @@ def _CmdDefaults( if self.inventory: self.inventory.SetFiltersFromDefaults(self.cli_parser) return - + try: return self.cli_parser.ExecWithDefault(default) - except Exception: - raise ValueError("Cannot set '{default}' to defaults.") + except Exception as exc: + raise ValueError(f"Cannot set '{default}' to defaults.") from exc def _CmdDisplay( - self, command:str, args:list[str], append:bool) -> str|None: + self, command: str, args: list[str], append: bool + ) -> str | None: """Set the layout format.""" if not args: @@ -730,7 +819,8 @@ def _CmdDisplay( else: raise ValueError( f"Unknown display '{repr(display_format)}'." - f" Available displays are '{DISPLAY_FORMATS}'") + f" Available displays are '{DISPLAY_FORMATS}'" + ) def _CmdEnv(self, command: str, args: list[str], append: bool) -> str: """Display various environment variables.""" @@ -740,69 +830,77 @@ def _CmdEnv(self, command: str, args: list[str], append: bool) -> str: inventory_str = self.inventory.ShowEnv() return '\n'.join([ - f'Format: {self.display}, Filter: {self.filter}', - f'Record: {self.record}, Recordall: {self.recordall}', - f'Log: {self.log}, Logall: {self.logall}', - f'Color: {self.color}, Scheme: {self.dsp.color_scheme}', - f'Timeout: {self.timeout}, Verbose: {self.verbose}', - f'CLI Mode: {self.mode}, Safemode: {self.safemode}', - f'Line Wrap: {self.linewrap}\n{inventory_str}' - ]) - - def _CmdExecShell( - self, command:str, args:list[str], append:bool) -> str: + f'Format: {self.display}, Filter: {self.filter}', + f'Record: {self.record}, Recordall: {self.recordall}', + f'Log: {self.log}, Logall: {self.logall}', + f'Color: {self.color}, Scheme: {self.dsp.color_scheme}', + f'Timeout: {self.timeout}, Verbose: {self.verbose}', + f'CLI Mode: {self.mode}, Safemode: {self.safemode}', + f'Line Wrap: {self.linewrap}\n{inventory_str}', + ]) + + def _CmdExecShell(self, command: str, args: list[str], append: bool) -> str: """Executes a shell command.""" + if FLAGS.restricted: + raise ValueError('Command disabled in restricted mode.') try: - exec_out = os.popen(args[0]) - output = exec_out.read() - exec_out.close() + with os.popen(args[0]) as exec_out: + output = exec_out.read() except IOError as error_message: - raise ValueError(error_message) + raise ValueError(error_message) from error_message return output def _CmdEditor(self, command: str, args: list[str], append: bool) -> None: """Edits the named buffer content.""" + if FLAGS.restricted: + raise ValueError('Command disabled in restricted mode.') buf = args[0] - buf_file: tempfile._TemporaryFileWrapper[bytes] = tempfile.NamedTemporaryFile() - # Write out the buffer data to file. - content: str = self.buffers.GetBuffer(buf) - if content: - buf_file.write(content.encode('ascii')) - # Flush content so editor will see it. - buf_file.flush() - #TODO(#39): Support os.getenv('EDITOR', 'vi'). - #TODO(harro): Maybe catch exceptions here. - # Open file with vi. - os.system('vi -Z -n -u NONE -U NONE -- %s' % (buf_file.name)) - # Read back the data into the buffer. - buf_file.seek(0) - self.buffers.Clear(buf) - self.buffers.Append(buf, buf_file.read().decode('ascii')) - buf_file.close() + with tempfile.NamedTemporaryFile() as buf_file: + # Write out the buffer data to file. + content: str = self.buffers.GetBuffer(buf) + if content: + buf_file.write(content.encode('utf-8')) + # Flush content so editor will see it. + buf_file.flush() + # TODO: Support os.getenv('EDITOR', 'vi'). + # TODO: Maybe catch exceptions here. + # Open file with vi. + subprocess.run( + ['vi', '-Z', '-n', '-u', 'NONE', '-U', 'NONE', '--', buf_file.name], + check=False, + ) + # Read back the data into the buffer. + buf_file.seek(0) + self.buffers.Clear(buf) + self.buffers.Append(buf, buf_file.read().decode('utf-8')) def _CmdExit( - self, command:str, args:list[str], append:bool=False) -> None: + self, command: str, args: list[str], append: bool = False + ) -> None: """Exit TCLI.""" raise EOFError() def _CmdExpandTargets( - self, command: str, args: list[str], append: bool) -> str: + self, command: str, args: list[str], append: bool + ) -> str: return ','.join(self.device_list) def _CmdFilter( - self, command: str, args: list[str], append: bool) -> str|None: + self, command: str, args: list[str], append: bool + ) -> str | None: """Sets the clitable filter.""" - if not args: return 'Filter: %s' % self.filter + if not args: + return 'Filter: %s' % self.filter filter_name = args[0] try: self.filter_engine = clitable.CliTable(filter_name, FLAGS.template_dir) self.filter = filter_name - except (clitable.CliTableError, texttable.TableError, IOError): - raise ValueError('Invalid filter %s.' % repr(filter_name)) + except (clitable.CliTableError, texttable.TableError, IOError) as exc: + raise ValueError('Invalid filter %s.' % repr(filter_name)) from exc def _CmdHelp(self, command: str, args: list[str], append: bool) -> str: """Display help.""" @@ -813,8 +911,7 @@ def _CmdHelp(self, command: str, args: list[str], append: bool) -> str: cmd_obj = self.cli_parser[cmd_name] append_str = '[+]' if cmd_obj.append else '' arg = f' <{cmd_name}>' if cmd_obj.min_args else '' - result.append( - f'{cmd_name}{append_str}{arg} {cmd_obj.help_str}') + result.append(f'{cmd_name}{append_str}{arg} {cmd_obj.help_str}') return '\n\n'.join(result) def _CmdInventory(self, command: str, args: list[str], append: bool) -> str: @@ -824,9 +921,11 @@ def _CmdInventory(self, command: str, args: list[str], append: bool) -> str: for device_name in self.device_list: device = self.devices[device_name] attr_list = [device_name] - for name in self.inventory.attributes: # type: ignore - if name == 'flags': continue - if not getattr(device, name): continue + for name in self.inventory.attributes: + if name == 'flags': + continue + if not getattr(device, name): + continue attr_list.append(f'{name.title()}:{str(getattr(device, name)) or ""}') for fl in device.flags: @@ -836,7 +935,8 @@ def _CmdInventory(self, command: str, args: list[str], append: bool) -> str: return '\n'.join(dlist) def _CmdLogging( - self, command: str, args: list[str], append: bool) -> str|None: + self, command: str, args: list[str], append: bool + ) -> str | None: """Activates one of the various logging functions.""" # If no arg then display what buffer is currently active. @@ -849,14 +949,17 @@ def _CmdLogging( buf = args[0] # In this we are appending but still need to check that we are not # already logging, or playing out from it. - if self._BufferInUse(buf): return + if self._BufferInUse(buf): + return # Clear the buffer as we are not appending. - if not append: self.buffers.Clear(buf) + if not append: + self.buffers.Clear(buf) setattr(self, command, buf) def _CmdLogStop( - self, command:str, args:list[str], append:bool=False) -> None: + self, command: str, args: list[str], append: bool = False + ) -> None: """Stop logging to a buffer.""" for attr in ('record', 'recordall', 'log', 'logall'): @@ -866,16 +969,17 @@ def _CmdLogStop( raise ValueError('Buffer not in use for logging or recording.') - def _CmdMode( - self, command:str, args:list[str], append:bool) -> str|None: + def _CmdMode(self, command: str, args: list[str], append: bool) -> str | None: """Target CLI Mode to send to commands for.""" - if not args: return f'Mode: {self.mode}' + if not args: + return f'Mode: {self.mode}' mode = args[0] if mode not in MODE_FORMATS: raise ValueError( - f"Unknown mode {repr(mode)}. Available modes are '{MODE_FORMATS}'") + f"Unknown mode {repr(mode)}. Available modes are '{MODE_FORMATS}'" + ) self.mode = mode def _CmdPlay(self, command: str, args: list[str], append: bool) -> None: @@ -892,12 +996,12 @@ def _CmdPlay(self, command: str, args: list[str], append: bool) -> None: try: content = self.buffers.GetBuffer(buf) self._ParseCommands(content) - except(AttributeError): + except AttributeError: self._Print(f"Nonexistent buffer: '{buf}'.", msgtype='warning') self.playback = None def _CmdRead(self, command: str, args: list[str], append: bool) -> str: - """"Write buffer content to file.""" + """Reads file content into buffer.""" buf = args[0] if len(args) > 1: @@ -908,22 +1012,24 @@ def _CmdRead(self, command: str, args: list[str], append: bool) -> str: filename = os.path.expanduser(os.path.expandvars(filename)) try: - buf_file = open(filename, 'r') + with open(filename, 'r', encoding='utf-8') as buf_file: + content = buf_file.read() except IOError as error_message: - raise ValueError(str(error_message)) + raise ValueError(str(error_message)) from error_message - if append: + if not append: self.buffers.Clear(buf) - self.buffers.Append(buf, buf_file.read()) - buf_file.close() + self.buffers.Append(buf, content) return '%d lines read.' % self.buffers.GetBuffer(buf).count('\n') def _CmdTimeout( - self, command: str, args: list[str], append: bool) -> str|None: + self, command: str, args: list[str], append: bool + ) -> str | None: """Sets or display the timeout setting.""" - if not args: return f'Timeout: {self.timeout}' + if not args: + return f'Timeout: {self.timeout}' try: timeout = int(args[0]) @@ -931,11 +1037,12 @@ def _CmdTimeout( self.timeout = int(args[0]) else: raise ValueError - except ValueError: - raise ValueError('Invalid timeout value %s.' % repr(args[0])) + except ValueError as exc: + raise ValueError('Invalid timeout value %s.' % repr(args[0])) from exc def _CmdToggleValue( - self, command: str, args: list[str], append: bool) -> None: + self, command: str, args: list[str], append: bool + ) -> None: """Commands that can 'toggle' their value.""" if not args: @@ -947,16 +1054,19 @@ def _CmdToggleValue( if value not in ('on', 'true', 'off', 'false'): raise ValueError("Error: Argument must be 'on' or 'off'.") - if value in ('on', 'true'): setattr(self, command, True) - elif value in ('off', 'false'): setattr(self, command, False) + if value in ('on', 'true'): + setattr(self, command, True) + elif value in ('off', 'false'): + setattr(self, command, False) - #TODO(#40): Add flag to disable/block being able to write to file. - def _CmdWrite( - self, command: str, args: list[str], append: bool) -> str: + def _CmdWrite(self, command: str, args: list[str], append: bool) -> str: """Writes out buffer content to file.""" + if FLAGS.restricted: + raise ValueError('Command disabled in restricted mode.') content = self.buffers.GetBuffer(args[0]) - if not content: raise ValueError('Buffer empty.') + if not content: + raise ValueError('Buffer empty.') if len(args) > 1: filename = args[1] @@ -966,16 +1076,16 @@ def _CmdWrite( filename = os.path.expanduser(os.path.expandvars(filename)) + mode = 'a' if append else 'w' try: - buf_file = open(filename, 'a') if append else open(filename, 'w') - buf_file.writelines(content) - buf_file.close() + with open(filename, mode, encoding='utf-8') as buf_file: + buf_file.writelines(content) except IOError as error_message: - raise ValueError(str(error_message)) + raise ValueError(str(error_message)) from error_message return '%d lines written.' % content.count('\n') - + # pylint: enable=unused-argument ############################################################################## # End of command handles. # - ############################################################################## \ No newline at end of file + ############################################################################## diff --git a/tcli/tcli_test.py b/tcli/tcli_test.py index 4e36bbc..91979ef 100644 --- a/tcli/tcli_test.py +++ b/tcli/tcli_test.py @@ -17,18 +17,18 @@ import collections import copy import os +import shutil +import tempfile import unittest from unittest import mock from absl import flags - +from tcli import accessor_base as accessor from tcli import command_register from tcli import inventory_base as inventory -from tcli import accessor_base as accessor from tcli import tcli_lib as tcli from tcli.tcli_textfsm import clitable - APPEND = tcli.command_parser.APPEND FLAGS = flags.FLAGS @@ -40,14 +40,14 @@ class FakeLock(object): conflict = False was_locked = False - def acquire(self): # pylint: disable=g-bad-name + def acquire(self): # pylint: disable=g-bad-name if self.locked: self.conflict = True else: self.locked = True self.was_locked = True - def release(self): # pylint: disable=g-bad-name + def release(self): # pylint: disable=g-bad-name self.locked = False def __exit__(self, *args, **kwargs): @@ -86,7 +86,8 @@ def setUpClass(cls): tcli.command_response.threading.Lock = mock.MagicMock() tcli.command_response.tqdm = mock.MagicMock() tcli.FLAGS.template_dir = os.path.join( - os.path.dirname(__file__), 'testdata') + os.path.dirname(__file__), 'testdata' + ) return super().setUpClass() @classmethod @@ -96,34 +97,39 @@ def tearDownClass(cls): def setUp(self): # Instantiate FLAGS global var. - tcli.FLAGS([__file__,]) + tcli.FLAGS([ + __file__, + ]) # Turn off looking for .tclirc tcli.FLAGS.color = False tcli.FLAGS.config_file = 'none' tcli.FLAGS.display = 'raw' tcli.FLAGS.filter = None - self.orig_terminal_size = tcli.terminal.TerminalSize - tcli.terminal.TerminalSize = lambda: (10, 20) self.orig_dev_attr = tcli.inventory.DEVICE_ATTRIBUTES tcli.inventory.DEVICE_ATTRIBUTES = {} with mock.patch.object(tcli.inventory, 'Inventory'): self.tcli_obj = tcli.TCLI(interactive=False) - # pylance: ignore - self.tcli_obj.inventory.device_list = ['a', 'b', 'c'] # type: ignore + self.tcli_obj.inventory.device_list = ['a', 'b', 'c'] dev_attr = collections.namedtuple('dev_attr', []) - self.tcli_obj.inventory.devices = { # type: ignore - 'a': dev_attr(), 'b': dev_attr(), 'c': dev_attr()} - self.tcli_obj.inventory.targets = '' # type: ignore - # type: ignore + self.tcli_obj.inventory.devices = { + 'a': dev_attr(), + 'b': dev_attr(), + 'c': dev_attr(), + } + self.tcli_obj.inventory.targets = '' self.tcli_obj._Print = mock.Mock() accessor.SendRequests = mock.Mock() command_register.RegisterCommands(self.tcli_obj, self.tcli_obj.cli_parser) self.tcli_obj.cli_parser.RegisterCommand( - 'somecommand', 'somecommand help', append=True, regexp=True, - handler=lambda command, args, append: (command, args, append)) + 'somecommand', + 'somecommand help', + append=True, + regexp=True, + handler=lambda command, args, append: (command, args, append), + ) self.tcli_obj.verbose = True self.tcli_obj.linewrap = True @@ -131,17 +137,16 @@ def setUp(self): return super().setUp() def tearDown(self): - tcli.terminal.TerminalSize = self.orig_terminal_size tcli.inventory.DEVICE_ATTRIBUTES = self.orig_dev_attr return super().tearDown() def testInlineOnly(self): """Tests that only inline supported commands are ruin when inline.""" - #TODO(harro): Add tests here. + # TODO: Add tests here. pass def testCopy(self): - # TODO(harro): Tests for extended commands? + # TODO: Tests for extended commands? self.tcli_obj.buffers.Append('label', 'content') self.tcli_obj.record = '' inline_tcli = copy.copy(self.tcli_obj) @@ -158,8 +163,7 @@ def testCopy(self): self.assertEqual('content\nmore', self.tcli_obj.buffers.GetBuffer('label')) self.assertEqual('label', self.tcli_obj.record) # Test the child. - self.assertEqual('content\nmore', - inline_tcli.buffers.GetBuffer('label')) + self.assertEqual('content\nmore', inline_tcli.buffers.GetBuffer('label')) self.assertEqual('anotherlabel', inline_tcli.record) def testSetDefaults(self): @@ -192,12 +196,13 @@ def testCallback(self): self.tcli_obj.cmd_response.SetRequest(1, 4) self.tcli_obj.cmd_response._row_response[0] = [] - self.tcli_obj.inventory.device_list = set(['device_a', 'device_b']) # type: ignore - self.tcli_obj.command_list = ['cat alpha', 'cat beta'] # type: ignore + self.tcli_obj.inventory.device_list = set(['device_a', 'device_b']) + self.tcli_obj.command_list = ['cat alpha', 'cat beta'] # Call with valid uid and check response count increments. self.tcli_obj._Callback( - inventory.Response(1, 'device_a', 'cat alpha', 'data', 'error')) + inventory.Response(1, 'device_a', 'cat alpha', 'data', 'error') + ) self.assertTrue(self.tcli_obj.cmd_response._response_count) self.assertEqual(self.tcli_obj.cmd_response._row_response[0], [1]) @@ -206,7 +211,8 @@ def testCallback(self): # Second call for last argument of first row. self.tcli_obj._Callback( - inventory.Response(2, 'device_b', 'cat alpha', 'data', 'error')) + inventory.Response(2, 'device_b', 'cat alpha', 'data', 'error') + ) self.assertEqual(self.tcli_obj.cmd_response._row_response[0], [1, 2]) self.assertFalse(self.tcli_obj.cmd_response._row_response[1]) @@ -219,18 +225,22 @@ def testCallback(self): # Test populating the second row before the first. self.tcli_obj._Callback( - inventory.Response(4, 'device_b', 'cat beta', 'data', 'error')) + inventory.Response(4, 'device_b', 'cat beta', 'data', 'error') + ) self.tcli_obj._Callback( - inventory.Response(3, 'device_a', 'cat beta', 'data', 'error')) + inventory.Response(3, 'device_a', 'cat beta', 'data', 'error') + ) self.assertFalse(self.tcli_obj.cmd_response._row_response[0]) self.assertEqual(self.tcli_obj.cmd_response._row_response[1], [4, 3]) # Once first row gets fully pop'd then both should be reported and cleared. self.tcli_obj._Callback( - inventory.Response(1, 'device_a', 'cat alpha', 'data', 'error')) + inventory.Response(1, 'device_a', 'cat alpha', 'data', 'error') + ) self.tcli_obj._Callback( - inventory.Response(2, 'device_b', 'cat alpha', 'data', 'error')) + inventory.Response(2, 'device_b', 'cat alpha', 'data', 'error') + ) self.assertEqual(self.tcli_obj.cmd_response._row_response[0], [1, 2]) self.assertEqual(self.tcli_obj.cmd_response._row_response[1], [4, 3]) @@ -243,22 +253,35 @@ def testDisplayRaw(self): with mock.patch.object(self.tcli_obj, '_Print') as mock_print: self.tcli_obj._DisplayRaw( inventory.Response( - device_name='device1', command='time of day', - data='a random\nmulti line\nstring.', error='', uid='')) + device_name='device1', + command='time of day', + data='a random\nmulti line\nstring.', + error='', + uid='', + ) + ) mock_print.assert_has_calls([ mock.call('#!# device1:time of day #!#', 'title'), - mock.call('a random\nmulti line\nstring.') + mock.call('a random\nmulti line\nstring.'), ]) self.tcli_obj.cmd_response._results[1] = inventory.Response( - uid=1, device_name='device_1', data='hello world\n', - command='c alpha', error='') + uid=1, + device_name='device_1', + data='hello world\n', + command='c alpha', + error='', + ) self.tcli_obj.cmd_response._results[2] = inventory.Response( - uid=2, device_name='device_2', data='quick fox\n', - command='c alpha', error='') + uid=2, + device_name='device_2', + data='quick fox\n', + command='c alpha', + error='', + ) self.tcli_obj.display = 'raw' - self.tcli_obj.inventory._devices = { # type: ignore + self.tcli_obj.inventory._devices = { 'device_1': {'Vendor', 'Asterix'}, 'device_2': {'Vendor', 'Asterix'}, } @@ -269,10 +292,9 @@ def testDisplayRaw(self): with mock.patch.object(self.tcli_obj, '_Print') as mock_print: # Single entry, raw output. self.tcli_obj._FormatRow([1]) - mock_print.assert_has_calls([ - mock.call(header, 'title'), - mock.call('hello world\n') - ]) + mock_print.assert_has_calls( + [mock.call(header, 'title'), mock.call('hello world\n')] + ) header2 = '#!# %s:%s #!#' % ('device_2', 'c alpha') # Multiple ActionRequest objects, differing content. @@ -282,7 +304,7 @@ def testDisplayRaw(self): mock.call(header, 'title'), mock.call('hello world\n'), mock.call(header2, 'title'), - mock.call('quick fox\n') + mock.call('quick fox\n'), ]) # Multiple action request objects, same content. @@ -292,40 +314,69 @@ def testDisplayRaw(self): mock.call(header, 'title'), mock.call('hello world\n'), mock.call(header, 'title'), - mock.call('hello world\n') + mock.call('hello world\n'), + ]) + + # Out-of-order responses sorted by device_name. + with mock.patch.object(self.tcli_obj, '_Print') as mock_print: + self.tcli_obj._FormatRow([2, 1]) + mock_print.assert_has_calls([ + mock.call(header, 'title'), + mock.call('hello world\n'), + mock.call(header2, 'title'), + mock.call('quick fox\n'), ]) - def _CannedResponse(self): # pylint: disable=invalid-name + def _CannedResponse(self): # pylint: disable=invalid-name """Setup some canned commands and responses.""" - self.tcli_obj.inventory.attributes = { # type: ignore - 'vendor': tcli.inventory.inventory_base.Attribute( # type: ignore - 'vendor', '', None, '', display_case='title')} + self.tcli_obj.inventory.attributes = { + 'vendor': inventory.Attribute( + 'vendor', '', None, '', display_case='title' + ) + } # Initialise the textfsm engine in TCLI. self.tcli_obj.filter = 'default_index' - self.tcli_obj.filter_engine = clitable.CliTable(self.tcli_obj.filter, - tcli.FLAGS.template_dir) + self.tcli_obj.filter_engine = clitable.CliTable( + self.tcli_obj.filter, tcli.FLAGS.template_dir + ) self.tcli_obj.display = 'raw' dev_attr = collections.namedtuple('dev_attr', ['vendor']) - self.tcli_obj.inventory.devices = { # type: ignore + self.tcli_obj.inventory.devices = { 'device_1': dev_attr(vendor='asterix'), 'device_2': dev_attr(vendor='asterix'), 'device_3': dev_attr(vendor='asterix'), - 'device_4': dev_attr(vendor='obelix') + 'device_4': dev_attr(vendor='obelix'), } self.tcli_obj.cmd_response._results[1] = inventory.Response( - uid='1', device_name='device_1', error='', - command='c alpha', data='hello world\n') + uid='1', + device_name='device_1', + error='', + command='c alpha', + data='hello world\n', + ) self.tcli_obj.cmd_response._results[2] = inventory.Response( - uid='2', device_name='device_2', error='', - command='c alpha', data='quick fox\n') + uid='2', + device_name='device_2', + error='', + command='c alpha', + data='quick fox\n', + ) self.tcli_obj.cmd_response._results[3] = inventory.Response( - uid=3, device_name='device_3', error='', - command='cat epsilon', data='jumped over\n') + uid=3, + device_name='device_3', + error='', + command='cat epsilon', + data='jumped over\n', + ) self.tcli_obj.cmd_response._results[4] = inventory.Response( - uid=4, device_name='device_4', error='', - command='cat epsilon', data='the wall\n') + uid=4, + device_name='device_4', + error='', + command='cat epsilon', + data='the wall\n', + ) def testFormatResponse(self): """Tests format of command results - Single entry, csv format.""" @@ -341,7 +392,7 @@ def testFormatResponse(self): self.tcli_obj._FormatRow([1]) mock_print.assert_has_calls([ mock.call(header, 'title'), - mock.call('Host, ColAa, ColAb\ndevice_1, hello, world\n') + mock.call('Host, ColAa, ColAb\ndevice_1, hello, world\n'), ]) def testFormatResponseCSV1(self): @@ -358,9 +409,11 @@ def testFormatResponseCSV1(self): self.tcli_obj._FormatRow([1, 2]) mock_print.assert_has_calls([ mock.call(header, 'title'), - mock.call('Host, ColAa, ColAb\n' - 'device_1, hello, world\n' - 'device_2, quick, fox\n') + mock.call( + 'Host, ColAa, ColAb\n' + 'device_1, hello, world\n' + 'device_2, quick, fox\n' + ), ]) def testFormatResponseCSV2(self): @@ -377,8 +430,7 @@ def testFormatResponseCSV2(self): self.tcli_obj._FormatRow([3]) mock_print.assert_has_calls([ mock.call(header, 'title'), - mock.call('Host, ColCa, ColCb\n' - 'device_3, jumped, over\n') + mock.call('Host, ColCa, ColCb\ndevice_3, jumped, over\n'), ]) def testFormatResponseCSV3(self): @@ -395,9 +447,11 @@ def testFormatResponseCSV3(self): self.tcli_obj._FormatRow([3, 3]) mock_print.assert_has_calls([ mock.call(header, 'title'), - mock.call('Host, ColCa, ColCb\n' - 'device_3, jumped, over\n' - 'device_3, jumped, over\n') + mock.call( + 'Host, ColCa, ColCb\n' + 'device_3, jumped, over\n' + 'device_3, jumped, over\n' + ), ]) # Multiple entry - Vendor 'Obelix'. @@ -405,9 +459,9 @@ def testFormatResponseCSV3(self): self.tcli_obj._FormatRow([4, 4]) mock_print.assert_has_calls([ mock.call(header, 'title'), - mock.call('Host, ColDa, ColDb\n' - 'device_4, the, wall\n' - 'device_4, the, wall\n') + mock.call( + 'Host, ColDa, ColDb\ndevice_4, the, wall\ndevice_4, the, wall\n' + ), ]) # Multiple entry - Mixed vendors. @@ -415,12 +469,14 @@ def testFormatResponseCSV3(self): self.tcli_obj._FormatRow([3, 4, 3, 4]) mock_print.assert_has_calls([ mock.call(header, 'title'), - mock.call('Host, ColCa, ColCb\n' - 'device_3, jumped, over\n' - 'device_3, jumped, over\n'), - mock.call('Host, ColDa, ColDb\n' - 'device_4, the, wall\n' - 'device_4, the, wall\n') + mock.call( + 'Host, ColCa, ColCb\n' + 'device_3, jumped, over\n' + 'device_3, jumped, over\n' + ), + mock.call( + 'Host, ColDa, ColDb\ndevice_4, the, wall\ndevice_4, the, wall\n' + ), ]) def testFormatResponseVarz(self): @@ -440,62 +496,78 @@ def testFormatResponseVarz(self): mock_print.assert_has_calls([ mock.call(header, 'title'), mock.call( - nvp_label + '\n' - + 'device_1.ColAa hello\n' - 'device_1.ColAb world\n') + nvp_label + '\n' + 'device_1.ColAa hello\ndevice_1.ColAb world\n' + ), ]) def testFormatResponseGsh(self): """Tests format of command results - Gsh format.""" - # GSH formatted. + # TBL formatted. self.tcli_obj.display = 'tbl' - tcli.terminal.TerminalSize = lambda: (24, 10) - with mock.patch.object(self.tcli_obj, '_Print') as mock_print: - # Displays warning if width too narrow. - self.tcli_obj._FormatRow([1]) - mock_print.assert_called_once() + self.tcli_obj.filter = 'default' + self.tcli_obj.filter_engine = clitable.CliTable( + 'default_index', template_dir=tcli.FLAGS.template_dir + ) + self.tcli_obj.cmd_response._results[1] = inventory.Response( + uid=1, + device_name='a', + data='hello world\n', + command='c alpha', + error='', + ) + with mock.patch.object(shutil, 'get_terminal_size', return_value=(24, 1)): + with mock.patch.object(self.tcli_obj, '_Print') as mock_print: + # Displays warning if width too narrow. + self.tcli_obj._FormatRow([1]) + mock_print.assert_has_calls([ + mock.call('#!# c alpha #!#', 'title'), + mock.call('Width too narrow to display table.', msgtype='warning'), + ]) def testParseCommands(self): """Tests that commands are supplied to CmdRequests.""" - #TODO(harro): Better tests here. + # TODO: Better tests here. with mock.patch.object(self.tcli_obj, '_CmdRequests') as mock_request: # Single command. self.tcli_obj._ParseCommands('cat alpha') mock_request.assert_called_once_with( - self.tcli_obj.device_list, ['cat alpha']) + self.tcli_obj.device_list, ['cat alpha'] + ) with mock.patch.object(self.tcli_obj, '_CmdRequests') as mock_request: # Multiple commands. self.tcli_obj._ParseCommands('cat alpha\ncat alpha\ncat beta') mock_request.assert_called_once_with( - self.tcli_obj.device_list, - ['cat alpha', 'cat alpha', 'cat beta']) + self.tcli_obj.device_list, ['cat alpha', 'cat alpha', 'cat beta'] + ) # Multiple commands, with extra whitespace. with mock.patch.object(self.tcli_obj, '_CmdRequests') as mock_request: self.tcli_obj._ParseCommands(' cat alpha\n\n\ncat alpha \ncat beta ') mock_request.assert_called_once_with( - self.tcli_obj.device_list, - ['cat alpha', 'cat alpha', 'cat beta']) + self.tcli_obj.device_list, ['cat alpha', 'cat alpha', 'cat beta'] + ) # Mixed commands some for the device some TCLI commands. with mock.patch.object(self.tcli_obj, '_CmdRequests') as mock_request: - self.tcli_obj._ParseCommands(' cat alpha \n %shelp \n\n\n%scolor ' % - (tcli.SLASH, tcli.SLASH)) + self.tcli_obj._ParseCommands( + ' cat alpha \n %shelp \n\n\n%scolor ' % (tcli.SLASH, tcli.SLASH) + ) mock_request.assert_called_once_with( - self.tcli_obj.device_list, - ['cat alpha']) + self.tcli_obj.device_list, ['cat alpha'] + ) # Mixed commands with some inline TCLI commands. with mock.patch.object(self.tcli_obj, '_CmdRequests') as mock_request: - self.tcli_obj._ParseCommands(' cat alpha\n%scolor\n\nc alpha ' % - tcli.SLASH) + self.tcli_obj._ParseCommands( + ' cat alpha\n%scolor\n\nc alpha ' % tcli.SLASH + ) mock_request.assert_has_calls([ mock.call(self.tcli_obj.device_list, ['cat alpha']), - mock.call(self.tcli_obj.device_list, ['c alpha']) - ]) + mock.call(self.tcli_obj.device_list, ['c alpha']), + ]) def testBufferInUse(self): """Tests _BufferInUse function.""" @@ -524,11 +596,13 @@ def testNumberofArgs(self): with mock.patch.object(self.tcli_obj, '_Print') as mock_print: self.tcli_obj._TCLICmd('help badarg') mock_print.assert_called_once_with( - 'Invalid number of arguments, found: 1.', msgtype='warning') + 'Invalid number of arguments, found: 1.', msgtype='warning' + ) self.tcli_obj._TCLICmd('buffer boo badarg') mock_print.assert_called_with( - 'Invalid number of arguments, found: 2.', msgtype='warning') + 'Invalid number of arguments, found: 2.', msgtype='warning' + ) def testTCLIExit(self): @@ -557,7 +631,8 @@ def testTCLIColor(self): with mock.patch.object(self.tcli_obj, '_Print') as mock_print: self.tcli_obj._TCLICmd('color bogus') mock_print.assert_called_once_with( - "Error: Argument must be 'on' or 'off'.", msgtype='warning') + "Error: Argument must be 'on' or 'off'.", msgtype='warning' + ) def testSafeMode(self): """Tests safemode toggle.""" @@ -623,10 +698,10 @@ def testTCLIMode(self): self.assertEqual('gated', self.tcli_obj.mode) def testTCLICmd(self): - """b/2303768 Truncation of characters in /command.""" + """Truncation of characters in /command.""" - self.tcli_obj.inventory.targets = '' # type: ignore - self.tcli_obj.inventory.device_list = set() # type: ignore + self.tcli_obj.inventory.targets = '' + self.tcli_obj.inventory.device_list = set() cmd = 'cat bogus' with mock.patch.object(self.tcli_obj, '_CmdRequests') as mock_request: self.tcli_obj._TCLICmd('command %s' % cmd) @@ -636,8 +711,7 @@ def testFormatBadTable(self): # A bad format. self.tcli_obj.display = 'notaformat' - self.assertRaises(tcli.TcliCmdError, - self.tcli_obj._DisplayFormatted, 'boo') + self.assertRaises(tcli.TcliCmdError, self.tcli_obj._DisplayFormatted, 'boo') def testAlphaNumBuffer(self): @@ -646,28 +720,32 @@ def testAlphaNumBuffer(self): self.tcli_obj._TCLICmd('buffer a.b') # Argument to buffer must be an alphanum. mock_print.assert_called_once_with( - 'Arguments with alphanumeric characters only.', msgtype='warning') + 'Arguments with alphanumeric characters only.', msgtype='warning' + ) def testTCLIFilter(self): """Tests setting filter via cli.""" self.tcli_obj.filter_engine = None self.tcli_obj._TCLICmd('filter default_index') - self.assertTrue(self.tcli_obj.filter_engine) + self.assertIsNotNone(self.tcli_obj.filter_engine) - self.tcli_obj.filter_engine.template_dir = tcli.FLAGS.template_dir # type: ignore - self.tcli_obj.filter_engine.ReadIndex('default_index') # type: ignore + self.tcli_obj.filter_engine.template_dir = tcli.FLAGS.template_dir + self.tcli_obj.filter_engine.ReadIndex('default_index') - self.tcli_obj.filter_engine.ParseCmd( # type: ignore - 'two words', attributes={'Command': 'cat eps', 'Vendor': 'Asterix'}) + self.tcli_obj.filter_engine.ParseCmd( + 'two words', attributes={'Command': 'cat eps', 'Vendor': 'Asterix'} + ) self.assertEqual( - 'ColCa, ColCb\ntwo, words\n', self.tcli_obj.filter_engine.table) # type: ignore + 'ColCa, ColCb\ntwo, words\n', str(self.tcli_obj.filter_engine.table) + ) # Bad filter value. with mock.patch.object(self.tcli_obj, '_Print') as mock_print: self.tcli_obj._TCLICmd('filter not_a_valid_filter') mock_print.assert_called_once_with( - "Invalid filter 'not_a_valid_filter'.", msgtype='warning') + "Invalid filter 'not_a_valid_filter'.", msgtype='warning' + ) def testTCLIBufferAsignment(self): @@ -773,10 +851,11 @@ def testTCLIBufferRecord(self): # A valid SLASH command. self.tcli_obj._TCLICmd('help') self.assertEqual( - 'A test\nA two\nline test', - self.tcli_obj.buffers.GetBuffer('hello')) - mock_print.assert_any_call("Invalid escape command 'an'.", - msgtype='warning') + 'A test\nA two\nline test', self.tcli_obj.buffers.GetBuffer('hello') + ) + mock_print.assert_any_call( + "Invalid escape command 'an'.", msgtype='warning' + ) # Record and append. self.tcli_obj.record = '' @@ -785,7 +864,8 @@ def testTCLIBufferRecord(self): self.tcli_obj._ParseCommands('Append again\non two lines') self.assertEqual( 'A test\nA two\nline test\nAppend test\nAppend again\non two lines', - self.tcli_obj.buffers.GetBuffer('hello')) + self.tcli_obj.buffers.GetBuffer('hello'), + ) # Stop and restart recording. Buffer should be cleared. self.tcli_obj._TCLICmd('logstop hello') @@ -812,7 +892,8 @@ def testTCLIBufferRecord(self): self.tcli_obj._TCLICmd('buffer hello') self.assertEqual( 'A test\nA two\nline test\n%sbuffer hello' % tcli.SLASH, - self.tcli_obj.buffers.GetBuffer('hello')) + self.tcli_obj.buffers.GetBuffer('hello'), + ) self.tcli_obj.record = self.tcli_obj.recordall = '' self.tcli_obj.log = self.tcli_obj.logall = '' @@ -826,10 +907,12 @@ def testTCLIBufferRecord(self): self.tcli_obj._ParseCommands('A two\nline test') self.assertEqual( '%slogall world\nA test' % tcli.SLASH, - self.tcli_obj.buffers.GetBuffer('hello')) + self.tcli_obj.buffers.GetBuffer('hello'), + ) self.assertEqual( 'A test\n%slogstop hello\nA two\nline test' % tcli.SLASH, - self.tcli_obj.buffers.GetBuffer('world')) + self.tcli_obj.buffers.GetBuffer('world'), + ) self.assertEqual('world', self.tcli_obj.logall) self.tcli_obj.record = self.tcli_obj.recordall = '' @@ -851,7 +934,7 @@ def testTCLIBufferRecord(self): def testTCLIBufferLog(self): """Tests logging of commands to a buffer.""" - self.tcli_obj.inventory.device_list = [] # type: ignore + self.tcli_obj.inventory.device_list = [] with mock.patch.object(self.tcli_obj, '_Print') as mock_print: # Record commands but not escape commands. self.tcli_obj._TCLICmd('log hello') @@ -861,10 +944,11 @@ def testTCLIBufferLog(self): # A valid SLASH command. self.tcli_obj._TCLICmd('color') self.assertEqual( - 'A test\nA two\nline test', - self.tcli_obj.buffers.GetBuffer('hello')) - mock_print.assert_called_once_with("Invalid escape command 'an'.", - msgtype='warning') + 'A test\nA two\nline test', self.tcli_obj.buffers.GetBuffer('hello') + ) + mock_print.assert_called_once_with( + "Invalid escape command 'an'.", msgtype='warning' + ) self.tcli_obj.record = '' self.tcli_obj.recordall = '' @@ -880,7 +964,8 @@ def testTCLIBufferLog(self): self.tcli_obj._TCLICmd('color') self.assertEqual( 'A test\nA two\nline test\n%scolor' % tcli.SLASH, - self.tcli_obj.buffers.GetBuffer('hello')) + self.tcli_obj.buffers.GetBuffer('hello'), + ) def testDisplayBufname(self): """Tests that buffer name is displayed with logall.""" @@ -888,8 +973,9 @@ def testDisplayBufname(self): with mock.patch.object(self.tcli_obj, '_Print') as mock_print: self.tcli_obj.logall = 'hello' self.tcli_obj._TCLICmd('logall') - mock_print.assert_called_once_with("'logall' buffer is 'hello'", - msgtype='system') + mock_print.assert_called_once_with( + "'logall' buffer is 'hello'", msgtype='system' + ) def testTCLITimeout(self): """Tests setting the timeout value.""" @@ -901,8 +987,9 @@ def testTCLITimeout(self): with mock.patch.object(self.tcli_obj, '_Print') as mock_print: self.tcli_obj._TCLICmd('timeout a') self.assertEqual(10, self.tcli_obj.timeout) - mock_print.assert_called_once_with("Invalid timeout value 'a'.", - msgtype='warning') + mock_print.assert_called_once_with( + "Invalid timeout value 'a'.", msgtype='warning' + ) # Accepts only positive whole integers self.tcli_obj._TCLICmd('timeout 15.1') @@ -935,16 +1022,19 @@ def testTCLIBufferRecursivePlay1(self): self.tcli_obj.buffers.Append('boo', '%splay boo' % tcli.SLASH) self.tcli_obj._TCLICmd('play boo') mock_print.assert_called_once_with( - 'Recursive call of "play" rejected.', msgtype='warning') + 'Recursive call of "play" rejected.', msgtype='warning' + ) def testTCLIBufferRecursivePlay2(self): """Cannot assign buffer while playing out the content.""" with mock.patch.object(self.tcli_obj, '_Print') as mock_print: - self.tcli_obj.buffers.Append('boo', '%srecordall boo\n%scolor' % ( - tcli.SLASH, tcli.SLASH)) + self.tcli_obj.buffers.Append( + 'boo', '%srecordall boo\n%scolor' % (tcli.SLASH, tcli.SLASH) + ) self.tcli_obj._TCLICmd('play boo') mock_print.assert_called_once_with( - "Buffer: 'boo', already open by 'play' command.", msgtype='warning') + "Buffer: 'boo', already open by 'play' command.", msgtype='warning' + ) self.assertEqual(self.tcli_obj.recordall, '') def testTCLIBufferRecursivePlay3(self): @@ -954,7 +1044,8 @@ def testTCLIBufferRecursivePlay3(self): self.tcli_obj.buffers.Append('boo', '%scolor' % tcli.SLASH) self.tcli_obj._TCLICmd('play boo') mock_print.assert_called_once_with( - "Buffer: 'boo', already open for writing.", msgtype='warning') + "Buffer: 'boo', already open for writing.", msgtype='warning' + ) def testTCLIBufferRecursivePlay4(self): """Tests we are able log to a different buffer to what we play out from.""" @@ -963,8 +1054,10 @@ def testTCLIBufferRecursivePlay4(self): self.tcli_obj.recordall = 'hoo' self.tcli_obj.buffers.Append('boo', '%scolor' % tcli.SLASH) self.tcli_obj._TCLICmd('play boo') - self.assertEqual('%splay boo\n%scolor' % (tcli.SLASH, tcli.SLASH), - self.tcli_obj.buffers.GetBuffer('hoo')) + self.assertEqual( + '%splay boo\n%scolor' % (tcli.SLASH, tcli.SLASH), + self.tcli_obj.buffers.GetBuffer('hoo'), + ) # Color value was toggled. self.tcli_obj.color = False mock_print.assert_has_calls([]) @@ -976,15 +1069,17 @@ def testTCLIBuffer(self): self.tcli_obj._TCLICmd('buffer boo') mock_print.assert_has_calls([ - mock.call('#! BUFFER boo !#', msgtype='warning'), - mock.call('hello\nworld', msgtype='system'), - mock.call('#! ENDBUFFER !#', msgtype='warning')]) + mock.call('#! BUFFER boo !#', msgtype='warning'), + mock.call('hello\nworld', msgtype='system'), + mock.call('#! ENDBUFFER !#', msgtype='warning'), + ]) # Invalid buffer name displays null content. mock_print.reset_mock() self.tcli_obj._TCLICmd('buffer non_exist') - mock_print.assert_called_once_with('Invalid buffer name "non_exist".', - msgtype='warning') + mock_print.assert_called_once_with( + 'Invalid buffer name "non_exist".', msgtype='warning' + ) def testTCLIClear(self): """Tests that buffer contents are cleared.""" @@ -996,28 +1091,110 @@ def testTCLIClear(self): # Clearing a nonexistant buffer fails silently. self.tcli_obj._TCLICmd('clear non_exist') - def testTCLIExpandTargets(self): """Tests target expansion.""" with mock.patch.object(self.tcli_obj, '_Print') as mock_print: - self.tcli_obj.inventory.device_list = ['device_a', 'device_b'] # type: ignore + self.tcli_obj.inventory.device_list = ['device_a', 'device_b'] self.tcli_obj._TCLICmd('expandtargets') mock_print.assert_called_once_with('device_a,device_b', msgtype='system') def testAdvancedRegexpTargets(self): - """b/2725704 tcli fails to parse regexes.""" + """Tcli fails to parse regexes.""" # pylint: disable=anomalous-backslash-in-string self.assertEqual( ('somecommand', [r'^br\d{2}.*'], False), - self.tcli_obj.cli_parser.ParseCommandLine(r"somecommand '^br\d{2}.*'")) + self.tcli_obj.cli_parser.ParseCommandLine(r"somecommand '^br\d{2}.*'"), + ) # Complex TCLI commands targets/xtargets must be quoted. # pylint: disable=anomalous-backslash-in-string self.assertEqual( ('somecommand', [r'^brd{2}.*'], False), - self.tcli_obj.cli_parser.ParseCommandLine(r'somecommand ^br\d{2}.*')) + self.tcli_obj.cli_parser.ParseCommandLine(r'somecommand ^br\d{2}.*'), + ) + + def testInlineInventoryCommandsIsolation(self): + """Tests that inline inventory filters execute without mutating parent.""" + dev_cls = collections.namedtuple('Device', ['vendor']) + with mock.patch.object(inventory.Inventory, 'Load'): + real_inv = inventory.Inventory() + real_inv._devices = { + 'dev1': dev_cls(vendor='cisco'), + 'dev2': dev_cls(vendor='juniper'), + } + tcli_inst = tcli.TCLI(interactive=False, inv=real_inv) + tcli_inst._TCLICmd('targets dev1') + self.assertEqual(['dev1'], tcli_inst.device_list) + + with mock.patch.object(tcli.TCLI, '_CmdRequests') as mock_requests: + tcli_inst._ParseCommands('show version //targets dev2') + mock_requests.assert_called_once_with(['dev2'], ['show version']) + + # Parent inventory remains unchanged. + self.assertEqual('dev1', tcli_inst.inventory.targets) + self.assertEqual(['dev1'], tcli_inst.device_list) + + def testCmdReadAppendAndOverwrite(self): + """Tests /read overwrites buffer and /read+ appends to buffer.""" + self.tcli_obj.buffers.Append('buf1', 'initial') + with tempfile.NamedTemporaryFile('w', encoding='utf-8', suffix='.txt') as f: + f.write('from_file') + f.flush() + # /read (append=False) clears before reading. + self.tcli_obj._CmdRead('read', ['buf1', f.name], append=False) + self.assertEqual('from_file', self.tcli_obj.buffers.GetBuffer('buf1')) + + # /read+ (append=True) appends to existing content. + self.tcli_obj._CmdRead('read', ['buf1', f.name], append=True) + self.assertEqual( + 'from_file\nfrom_file', self.tcli_obj.buffers.GetBuffer('buf1') + ) + + def testCmdEditorUsesSubprocessAndUtf8(self): + """Tests _CmdEditor invokes subprocess.run without shell and handles UTF-8.""" + self.tcli_obj.buffers.Append('buf_utf8', 'café') + with mock.patch.object(tcli.subprocess, 'run') as mock_run: + self.tcli_obj._CmdEditor('vi', ['buf_utf8'], append=False) + mock_run.assert_called_once() + cmd_args = mock_run.call_args[0][0] + self.assertEqual( + ['vi', '-Z', '-n', '-u', 'NONE', '-U', 'NONE', '--'], cmd_args[:-1] + ) + self.assertEqual('café', self.tcli_obj.buffers.GetBuffer('buf_utf8')) + + def testCmdDefaultsErrorMessageInterpolation(self): + """Tests _CmdDefaults formats unknown command name in ValueError.""" + with self.assertRaisesRegex( + ValueError, "Cannot set 'bogus_cmd' to defaults." + ): + self.tcli_obj._CmdDefaults('defaults', ['bogus_cmd'], append=False) + + def testRestrictedModeBlocksShellEditorAndWrite(self): + """Tests --restricted flag blocks /exec, /vi, /write, and pipes.""" + self.tcli_obj.buffers.Append('buf1', 'some content') + tcli.FLAGS.restricted = True + try: + with self.assertRaisesRegex( + ValueError, 'Command disabled in restricted mode.' + ): + self.tcli_obj._CmdExecShell('exec', ['ls'], False) + with self.assertRaisesRegex( + ValueError, 'Command disabled in restricted mode.' + ): + self.tcli_obj._CmdEditor('vi', ['buf1'], False) + with self.assertRaisesRegex( + ValueError, 'Command disabled in restricted mode.' + ): + self.tcli_obj._CmdWrite('write', ['buf1', '/tmp/out'], False) + with mock.patch.object(self.tcli_obj, '_Print') as mock_print: + self.assertIsNone(self.tcli_obj._Pipe('data', '| cat')) + mock_print.assert_called_once_with( + 'Shell pipes are disabled in restricted mode.', msgtype='warning' + ) + finally: + tcli.FLAGS.restricted = False if __name__ == '__main__': diff --git a/tcli/tcli_textfsm/clitable.py b/tcli/tcli_textfsm/clitable.py index 82a6624..85953dc 100644 --- a/tcli/tcli_textfsm/clitable.py +++ b/tcli/tcli_textfsm/clitable.py @@ -14,15 +14,23 @@ """Clitable that supports additional TextFSM Value Attributes.""" - from absl import flags from tcli.tcli_textfsm import textfsm from textfsm import clitable from textfsm import texttable +# pylint: disable=g-importing-member from textfsm.clitable import CliTableError from textfsm.clitable import Error -from textfsm.clitable import IndexTableError # pylint: disable=unused-import +from textfsm.clitable import IndexTableError +__all__ = [ + 'CliTable', + 'CliTableError', + 'DeviceError', + 'Error', + 'IndexTableError', + 'QueryError', +] FLAGS = flags.FLAGS @@ -51,6 +59,7 @@ class CliTable(clitable.CliTable): template_dir: String, directory where index file and templates reside. """ + # pylint: disable=arguments-renamed def ParseCmd(self, cmd_input, attributes=None, templates=None, verbose=True): """Creates a TextTable table of values from cmd_input string. @@ -75,8 +84,9 @@ def ParseCmd(self, cmd_input, attributes=None, templates=None, verbose=True): if row_idx: templates = self.index.index[row_idx]['Template'] else: - raise CliTableError('No template found for attributes: "%s"' % - attributes) + raise CliTableError( + 'No template found for attributes: "%s"' % attributes + ) template_files = self._TemplateNamesToFiles(templates) @@ -84,13 +94,16 @@ def ParseCmd(self, cmd_input, attributes=None, templates=None, verbose=True): # Re-initialise the table. self.Reset() self._keys = set() - self.table = self._ParseCmdItem(self.raw, verbose=verbose, - template_file=template_files[0]) + self.table = self._ParseCmdItem( + self.raw, verbose=verbose, template_file=template_files[0] + ) # Add additional columns from any additional tables. for tmplt in template_files[1:]: - self.extend(self._ParseCmdItem(self.raw, verbose=verbose, - template_file=tmplt), set(self._keys)) + self.extend( + self._ParseCmdItem(self.raw, verbose=verbose, template_file=tmplt), + set(self._keys), + ) finally: for f in template_files: f.close() @@ -122,3 +135,5 @@ def _ParseCmdItem(self, cmd_input, template_file=None, verbose=True): for record in fsm.ParseText(cmd_input): table.Append(record) return table + + diff --git a/tcli/tcli_textfsm/textfsm.py b/tcli/tcli_textfsm/textfsm.py index c2b4954..295adff 100644 --- a/tcli/tcli_textfsm/textfsm.py +++ b/tcli/tcli_textfsm/textfsm.py @@ -25,13 +25,21 @@ https://github.com/google/textfsm """ - import json from absl import flags import textfsm -from textfsm import TextFSMError # pylint: disable=unused-import -from textfsm import TextFSMTemplateError # pylint: disable=unused-import -from textfsm import Usage # pylint: disable=unused-import +# pylint: disable=g-importing-member +from textfsm import TextFSMError +from textfsm import TextFSMTemplateError +from textfsm import UsageError + +__all__ = [ + 'TextFSM', + 'TextFSMOptions', + 'TextFSMError', + 'TextFSMTemplateError', + 'UsageError', +] FLAGS = flags.FLAGS @@ -45,7 +53,8 @@ class Blank(textfsm.TextFSMOptions.OptionBase): def OnCreateOptions(self): if len(self.value.options) != 1: raise textfsm.TextFSMTemplateError( - 'Blank is a mutually exclusive option.') + 'Blank is a mutually exclusive option.' + ) def OnSaveRecord(self): self.value.value = '' @@ -56,7 +65,8 @@ class Verbose(textfsm.TextFSMOptions.OptionBase): def OnCreateOptions(self): if 'Key' in self.value.OptionNames(): raise textfsm.TextFSMTemplateError( - 'Options "Key" and "Verbose" are mutually exclusive') + 'Options "Key" and "Verbose" are mutually exclusive' + ) def OnGetValue(self): # pylint: disable=protected-access @@ -74,7 +84,8 @@ class Key(textfsm.TextFSMOptions.OptionBase): def OnCreateOptions(self): if 'Verbose' in self.value.OptionNames(): raise textfsm.TextFSMTemplateError( - 'Options "Key" and "Verbose" are mutually exclusive') + 'Options "Key" and "Verbose" are mutually exclusive' + ) class TextFSM(textfsm.TextFSM): @@ -83,8 +94,7 @@ class TextFSM(textfsm.TextFSM): Options useful for formatting and displaying of the data. """ - def __init__(self, template, verbose=True, - options_class=TextFSMOptions): + def __init__(self, template, verbose=True, options_class=TextFSMOptions): """Initialises and also parses the template file.""" # Display 'all' or 'only some' of the columns. @@ -105,4 +115,4 @@ def Dump(self): output = [json.dumps(self.header, separators=(',', ':'))] for row in self._result: output.append(json.dumps(row, separators=(',', ':'))) - return '\n'.join(output) + return '\n'.join(output) diff --git a/tcli/text_buffer.py b/tcli/text_buffer.py index 5b1a33f..00267f2 100644 --- a/tcli/text_buffer.py +++ b/tcli/text_buffer.py @@ -33,7 +33,8 @@ def __init__(self) -> None: def Append(self, text_buffer: str, line: str) -> None: """Append row of text to buffer.""" # Accept and silently discard empty buffer name or data. - if not text_buffer or not line: return + if not text_buffer or not line: + return if self._buffers[text_buffer]: self._buffers[text_buffer] += '\n' + line @@ -50,7 +51,8 @@ def Clear(self, text_buffer: str) -> bool: def GetBuffer(self, text_buffer: str) -> str: """Returns content of named buffer if it exists.""" content = self._buffers.get(text_buffer) - if content is None: raise AttributeError(f"No Buffer: '{text_buffer}'.") + if content is None: + raise AttributeError(f"No Buffer: '{text_buffer}'.") return content def ListBuffers(self) -> str: diff --git a/tcli/text_buffer_test.py b/tcli/text_buffer_test.py index ad0b2c3..e4f9cd2 100644 --- a/tcli/text_buffer_test.py +++ b/tcli/text_buffer_test.py @@ -12,7 +12,7 @@ # implied. See the License for the specific language governing # permissions and limitations under the License. -"""Tests for tcli.text_buffer.""" +"""Unit tests for the text_buffer module.""" import unittest @@ -55,7 +55,7 @@ def testClear(self): def testGetBuffer(self): self.buf._buffers['boo'] = 'hello\nworld' self.assertEqual('hello\nworld', self.buf.GetBuffer('boo')) - self.assertRaises(AttributeError, self.buf.GetBuffer,'non_exist') + self.assertRaises(AttributeError, self.buf.GetBuffer, 'non_exist') def testListBuffer(self): self.buf._buffers['boo'] = 'hello\nworld' diff --git a/tcli/textfsm_test.py b/tcli/textfsm_test.py index 9ee0268..f700b19 100644 --- a/tcli/textfsm_test.py +++ b/tcli/textfsm_test.py @@ -14,14 +14,12 @@ """Unittest for textfsm module.""" +from io import StringIO # pylint: disable=g-importing-member import unittest -from io import StringIO # pylint: disable=g-importing-member from absl import flags - from tcli.tcli_textfsm import textfsm - FLAGS = flags.FLAGS @@ -30,34 +28,43 @@ class UnitTestFSM(unittest.TestCase): def testKey(self): - tplt = ('Value Required boo (on.)\n' - 'Value Required,Key hoo (on.)\n\n' - 'Start\n ^$boo -> Continue\n ^$hoo -> Record') + tplt = ( + 'Value Required boo (on.)\n' + 'Value Required,Key hoo (on.)\n\n' + 'Start\n ^$boo -> Continue\n ^$hoo -> Record' + ) # Explicit default. t = textfsm.TextFSM(StringIO(tplt)) - self.assertTrue(t._GetValue('hoo')) - self.assertIn('Key', t._GetValue('hoo').OptionNames()) # type: ignore - self.assertTrue(t._GetValue('boo')) - self.assertNotIn('Key', t._GetValue('boo').OptionNames()) # type: ignore - - tplt = ('Value Required boo (on.)\n' - 'Value Verbose,Key hoo (on.)\n\n' - 'Start\n ^$boo -> Continue\n ^$hoo -> Record') - self.assertRaises(textfsm.textfsm.TextFSMTemplateError, textfsm.TextFSM, - StringIO(tplt)) + hoo_val = t._GetValue('hoo') + self.assertIsNotNone(hoo_val) + self.assertIn('Key', hoo_val.OptionNames()) + boo_val = t._GetValue('boo') + self.assertIsNotNone(boo_val) + self.assertNotIn('Key', boo_val.OptionNames()) + + tplt = ( + 'Value Required boo (on.)\n' + 'Value Verbose,Key hoo (on.)\n\n' + 'Start\n ^$boo -> Continue\n ^$hoo -> Record' + ) + self.assertRaises( + textfsm.textfsm.TextFSMTemplateError, textfsm.TextFSM, StringIO(tplt) + ) def testVerbose(self): # Suppress second column. - tplt = ('Value Required boo (on.)\n' - 'Value Filldown,Required,Verbose hoo (on.)\n\n' - 'Start\n ^$boo -> Continue\n ^$hoo -> Record') + tplt = ( + 'Value Required boo (on.)\n' + 'Value Filldown,Required,Verbose hoo (on.)\n\n' + 'Start\n ^$boo -> Continue\n ^$hoo -> Record' + ) t = textfsm.TextFSM(StringIO(tplt), verbose=False) data = 'one\non0' result = t.ParseText(data) - self.assertEqual(str(result), ("[['one'], ['on0']]")) + self.assertEqual(str(result), "[['one'], ['on0']]") # The headers displayed is less too. self.assertEqual(t.header, ['boo']) @@ -65,12 +72,14 @@ def testVerbose(self): t = textfsm.TextFSM(StringIO(tplt), verbose=True) data = 'one\non0' result = t.ParseText(data) - self.assertEqual(str(result), ("[['one', 'one'], ['on0', 'on0']]")) + self.assertEqual(str(result), "[['one', 'one'], ['on0', 'on0']]") # Required Verbose fields still apply. - tplt = ('Value Required boo (on.)\n' - 'Value Required,Verbose hoo (tw.)\n\n' - 'Start\n ^$boo -> Continue\n ^$hoo -> Record') + tplt = ( + 'Value Required boo (on.)\n' + 'Value Required,Verbose hoo (tw.)\n\n' + 'Start\n ^$boo -> Continue\n ^$hoo -> Record' + ) t = textfsm.TextFSM(StringIO(tplt), verbose=False) data = 'one\non0' @@ -80,29 +89,36 @@ def testVerbose(self): def testBlank(self): # Value may be matched by 'boo' but are never retained. - tplt = ('Value Required boo (one)\n' - 'Value Blank hoo (two)\n\n' - 'Start\n ^$boo $hoo -> Record') + tplt = ( + 'Value Required boo (one)\n' + 'Value Blank hoo (two)\n\n' + 'Start\n ^$boo $hoo -> Record' + ) t = textfsm.TextFSM(StringIO(tplt)) data = 'one two' result = t.ParseText(data) - self.assertEqual(str(result), ("[['one', '']]")) + self.assertEqual(str(result), "[['one', '']]") # The headers displayed is for both. self.assertEqual(t.header, ['boo', 'hoo']) # Blank values can have no other options - tplt = ('Value Required boo (one)\n' - 'Value Blank,Verbose hoo (two)\n\n' - 'Start\n ^$boo $hoo -> Record') - self.assertRaises(textfsm.textfsm.TextFSMTemplateError, textfsm.TextFSM, - StringIO(tplt)) + tplt = ( + 'Value Required boo (one)\n' + 'Value Blank,Verbose hoo (two)\n\n' + 'Start\n ^$boo $hoo -> Record' + ) + self.assertRaises( + textfsm.textfsm.TextFSMTemplateError, textfsm.TextFSM, StringIO(tplt) + ) def testError(self): - tplt = ('Value Required boo (on.)\n' - 'Value Filldown,Required hoo (on.)\n\n' - 'Start\n ^$boo -> Continue\n ^$hoo -> Error') + tplt = ( + 'Value Required boo (on.)\n' + 'Value Filldown,Required hoo (on.)\n\n' + 'Start\n ^$boo -> Continue\n ^$hoo -> Error' + ) t = textfsm.TextFSM(StringIO(tplt)) data = 'one' @@ -110,9 +126,11 @@ def testError(self): self.assertRaises(textfsm.TextFSMError, t.ParseText, data) def testDump(self): - tplt = ('Value Required boo (on.)\n' - 'Value Required hoo (of.)\n\n' - 'Start\n ^${boo}\n ^${hoo} -> Record') + tplt = ( + 'Value Required boo (on.)\n' + 'Value Required hoo (of.)\n\n' + 'Start\n ^${boo}\n ^${hoo} -> Record' + ) data = 'one\noff\n' t = textfsm.TextFSM(StringIO(tplt))