Keep '#' in unquoted CLI values instead of treating it as a comment

fire/parser.py

@@ -16,7 +16,9 @@
 
 import argparse
 import ast
+import io
 import sys
+import tokenize
 
 if sys.version_info[0:2] < (3, 8):
   _StrNode = ast.Str  # type: ignore  # pylint: disable=no-member  # deprecated but needed for Python < 3.8
@@ -68,12 +70,29 @@ def DefaultParseValue(value):
   Returns:
     The parsed value, of the type determined most appropriate.
   """
-  # Note: _LiteralEval will treat '#' as the start of a comment.
   try:
-    return _LiteralEval(value)
+    parsed = _LiteralEval(value)
   except (SyntaxError, ValueError):
     # If _LiteralEval can't parse the value, treat it as a string.
     return value
+  # ast.parse treats '#' as a comment, so 'hi#there' would become 'hi'.
+  # CLI values like hashes and fragments should keep the '#'. If a '#' was
+  # tokenized as a comment rather than as part of a string literal, keep the
+  # original value. Quoted forms such as '"0#comments"' still parse as usual.
+  if _HasCommentToken(value):
+    return value
+  return parsed
+
+
+def _HasCommentToken(value):
+  """Return whether tokenize treats a '#' in value as a Python comment."""
+  if '#' not in value:
+    return False
+  try:
+    tokens = tokenize.generate_tokens(io.StringIO(value).readline)
+    return any(token.type == tokenize.COMMENT for token in tokens)
+  except (tokenize.TokenError, IndentationError, SyntaxError):
+    return False
 
 
 def _LiteralEval(value):

fire/parser_fuzz_test.py

@@ -44,7 +44,7 @@ class ParserFuzzTest(testutils.BaseTestCase):
   @example('{test:a,b:(c,d)}')  # Note: Edit distance may be high for dicts.
   @example('0,')
   @example('#')
-  @example('A#00000')  # Note: '#'' is treated as a comment.
+  @example('A#00000')  # Kept as a string; '#' is not stripped.
   @example('\x80')  # Note: Causes UnicodeDecodeError.
   @example(100 * '[' + '0')  # Note: Causes MemoryError.
   @example('\r\r\r\r1\r\r')

fire/parser_test.py

@@ -119,8 +119,12 @@ def testDefaultParseValueNestedContainers(self):
 
   def testDefaultParseValueComments(self):
     self.assertEqual(parser.DefaultParseValue('"0#comments"'), '0#comments')
-    # Comments are stripped. This behavior may change in the future.
-    self.assertEqual(parser.DefaultParseValue('0#comments'), 0)
+    # A '#' outside a string used to be stripped as a Python comment
+    # ('0#comments' -> 0, 'hi#there' -> 'hi'). Keep the original value so
+    # CLI args with hashes are not truncated. See issue #338.
+    self.assertEqual(parser.DefaultParseValue('0#comments'), '0#comments')
+    self.assertEqual(parser.DefaultParseValue('hi#there'), 'hi#there')
+    self.assertEqual(parser.DefaultParseValue('path#ref'), 'path#ref')
 
   def testDefaultParseValueBadLiteral(self):
     # If it can't be parsed, we treat it as a string. This behavior may change.