Expose quote preservation and entity decoding in the CLI

docs/cli.rst

@@ -8,7 +8,7 @@ Here's the usage documentation for the `ftfy` command:
 .. code-block:: text
 
     usage: ftfy [-h] [-o OUTPUT] [-g] [-e ENCODING] [-n NORMALIZATION]
-                [--preserve-entities]
+                [--preserve-entities | --unescape-entities] [--preserve-quotes]
                 [filename]
 
     ftfy (fixes text for you), version 6.0
@@ -32,3 +32,17 @@ Here's the usage documentation for the `ftfy` command:
       --preserve-entities   Leave HTML entities as they are. The default is to
                             decode them, as long as no HTML tags have appeared in
                             the file.
+      --unescape-entities   Decode HTML entities even when HTML tags appear in
+                            the file.
+      --preserve-quotes     Leave curly quotation marks as they are instead of
+                            converting them to ASCII.
+
+For example, to keep curly quotes while decoding HTML entities even in text
+containing ``<`` characters::
+
+    ftfy --preserve-quotes --unescape-entities input.txt -o output.txt
+
+These options set :class:`ftfy.TextFixerConfig`'s ``uncurl_quotes=False`` and
+``unescape_html=True`` respectively. Other fixes, including mojibake repair,
+still run. ``--unescape-entities`` and ``--preserve-entities`` cannot be combined.
+Without either option, HTML entity decoding retains its automatic behavior.

ftfy/cli.py

@@ -5,7 +5,6 @@
 import os
 import sys
 from pathlib import Path
-from typing import Union
 
 from ftfy import TextFixerConfig, __version__, fix_file
 
@@ -84,13 +83,30 @@ def main() -> None:
         default="NFC",
         help='The normalization of Unicode to apply. Defaults to NFC. Can be "none".',
     )
-    parser.add_argument(
+    entity_options = parser.add_mutually_exclusive_group()
+    entity_options.add_argument(
         "--preserve-entities",
-        action="store_true",
+        dest="unescape_html",
+        action="store_const",
+        const=False,
+        default="auto",
         help="Leave HTML entities as they are. The default "
         "is to decode them, as long as no HTML tags have appeared in the file.",
     )
 
+    entity_options.add_argument(
+        "--unescape-entities",
+        dest="unescape_html",
+        action="store_const",
+        const=True,
+        help="Decode HTML entities even when HTML tags appear in the file.",
+    )
+    parser.add_argument(
+        "--preserve-quotes",
+        action="store_true",
+        help="Leave curly quotation marks as they are instead of converting them to ASCII.",
+    )
+
     args = parser.parse_args()
 
     encoding = args.encoding
@@ -116,10 +132,11 @@ def main() -> None:
     if normalization.lower() == "none":
         normalization = None
 
-    unescape_html: Union[str, bool]
-    unescape_html = False if args.preserve_entities else "auto"
-
-    config = TextFixerConfig(unescape_html=unescape_html, normalization=normalization)
+    config = TextFixerConfig(
+        unescape_html=args.unescape_html,
+        uncurl_quotes=not args.preserve_quotes,
+        normalization=normalization,
+    )
 
     try:
         for line in fix_file(file, encoding=encoding, config=config):

tests/test_cli.py

@@ -65,3 +65,42 @@ def test_stdin():
     with TEST_FILENAME.open("rb") as infile:
         output = get_command_output(["ftfy"], stdin=infile)
         assert output == CORRECT_OUTPUT
+
+
+@pytest.mark.parametrize(
+    ("options", "expected"),
+    [
+        ([], '"café" <tag>&amp;\n&amp;\n'),
+        (["--preserve-quotes"], "“café” <tag>&amp;\n&amp;\n"),
+        (["--unescape-entities"], '"café" <tag>&\n&\n'),
+        (["--preserve-entities"], '"café" <tag>&amp;\n&amp;\n'),
+        (["--preserve-quotes", "--unescape-entities"], "“café” <tag>&\n&\n"),
+        (["--preserve-quotes", "--preserve-entities"], "“café” <tag>&amp;\n&amp;\n"),
+    ],
+)
+@pytest.mark.parametrize("use_stdin", [False, True])
+def test_cli_fix_options(tmp_path, options, expected, use_stdin):
+    source = tmp_path / "input.txt"
+    source.write_text("“café” <tag>&amp;\n&amp;\n", encoding="utf-8")
+    if use_stdin:
+        with source.open("rb") as infile:
+            output = get_command_output(["ftfy", *options], stdin=infile)
+    else:
+        destination = tmp_path / "output.txt"
+        get_command_output(["ftfy", *options, source, "-o", destination])
+        output = destination.read_text(encoding="utf-8")
+    if use_stdin:
+        expected = expected.replace("\n", os.linesep)
+    assert output == expected
+
+
+def test_conflicting_entity_options(tmp_path):
+    destination = tmp_path / "output.txt"
+    destination.write_text("keep this file", encoding="utf-8")
+    with pytest.raises(subprocess.CalledProcessError) as exception:
+        get_command_output(
+            ["ftfy", "--preserve-entities", "--unescape-entities", "-o", destination]
+        )
+    assert exception.value.returncode == 2
+    assert "not allowed with argument" in exception.value.output.decode("utf-8")
+    assert destination.read_text(encoding="utf-8") == "keep this file"