Title
+Body text.
+diff --git a/packages/markitdown/src/markitdown/__main__.py b/packages/markitdown/src/markitdown/__main__.py index cfb1c6e..2474558 100644 --- a/packages/markitdown/src/markitdown/__main__.py +++ b/packages/markitdown/src/markitdown/__main__.py @@ -42,8 +42,8 @@ def main(): OR markitdown example.pdf > example.md - """ - ).strip(), + """.strip(), + ), ) parser.add_argument( @@ -78,6 +78,15 @@ def main(): help="Provide a hint about the file's charset (e.g, UTF-8).", ) + # New CSS selector flag + parser.add_argument( + "-s", + "--selector", + metavar="CSS", + help="Only convert HTML nodes matching this CSS selector (e.g. 'article.main').", + default=None, + ) + parser.add_argument( "-d", "--use-docintel", @@ -187,22 +196,27 @@ def main(): else: markitdown = MarkItDown(enable_plugins=args.use_plugins) + # Pass selector through to conversion if args.filename is None: result = markitdown.convert_stream( sys.stdin.buffer, stream_info=stream_info, keep_data_uris=args.keep_data_uris, + selector=args.selector, ) else: result = markitdown.convert( - args.filename, stream_info=stream_info, keep_data_uris=args.keep_data_uris + args.filename, + stream_info=stream_info, + keep_data_uris=args.keep_data_uris, + selector=args.selector, ) _handle_output(args, result) + def _handle_output(args, result: DocumentConverterResult): + """Handle output to stdout or file""" -def _handle_output(args, result: DocumentConverterResult): - """Handle output to stdout or file""" if args.output: with open(args.output, "w", encoding="utf-8") as f: f.write(result.markdown) diff --git a/packages/markitdown/src/markitdown/converters/_html_converter.py b/packages/markitdown/src/markitdown/converters/_html_converter.py index dabb0d7..007f8f8 100644 --- a/packages/markitdown/src/markitdown/converters/_html_converter.py +++ b/packages/markitdown/src/markitdown/converters/_html_converter.py @@ -48,13 +48,24 @@ class HtmlConverter(DocumentConverter): encoding = "utf-8" if stream_info.charset is None else stream_info.charset soup = BeautifulSoup(file_stream, "html.parser", from_encoding=encoding) + # ——— CSS selector scoping ——— + selector = kwargs.pop("selector", None) + if selector: + nodes = soup.select(selector) + if not nodes: + raise ValueError(f"No elements match selector: {selector}") + scoped = BeautifulSoup("", "html.parser") + for node in nodes: + scoped.append(node) + soup = scoped + # —————————————————————————— + # Remove javascript and style blocks for script in soup(["script", "style"]): script.extract() # Print only the main content body_elm = soup.find("body") - webpage_text = "" if body_elm: webpage_text = _CustomMarkdownify(**kwargs).convert_soup(body_elm) else: @@ -71,7 +82,7 @@ class HtmlConverter(DocumentConverter): ) def convert_string( - self, html_content: str, *, url: Optional[str] = None, **kwargs + self, html_content: str, *, url: Optional[str] = None, **kwargs: Any ) -> DocumentConverterResult: """ Non-standard convenience method to convert a string to markdown. diff --git a/packages/markitdown/tests/test_html_selector.py b/packages/markitdown/tests/test_html_selector.py new file mode 100644 index 0000000..4027148 --- /dev/null +++ b/packages/markitdown/tests/test_html_selector.py @@ -0,0 +1,76 @@ +import io +import pytest +from markitdown.converters._html_converter import HtmlConverter +from markitdown._stream_info import StreamInfo + +# Sample HTML to test selector scoping +SAMPLE_HTML = """ + +
+Body text.
+