Merge branch 'main' into feature/xls-support
This commit is contained in:
commit
3acd5067f8
7 changed files with 101 additions and 3 deletions
|
|
@ -6,7 +6,10 @@
|
||||||
// Sets the run context to one level up instead of the .devcontainer folder.
|
// Sets the run context to one level up instead of the .devcontainer folder.
|
||||||
"context": "..",
|
"context": "..",
|
||||||
// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
|
// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
|
||||||
"dockerfile": "../Dockerfile"
|
"dockerfile": "../Dockerfile",
|
||||||
|
"args": {
|
||||||
|
"INSTALL_GIT": "true"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
// Features to add to the dev container. More info: https://containers.dev/features.
|
// Features to add to the dev container. More info: https://containers.dev/features.
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,11 @@ FROM python:3.13-slim-bullseye
|
||||||
|
|
||||||
USER root
|
USER root
|
||||||
|
|
||||||
|
ARG INSTALL_GIT=false
|
||||||
|
RUN if [ "$INSTALL_GIT" = "true" ]; then \
|
||||||
|
apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*; \
|
||||||
|
fi
|
||||||
|
|
||||||
# Runtime dependency
|
# Runtime dependency
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
ffmpeg \
|
ffmpeg \
|
||||||
|
|
|
||||||
|
|
@ -125,7 +125,7 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio
|
||||||
|
|
||||||
### How to Contribute
|
### How to Contribute
|
||||||
|
|
||||||
You can help by looking at issues or helping review PRs. Any issue or PR is welcome, but we have also marked some as 'open for contribution' and 'open for reviewing' to help faciliate community contributions. These are ofcourse just suggestions and you are welcome to contribute in any way you like.
|
You can help by looking at issues or helping review PRs. Any issue or PR is welcome, but we have also marked some as 'open for contribution' and 'open for reviewing' to help facilitate community contributions. These are ofcourse just suggestions and you are welcome to contribute in any way you like.
|
||||||
|
|
||||||
|
|
||||||
<div align="center">
|
<div align="center">
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,7 @@ dependencies = [
|
||||||
"pdfminer.six",
|
"pdfminer.six",
|
||||||
"puremagic",
|
"puremagic",
|
||||||
"pydub",
|
"pydub",
|
||||||
|
"olefile",
|
||||||
"youtube-transcript-api",
|
"youtube-transcript-api",
|
||||||
"SpeechRecognition",
|
"SpeechRecognition",
|
||||||
"pathvalidate",
|
"pathvalidate",
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ from warnings import warn, resetwarnings, catch_warnings
|
||||||
|
|
||||||
import mammoth
|
import mammoth
|
||||||
import markdownify
|
import markdownify
|
||||||
|
import olefile
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import pdfminer
|
import pdfminer
|
||||||
import pdfminer.high_level
|
import pdfminer.high_level
|
||||||
|
|
@ -33,6 +34,7 @@ from bs4 import BeautifulSoup
|
||||||
from charset_normalizer import from_path
|
from charset_normalizer import from_path
|
||||||
|
|
||||||
# Optional Transcription support
|
# Optional Transcription support
|
||||||
|
IS_AUDIO_TRANSCRIPTION_CAPABLE = False
|
||||||
try:
|
try:
|
||||||
# Using warnings' catch_warnings to catch
|
# Using warnings' catch_warnings to catch
|
||||||
# pydub's warning of ffmpeg or avconv missing
|
# pydub's warning of ffmpeg or avconv missing
|
||||||
|
|
@ -1100,6 +1102,79 @@ class ImageConverter(MediaConverter):
|
||||||
return response.choices[0].message.content
|
return response.choices[0].message.content
|
||||||
|
|
||||||
|
|
||||||
|
class OutlookMsgConverter(DocumentConverter):
|
||||||
|
"""Converts Outlook .msg files to markdown by extracting email metadata and content.
|
||||||
|
|
||||||
|
Uses the olefile package to parse the .msg file structure and extract:
|
||||||
|
- Email headers (From, To, Subject)
|
||||||
|
- Email body content
|
||||||
|
"""
|
||||||
|
|
||||||
|
def convert(
|
||||||
|
self, local_path: str, **kwargs: Any
|
||||||
|
) -> Union[None, DocumentConverterResult]:
|
||||||
|
# Bail if not a MSG file
|
||||||
|
extension = kwargs.get("file_extension", "")
|
||||||
|
if extension.lower() != ".msg":
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
msg = olefile.OleFileIO(local_path)
|
||||||
|
# Extract email metadata
|
||||||
|
md_content = "# Email Message\n\n"
|
||||||
|
|
||||||
|
# Get headers
|
||||||
|
headers = {
|
||||||
|
"From": self._get_stream_data(msg, "__substg1.0_0C1F001F"),
|
||||||
|
"To": self._get_stream_data(msg, "__substg1.0_0E04001F"),
|
||||||
|
"Subject": self._get_stream_data(msg, "__substg1.0_0037001F"),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Add headers to markdown
|
||||||
|
for key, value in headers.items():
|
||||||
|
if value:
|
||||||
|
md_content += f"**{key}:** {value}\n"
|
||||||
|
|
||||||
|
md_content += "\n## Content\n\n"
|
||||||
|
|
||||||
|
# Get email body
|
||||||
|
body = self._get_stream_data(msg, "__substg1.0_1000001F")
|
||||||
|
if body:
|
||||||
|
md_content += body
|
||||||
|
|
||||||
|
msg.close()
|
||||||
|
|
||||||
|
return DocumentConverterResult(
|
||||||
|
title=headers.get("Subject"), text_content=md_content.strip()
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise FileConversionException(
|
||||||
|
f"Could not convert MSG file '{local_path}': {str(e)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _get_stream_data(
|
||||||
|
self, msg: olefile.OleFileIO, stream_path: str
|
||||||
|
) -> Union[str, None]:
|
||||||
|
"""Helper to safely extract and decode stream data from the MSG file."""
|
||||||
|
try:
|
||||||
|
if msg.exists(stream_path):
|
||||||
|
data = msg.openstream(stream_path).read()
|
||||||
|
# Try UTF-16 first (common for .msg files)
|
||||||
|
try:
|
||||||
|
return data.decode("utf-16-le").strip()
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
# Fall back to UTF-8
|
||||||
|
try:
|
||||||
|
return data.decode("utf-8").strip()
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
# Last resort - ignore errors
|
||||||
|
return data.decode("utf-8", errors="ignore").strip()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
class ZipConverter(DocumentConverter):
|
class ZipConverter(DocumentConverter):
|
||||||
"""Converts ZIP files to markdown by extracting and converting all contained files.
|
"""Converts ZIP files to markdown by extracting and converting all contained files.
|
||||||
|
|
||||||
|
|
@ -1310,6 +1385,7 @@ class MarkItDown:
|
||||||
self.register_page_converter(IpynbConverter())
|
self.register_page_converter(IpynbConverter())
|
||||||
self.register_page_converter(PdfConverter())
|
self.register_page_converter(PdfConverter())
|
||||||
self.register_page_converter(ZipConverter())
|
self.register_page_converter(ZipConverter())
|
||||||
|
self.register_page_converter(OutlookMsgConverter())
|
||||||
|
|
||||||
def convert(
|
def convert(
|
||||||
self, source: Union[str, requests.Response, Path], **kwargs: Any
|
self, source: Union[str, requests.Response, Path], **kwargs: Any
|
||||||
|
|
|
||||||
BIN
tests/test_files/test_outlook_msg.msg
vendored
Normal file
BIN
tests/test_files/test_outlook_msg.msg
vendored
Normal file
Binary file not shown.
|
|
@ -69,6 +69,15 @@ DOCX_TEST_STRINGS = [
|
||||||
"AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
|
"AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
MSG_TEST_STRINGS = [
|
||||||
|
"# Email Message",
|
||||||
|
"**From:** test.sender@example.com",
|
||||||
|
"**To:** test.recipient@example.com",
|
||||||
|
"**Subject:** Test Email Message",
|
||||||
|
"## Content",
|
||||||
|
"This is the body of the test email message",
|
||||||
|
]
|
||||||
|
|
||||||
DOCX_COMMENT_TEST_STRINGS = [
|
DOCX_COMMENT_TEST_STRINGS = [
|
||||||
"314b0a30-5b04-470b-b9f7-eed2c2bec74a",
|
"314b0a30-5b04-470b-b9f7-eed2c2bec74a",
|
||||||
"49e168b7-d2ae-407f-a055-2167576f39a1",
|
"49e168b7-d2ae-407f-a055-2167576f39a1",
|
||||||
|
|
@ -244,6 +253,10 @@ def test_markitdown_local() -> None:
|
||||||
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_mskanji.csv"))
|
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_mskanji.csv"))
|
||||||
validate_strings(result, CSV_CP932_TEST_STRINGS)
|
validate_strings(result, CSV_CP932_TEST_STRINGS)
|
||||||
|
|
||||||
|
# Test MSG (Outlook email) processing
|
||||||
|
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_outlook_msg.msg"))
|
||||||
|
validate_strings(result, MSG_TEST_STRINGS)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(
|
@pytest.mark.skipif(
|
||||||
skip_exiftool,
|
skip_exiftool,
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue