# coding=utf-8
import logging
import os
import time
import re
import requests
from bs4 import BeautifulSoup
import markdown2
import pdfkit
import sys
import hashlib
from urllib.parse import urljoin, urlparse
from io import BytesIO
from PIL import Image
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class OdooMarkdownCrawler:
    """
    Odoo文檔爬蟲(chóng)，生成Markdown格式，最后轉(zhuǎn)換為PDF
    """
    def __init__(self, base_url, output_name="Odoo18_User_Tutorial"):
        """
        初始化爬蟲(chóng)
        :param base_url: 基礎(chǔ)URL
        :param output_name: 輸出文件名
        """
        self.base_url = base_url
        self.output_name = output_name
        self.domain = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(self.base_url))
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Connection': 'keep-alive',
        }
        self.session = requests.Session()
        self.session.headers.update(self.headers)
        # 創(chuàng)建目錄
        self.temp_dir = "temp_markdown_files"
        self.images_dir = os.path.join(self.temp_dir, "images")
        os.makedirs(self.temp_dir, exist_ok=True)
        os.makedirs(self.images_dir, exist_ok=True)
        # 存儲(chǔ)已訪(fǎng)問(wèn)的URL，避免重復(fù)
        self.visited_urls = set()
        # 存儲(chǔ)Markdown文件
        self.markdown_files = []
        # 存儲(chǔ)所有頁(yè)面內(nèi)容
        self.all_pages = []
        logger.info(f"爬蟲(chóng)初始化完成，基礎(chǔ)URL: {self.base_url}")
    def get_page_content(self, url, max_retries=3):
        """
        獲取頁(yè)面內(nèi)容，帶重試機(jī)制
        """
        for attempt in range(max_retries):
            try:
                logger.info(f"正在獲取頁(yè)面: {url} (嘗試 {attempt + 1}/{max_retries})")
                response = self.session.get(url, timeout=30)
                if response.status_code == 200:
                    return response.content
                elif response.status_code == 429:
                    logger.warning(f"請(qǐng)求過(guò)于頻繁，等待 3 秒后重試...")
                    time.sleep(3)
                else:
                    logger.warning(f"獲取頁(yè)面失敗，狀態(tài)碼: {response.status_code}")
                    if response.status_code == 404:
                        return None
                time.sleep(1)
            except requests.exceptions.RequestException as e:
                logger.error(f"請(qǐng)求異常: {e}")
                if attempt < max_retries - 1:
                    logger.info(f"等待 2 秒后重試...")
                    time.sleep(2)
        logger.error(f"重試 {max_retries} 次后仍無(wú)法獲取頁(yè)面: {url}")
        return None
    def discover_links(self, start_url):
        """
        從起始頁(yè)面發(fā)現(xiàn)所有相關(guān)鏈接
        """
        logger.info(f"開(kāi)始發(fā)現(xiàn)鏈接: {start_url}")
        content = self.get_page_content(start_url)
        if not content:
            logger.error("無(wú)法獲取起始頁(yè)面內(nèi)容")
            return []
        soup = BeautifulSoup(content, 'html.parser')
        # 查找所有文檔鏈接
        links = []
        visited = set()
        # 方法1：查找導(dǎo)航菜單中的鏈接
        nav_elements = soup.find_all(['nav', 'ul', 'div'], class_=re.compile('nav|menu|toc|sidebar', re.I))
        for nav in nav_elements:
            for a_tag in nav.find_all('a', href=True):
                href = a_tag['href'].strip()
                text = a_tag.get_text(strip=True)
                if not href or href.startswith('#') or 'javascript:' in href or href.startswith('mailto:'):
                    continue
                # 構(gòu)建完整URL
                full_url = urljoin(self.base_url, href)
                # 檢查是否是文檔頁(yè)面
                if '/documentation/18.0/zh_CN/' in full_url and full_url.endswith('.html'):
                    if full_url not in visited:
                        visited.add(full_url)
                        links.append({
                            'url': full_url,
                            'title': text,
                            'section': '導(dǎo)航菜單'
                        })
                        logger.debug(f"發(fā)現(xiàn)導(dǎo)航鏈接: {text} -> {full_url}")
        # 方法2：查找主要內(nèi)容中的鏈接
        main_content = soup.find('main') or soup.find(class_=re.compile('content|article|body', re.I))
        if main_content:
            for a_tag in main_content.find_all('a', href=True):
                href = a_tag['href'].strip()
                text = a_tag.get_text(strip=True)
                if not href or href.startswith('#') or 'javascript:' in href:
                    continue
                full_url = urljoin(self.base_url, href)
                if '/documentation/18.0/zh_CN/' in full_url and full_url.endswith('.html'):
                    if full_url not in visited:
                        visited.add(full_url)
                        links.append({
                            'url': full_url,
                            'title': text,
                            'section': '主要內(nèi)容'
                        })
                        logger.debug(f"發(fā)現(xiàn)內(nèi)容鏈接: {text} -> {full_url}")
        # 方法3：添加一些常見(jiàn)的核心頁(yè)面（備用）
        if not links:
            logger.warning("未發(fā)現(xiàn)鏈接，使用備用核心頁(yè)面列表")
            core_pages = [
                'applications.html',
                'applications/essentials/activities.html',
                'applications/general/overview.html',
                'applications/sales.html',
                'applications/inventory.html',
                'applications/crm.html',
                'applications/accounting.html',
                'applications/reporting.html',
                'applications/contacts.html',
                'applications/general/search.html'
            ]
            for page in core_pages:
                full_url = f"https://www.odooai.cn/documentation/18.0/zh_CN/{page}"
                title = page.replace('.html', '').replace('/', ' ').replace('_', ' ').title()
                links.append({
                    'url': full_url,
                    'title': title,
                    'section': '備用核心頁(yè)面'
                })
        logger.info(f"共發(fā)現(xiàn) {len(links)} 個(gè)頁(yè)面鏈接")
        return links
    def download_image(self, img_url, page_url):
        """
        下載圖片并返回本地路徑
        """
        try:
            if not img_url.startswith('http'):
                img_url = urljoin(page_url, img_url)
            logger.debug(f"下載圖片: {img_url}")
            # 獲取圖片內(nèi)容
            response = self.session.get(img_url, timeout=15)
            if response.status_code != 200:
                logger.warning(f"圖片下載失敗，狀態(tài)碼: {response.status_code}")
                return None
            # 檢查文件類(lèi)型
            content_type = response.headers.get('Content-Type', '')
            if not content_type.startswith('image/'):
                logger.warning(f"非圖片內(nèi)容類(lèi)型: {content_type}")
                return None
            # 生成文件名
            img_hash = hashlib.md5(response.content).hexdigest()[:8]
            ext = '.jpg' if 'jpeg' in content_type else '.png' if 'png' in content_type else '.gif'
            filename = f"img_{img_hash}{ext}"
            filepath = os.path.join(self.images_dir, filename)
            # 保存圖片
            with open(filepath, 'wb') as f:
                f.write(response.content)
            # 返回相對(duì)路徑
            return f"./images/{filename}"
        except Exception as e:
            logger.error(f"圖片下載失敗 {img_url}: {e}")
            return None
    def process_element(self, element, page_url, depth=0):
        """
        遞歸處理HTML元素，保持原始結(jié)構(gòu)
        """
        markdown_lines = []
        # 處理圖片
        if element.name == 'img':
            img_url = element.get('src', '')
            if img_url:
                local_path = self.download_image(img_url, page_url)
                if local_path:
                    alt_text = element.get('alt', '圖片')
                    markdown_lines.append(f"![{alt_text}]({local_path})\n")
            return markdown_lines
        # 處理文本節(jié)點(diǎn)
        if element.name is None:
            text = element.strip()
            if text:
                # 保持縮進(jìn)
                indent = '  ' * depth
                markdown_lines.append(f"{indent}{text}")
            return markdown_lines
        # 處理段落
        if element.name == 'p':
            text = element.get_text(strip=True)
            if text:
                markdown_lines.append(f"{text}\n")
            return markdown_lines
        # 處理標(biāo)題
        if element.name.startswith('h'):
            level = int(element.name[1])
            text = element.get_text(strip=True)
            if text:
                markdown_lines.append(f"{'#' * level} {text}\n")
            return markdown_lines
        # 處理列表
        if element.name in ['ul', 'ol']:
            list_type = '-' if element.name == 'ul' else '1.'
            for i, li in enumerate(element.find_all('li', recursive=False)):
                items = self.process_element(li, page_url, depth + 1)
                if items:
                    # 處理多行列表項(xiàng)
                    for j, item in enumerate(items):
                        if j == 0:
                            markdown_lines.append(f"{'  ' * depth}{list_type} {item}")
                        else:
                            markdown_lines.append(f"{'  ' * depth}   {item}")
            markdown_lines.append("\n")
            return markdown_lines
        # 處理表格
        if element.name == 'table':
            return [self.table_to_markdown(element)]
        # 處理代碼塊
        if element.name == 'pre':
            code = element.get_text(strip=True)
            if code:
                markdown_lines.append(f"```\n[code]\n```\n")
            return markdown_lines
        # 處理引用塊
        if element.name == 'blockquote':
            text = element.get_text(strip=True)
            if text:
                for line in text.split('\n'):
                    markdown_lines.append(f"> {line}\n")
            return markdown_lines
        # 處理其他元素
        if element.contents:
            for child in element.children:
                lines = self.process_element(child, page_url, depth + 1)
                markdown_lines.extend(lines)
        return markdown_lines
    def html_to_markdown(self, html_content, page_url, title):
        """
        將HTML內(nèi)容轉(zhuǎn)換為Markdown格式，保持元素位置
        """
        soup = BeautifulSoup(html_content, 'html.parser')
        # 移除不需要的元素
        for selector in ['nav', 'footer', '.o_header', '.o_footer', '.o_search_header', 
                        '.o_mobile-overlay', '.header', '.footer', '.edit-page',
                        '.share-buttons', '.search-box', '.o_topbar', '.o_bottombar',
                        '.o_edit_page', '.o_share_btn', '.o_search', '.edit-button', '.feedback',
                        '.o_actions', '.toc', '.sidebar']:
            for element in soup.select(selector):
                element.decompose()
        # 找到主要內(nèi)容區(qū)域
        main_content = soup.find('main') or soup.find(class_=re.compile('content|article|body', re.I)) or soup.find(id=re.compile('content|main'))
        if not main_content:
            logger.warning("未找到主要內(nèi)容區(qū)域，使用整個(gè)body")
            main_content = soup.body
        # 提取標(biāo)題
        page_title = title
        if not page_title:
            h1 = soup.find('h1')
            if h1:
                page_title = h1.get_text(strip=True)
        # 創(chuàng)建Markdown內(nèi)容
        markdown_content = []
        # 添加標(biāo)題
        if page_title:
            markdown_content.append(f"# {page_title}\n")
        # 添加元信息
        markdown_content.append(f"**來(lái)源:** [{page_url}]({page_url})\n")
        markdown_content.append(f"**爬取時(shí)間:** {time.strftime('%Y-%m-%d %H:%M:%S')}\n")
        markdown_content.append("---\n")
        # 處理所有內(nèi)容元素，保持原始位置
        for element in main_content.children:
            lines = self.process_element(element, page_url)
            markdown_content.extend(lines)
        return '\n'.join(markdown_content)
    def table_to_markdown(self, table_element):
        """
        將HTML表格轉(zhuǎn)換為Markdown表格
        """
        if not table_element.find('tr'):
            return ""
        rows = table_element.find_all('tr')
        if not rows:
            return ""
        markdown_rows = []
        for i, row in enumerate(rows):
            cells = row.find_all(['td', 'th'])
            if not cells:
                continue
            row_data = [cell.get_text(strip=True) for cell in cells]
            if i == 0:  # 表頭
                markdown_rows.append('| ' + ' | '.join(row_data) + ' |')
                markdown_rows.append('| ' + ' | '.join(['---'] * len(row_data)) + ' |')
            else:
                markdown_rows.append('| ' + ' | '.join(row_data) + ' |')
        return '\n'.join(markdown_rows) + '\n'
    def save_markdown_file(self, content, filename):
        """
        保存Markdown文件
        """
        # 處理文件名中的特殊字符
        safe_filename = re.sub(r'[\\/*?:"<>|]', '', filename)
        safe_filename = safe_filename.replace(' ', '_')
        safe_filename = safe_filename[:50] + '.md'
        filepath = os.path.join(self.temp_dir, safe_filename)
        try:
            with open(filepath, 'w', encoding='utf-8') as f:
                f.write(content)
            logger.info(f"已保存Markdown文件: {safe_filename}")
            return filepath
        except Exception as e:
            logger.error(f"保存文件失敗 {safe_filename}: {e}")
            return None
    def crawl_pages(self, page_links):
        """
        爬取所有頁(yè)面并轉(zhuǎn)換為Markdown
        """
        logger.info(f"開(kāi)始爬取 {len(page_links)} 個(gè)頁(yè)面...")
        for i, page_info in enumerate(page_links):
            url = page_info['url']
            title = page_info['title'] or f"頁(yè)面_{i+1}"
            # 檢查是否已訪(fǎng)問(wèn)
            if url in self.visited_urls:
                logger.info(f"跳過(guò)已訪(fǎng)問(wèn)的頁(yè)面: {url}")
                continue
            self.visited_urls.add(url)
            # 獲取頁(yè)面內(nèi)容
            html_content = self.get_page_content(url)
            if not html_content:
                # 嘗試常見(jiàn)的URL變體
                url_variants = [
                    url.replace('/zh_CN/', '/zh_CN/applications/'),
                    url.replace('/zh_CN/', '/zh_CN/applications/general/'),
                    url.replace('/zh_CN/', '/zh_CN/applications/essentials/'),
                    url.replace('/zh_CN/', '/zh_CN/applications/sales/'),
                    url.replace('/zh_CN/', '/zh_CN/applications/inventory/'),
                    url.replace('/zh_CN/', '/zh_CN/applications/crm/'),
                    url.replace('/zh_CN/', '/zh_CN/applications/accounting/'),
                ]
                for variant_url in url_variants:
                    logger.info(f"嘗試URL變體: {variant_url}")
                    html_content = self.get_page_content(variant_url)
                    if html_content:
                        url = variant_url
                        break
            if not html_content:
                logger.warning(f"無(wú)法獲取頁(yè)面內(nèi)容，跳過(guò): {url}")
                continue
            # 轉(zhuǎn)換為Markdown
            markdown_content = self.html_to_markdown(html_content, url, title)
            # 生成文件名
            safe_title = re.sub(r'[\\/*?:"<>|]', '', title)[:30]
            safe_title = safe_title.replace(' ', '_')
            filename = f"{i+1:03d}_{safe_title}"
            # 保存Markdown文件
            filepath = self.save_markdown_file(markdown_content, filename)
            if filepath:
                self.markdown_files.append(filepath)
                self.all_pages.append({
                    'title': title,
                    'content': markdown_content,
                    'url': url
                })
            # 每3個(gè)頁(yè)面暫停一下，避免請(qǐng)求過(guò)于頻繁
            if (i + 1) % 3 == 0:
                logger.info(f"已爬取 {i + 1} 個(gè)頁(yè)面，暫停 2 秒...")
                time.sleep(2)
            # 每個(gè)頁(yè)面之間稍作延遲
            time.sleep(1.0)
        logger.info(f"爬取完成，共成功保存 {len(self.markdown_files)} 個(gè)Markdown文件")
    def combine_markdown_files(self):
        """
        合并所有Markdown文件
        """
        if not self.all_pages:
            logger.error("沒(méi)有可合并的Markdown文件")
            return None
        logger.info("開(kāi)始合并Markdown文件...")
        combined_content = []
        # 添加封面
        combined_content.append("# Odoo 18 用戶(hù)教程\n")
        combined_content.append(f"**生成時(shí)間:** {time.strftime('%Y-%m-%d %H:%M:%S')}\n")
        combined_content.append(f"**版本:** 18.0\n")
        combined_content.append(f"**語(yǔ)言:** 簡(jiǎn)體中文\n")
        combined_content.append(f"**總頁(yè)數(shù):** {len(self.all_pages)}\n")
        combined_content.append("---\n")
        # 添加目錄
        combined_content.append("## 目錄\n")
        for i, page in enumerate(self.all_pages):
            title = page['title']
            anchor = re.sub(r'[^\w\s-]', '', title).lower().replace(' ', '-')
            combined_content.append(f"{i+1}. [{title}](#{anchor})")
        combined_content.append("\n")
        # 添加所有頁(yè)面內(nèi)容
        for i, page in enumerate(self.all_pages):
            title = page['title']
            content = page['content']
            # 為每個(gè)頁(yè)面添加分頁(yè)符
            combined_content.append(f"\n\n<!-- 分頁(yè)符 -->\n\n")
            combined_content.append(f"## {title}\n")
            combined_content.append(content)
        combined_content.append("\n\n---\n")
        combined_content.append("**文檔生成完成**\n")
        combined_filepath = os.path.join(self.temp_dir, "combined_document.md")
        try:
            with open(combined_filepath, 'w', encoding='utf-8') as f:
                f.write('\n'.join(combined_content))
            logger.info(f"已保存合并的Markdown文件: {combined_filepath}")
            return combined_filepath
        except Exception as e:
            logger.error(f"保存合并文件失敗: {e}")
            return None
    def convert_markdown_to_html(self, markdown_file):
        """
        將Markdown轉(zhuǎn)換為HTML
        """
        try:
            with open(markdown_file, 'r', encoding='utf-8') as f:
                markdown_content = f.read()
            # 轉(zhuǎn)換為HTML
            html_content = markdown2.markdown(markdown_content, extras=["tables", "fenced-code-blocks", "footnotes"])
            # 添加HTML結(jié)構(gòu)和樣式
            styled_html = f"""
            <!DOCTYPE html>
            <html lang="zh-CN">
            <head>
                <meta charset="UTF-8">
                <meta name="viewport" content="width=device-width, initial-scale=1.0">
                <title>Odoo 18 用戶(hù)教程</title>
                <style>
                    body {{
                        font-family: "Microsoft YaHei", "SimSun", Arial, sans-serif;
                        line-height: 1.6;
                        color: #333;
                        max-width: 1000px;
                        margin: 0 auto;
                        padding: 20px;
                    }}
                    h1, h2, h3, h4, h5, h6 {{
                        color: #2c3e50;
                        margin-top: 1.5em;
                        margin-bottom: 0.5em;
                    }}
                    h1 {{
                        border-bottom: 2px solid #3498db;
                        padding-bottom: 10px;
                        font-size: 2em;
                    }}
                    h2 {{
                        border-left: 4px solid #3498db;
                        padding-left: 10px;
                        font-size: 1.8em;
                    }}
                    a {{
                        color: #3498db;
                        text-decoration: none;
                    }}
                    a:hover {{
                        text-decoration: underline;
                    }}
                    img {{
                        max-width: 100%;
                        height: auto;
                        display: block;
                        margin: 10px auto;
                        border: 1px solid #ddd;
                        padding: 5px;
                        border-radius: 4px;
                    }}
                    pre, code {{
                        background-color: #f8f9fa;
                        padding: 5px 10px;
                        border-radius: 4px;
                        font-family: Consolas, Monaco, "Andale Mono", monospace;
                        font-size: 0.9em;
                        overflow-x: auto;
                    }}
                    pre {{
                        padding: 15px;
                        border-left: 4px solid #3498db;
                        margin: 15px 0;
                        background-color: #f9f9f9;
                    }}
                    table {{
                        border-collapse: collapse;
                        width: 100%;
                        margin: 15px 0;
                        border: 1px solid #ddd;
                    }}
                    th, td {{
                        border: 1px solid #ddd;
                        padding: 8px 12px;
                        text-align: left;
                    }}
                    th {{
                        background-color: #f2f2f2;
                        font-weight: bold;
                    }}
                    tr:nth-child(even) {{
                        background-color: #f9f9f9;
                    }}
                    blockquote {{
                        border-left: 4px solid #3498db;
                        padding-left: 15px;
                        margin: 15px 0;
                        color: #666;
                        background-color: #f8f9fa;
                        border-radius: 0 4px 4px 0;
                    }}
                    .page-break {{
                        page-break-after: always;
                    }}
                </style>
            </head>
            <body>
                {html_content}
            </body>
            </html>
            """
            html_filepath = os.path.join(self.temp_dir, "combined_document.html")
            with open(html_filepath, 'w', encoding='utf-8') as f:
                f.write(styled_html)
            logger.info(f"已保存HTML文件: {html_filepath}")
            return html_filepath
        except Exception as e:
            logger.error(f"Markdown轉(zhuǎn)HTML失敗: {e}")
            return None
    def convert_to_pdf(self, html_file):
        """
        將HTML轉(zhuǎn)換為PDF
        """
        if not html_file or not os.path.exists(html_file):
            logger.error("HTML文件不存在，無(wú)法轉(zhuǎn)換為PDF")
            return False
        logger.info("開(kāi)始轉(zhuǎn)換PDF...")
        try:
            # 配置PDF選項(xiàng)
            options = {
                'page-size': 'A4',
                'margin-top': '0.75in',
                'margin-right': '0.75in',
                'margin-bottom': '0.75in',
                'margin-left': '0.75in',
                'encoding': "UTF-8",
                'no-outline': None,
                'enable-local-file-access': True,
                'footer-center': '[page] / [topage]',
                'footer-font-size': '8',
                'header-center': 'Odoo 18 用戶(hù)教程',
                'header-font-size': '8',
                'header-spacing': '5',
                'footer-spacing': '5',
                'minimum-font-size': '10',
                'zoom': '1.25',
                'quiet': ''
            }
            output_path = f"{self.output_name}.pdf"
            logger.info(f"開(kāi)始生成PDF文件: {output_path}")
            # 轉(zhuǎn)換為PDF
            pdfkit.from_file(html_file, output_path, options=options)
            logger.info(f"PDF轉(zhuǎn)換成功！文件保存為: {output_path}")
            return True
        except Exception as e:
            logger.error(f"PDF轉(zhuǎn)換失敗: {e}")
            logger.error("可能的原因：")
            logger.error("1. 未安裝wkhtmltopdf，請(qǐng)安裝：https://wkhtmltopdf.org/downloads.html")
            logger.error("2. wkhtmltopdf路徑未配置")
            logger.error("3. 文件路徑包含中文或特殊字符")
            logger.error("4. 內(nèi)存不足或文件過(guò)大")
            return False
    def cleanup(self):
        """
        清理臨時(shí)文件
        """
        logger.info("開(kāi)始清理臨時(shí)文件...")
        # 不刪除Markdown文件，保留作為備份
        logger.info("保留Markdown文件作為備份")
        logger.info("清理完成")
    def run(self):
        """
        運(yùn)行爬蟲(chóng)
        """
        start_time = time.time()
        logger.info("Odoo Markdown爬蟲(chóng)啟動(dòng)！")
        # 1. 發(fā)現(xiàn)所有鏈接
        page_links = self.discover_links(self.base_url)
        if not page_links:
            logger.error("未發(fā)現(xiàn)任何頁(yè)面鏈接")
            return
        # 2. 爬取所有頁(yè)面
        self.crawl_pages(page_links)
        if not self.all_pages:
            logger.error("沒(méi)有成功爬取任何頁(yè)面")
            return
        # 3. 合并Markdown文件
        combined_md_file = self.combine_markdown_files()
        if not combined_md_file:
            logger.error("合并Markdown文件失敗")
            return
        # 4. 轉(zhuǎn)換為HTML
        html_file = self.convert_markdown_to_html(combined_md_file)
        if not html_file:
            logger.error("轉(zhuǎn)換為HTML失敗")
            return
        # 5. 轉(zhuǎn)換為PDF
        self.convert_to_pdf(html_file)
        # 6. 清理
        self.cleanup()
        total_time = time.time() - start_time
        logger.info(f"爬蟲(chóng)運(yùn)行完成！總耗時(shí): {total_time:.2f} 秒")
        logger.info(f"共處理頁(yè)面: {len(page_links)} 個(gè)")
        logger.info(f"成功保存頁(yè)面: {len(self.all_pages)} 個(gè)")
        # 顯示輸出文件信息
        logger.info(f"輸出文件:")
        logger.info(f"- Markdown文件: {self.temp_dir}/")
        logger.info(f"- PDF文件: {self.output_name}.pdf")
def main():
    """
    主函數(shù)
    """
    # 配置參數(shù)
    base_url = "https://www.odooai.cn/documentation/18.0/zh_CN/applications.html"
    output_name = "Odoo18_User_Tutorial"
    # 創(chuàng)建爬蟲(chóng)實(shí)例
    crawler = OdooMarkdownCrawler(base_url, output_name)
    # 運(yùn)行爬蟲(chóng)
    crawler.run()
if __name__ == "__main__":
    main()

# Odoo 18 文檔通常位于 odoo/documentation 目錄下
git clone https://github.com/odoo/odoo.git
cd odoo/documentation

# 安裝 Sphinx
pip install sphinx
# 安裝 sphinx-markdown-builder（用于導(dǎo)出 Markdown）
pip install sphinx-markdown-builder
# 根據(jù) Odoo 文檔的 requirements.txt 安裝其他依賴(lài)（如有）
# pip install -r requirements.txt

# 在 conf.py 中添加擴(kuò)展
extensions = [
    'sphinx_markdown_builder',  # 關(guān)鍵：添加 Markdown 導(dǎo)出支持
    # 其他 Odoo 文檔需要的擴(kuò)展...
]
# 如果需要保留中文內(nèi)容，設(shè)置語(yǔ)言
language = 'zh_CN'

# 方法一：使用 sphinx-build 直接生成 Markdown
sphinx-build -b markdown . ./build/markdown
# 方法二：如果文檔已構(gòu)建為 HTML，也可以從 HTML 轉(zhuǎn)換
# sphinx-build -b markdown ./_build/html ./build/markdown_from_html

# 查看生成的 Markdown 文件
ls ./build/markdown/

import requests
from html_to_markdown import convert, ConversionOptions
from urllib.parse import urljoin, urlparse
import time
import os
from bs4 import BeautifulSoup
# 配置
BASE_URL = "https://www.odoo.com/documentation/18.0/zh_CN/"
OUTPUT_DIR = "./odoo18_markdown"
# 創(chuàng)建輸出目錄
os.makedirs(OUTPUT_DIR, exist_ok=True)
# 配置轉(zhuǎn)換選項(xiàng)
options = ConversionOptions(
    heading_style="atx",      # 使用 ATX 風(fēng)格標(biāo)題（# 開(kāi)頭）
    list_indent_width=2,      # 列表縮進(jìn)寬度
    extract_metadata=True,    # 提取元數(shù)據(jù)
    extract_tables=True,      # 提取表格結(jié)構(gòu)
)

def fetch_and_convert(url):
    """獲取頁(yè)面HTML并轉(zhuǎn)換為Markdown"""
    try:
        resp = requests.get(url, timeout=30, headers={
            'User-Agent': 'Mozilla/5.0 (compatible; OdooDocBot/1.0)'
        })
        resp.raise_for_status()
        result = convert(resp.text, options)
        markdown = result["content"]
        return markdown
    except Exception as e:
        print(f"Error processing {url}: {e}")
        return None

import asyncio
from markgrab import extract
async def extract_markdown(url):
    result = await extract(
        url,
        max_chars=50000,           # 限制輸出長(zhǎng)度
        use_browser=False,         # 靜態(tài)頁(yè)面不需要瀏覽器渲染
    )
    return result.markdown
# 運(yùn)行異步函數(shù)
markdown = asyncio.run(extract_markdown("https://www.odoo.com/documentation/18.0/zh_CN/index.html"))

import requests
from html_to_markdown import convert, ConversionOptions
from urllib.parse import urljoin, urlparse
import time
import os
from bs4 import BeautifulSoup
BASE_URL = "https://www.odoo.com/documentation/18.0/zh_CN/"
OUTPUT_DIR = "./odoo18_markdown"
visited = set()
os.makedirs(OUTPUT_DIR, exist_ok=True)
options = ConversionOptions(
    heading_style="atx",
    list_indent_width=2,
    extract_metadata=True,
    extract_tables=True,
)
def get_page_links(html, current_url):
    """從頁(yè)面中提取同域文檔鏈接"""
    soup = BeautifulSoup(html, 'html.parser')
    links = set()
    for a in soup.find_all('a', href=True):
        href = urljoin(current_url, a['href'])
        # 過(guò)濾：同域、文檔路徑、非錨點(diǎn)、非外部資源
        if (BASE_URL in href and not href.endswith(('.png', '.jpg', '.pdf')) 
            and '#' not in href and href not in visited):
            links.add(href)
    return links
def crawl_and_convert(url):
    if url in visited:
        return
    visited.add(url)
    print(f"Processing: {url}")
    try:
        resp = requests.get(url, timeout=30, headers={
            'User-Agent': 'Mozilla/5.0 (compatible; OdooDocBot/1.0)'
        })
        resp.raise_for_status()
        # 轉(zhuǎn)換主內(nèi)容為 Markdown
        result = convert(resp.text, options)
        markdown = result["content"]
        # 保存 Markdown 文件
        filename = url.replace(BASE_URL, '').replace('/', '_') or 'index'
        filepath = os.path.join(OUTPUT_DIR, f"{filename}.md")
        with open(filepath, 'w', encoding='utf-8') as f:
            f.write(f"# Source: {url}\n\n")
            f.write(markdown)
        # 遞歸處理子鏈接
        for link in get_page_links(resp.text, url):
            if link not in visited:
                time.sleep(1)  # 禮貌性延遲，避免請(qǐng)求過(guò)快
                crawl_and_convert(link)
    except Exception as e:
        print(f"Error: {e}")
# 開(kāi)始爬取
crawl_and_convert(BASE_URL)
print(f"Done. Saved to {OUTPUT_DIR}")

from playwright.sync_api import sync_playwright
def fetch_with_playwright(url):
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(url, wait_until="networkidle")
        html = page.content()
        browser.close()
        return html